inteligr8/SearchServices
1
0
Fork 0
mirror of https://github.com/Alfresco/SearchServices.git synced 2026-09-16 18:12:56 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'feature/SEARCH-2090' into 'release/V2.0.x'

Browse Source 
SolrContentStore removal + fields deduplication

See merge request search_discovery/insightengine!413
This commit is contained in:
Andrea Gazzarini
2020-03-10 14:53:35 +00:00
parent c453a64143 a5fadf92ff
commit 4ae01befdf
187 changed files with 10160 additions and 14225 deletions
Download Patch File Download Diff File Expand all files Collapse all files
e2e-test/generator-alfresco-docker-compose/README.md
+8 -1
View File
@@ -17,6 +17,8 @@ $ tree generators/app/templates/
│   ├── .env
│   ├── docker-compose-ce.yml
│   └── docker-compose-ee.yml
├── empty
│   └── empty
├── images
│   ├── alfresco
│   │   ├── Dockerfile
@@ -111,7 +113,8 @@ When using Community, some different options can be combined:
? Would you like to use HTTP or mTLS for Alfresco-SOLR communication? http
? Would you like to use HTTP or HTTPs for Web Proxy? http
? Would you like to protect the access to SOLR REST API? Yes
? Would you like to use a SOLR Replication (2 nodes in master-slave)? No
? Would you like to use a SOLR Replication? No
? Would you like to compress Get Content responses? No
```
**Note** that when choosing *mTLS* or *HTTPs*, default certificates, truststores and keystores are provided for testing purposes. If you are planning to use this Docker Compose template for real environments, replace these cryptographic stores with another generated by yourself to increase the security of your system.
@@ -180,6 +183,10 @@ Sample configuration is available in [images/share/model/sharding-share-config-c
If *Sharding* is selected, a default `share-config-custom-dev.xml` file with required forms configuration for Sharding custom model will be available in deployment folder. Add your configuration to this file.
## Installing custom addons
The generator will create `amps` and `jars` directories within `alfresco/modules` and `share/modules`. Any amps or jars placed in these directories will be installed in the corresponding container.
## Passing parameters from command line
Default values for options can be specified in the command line, using a `--name=value` pattern. When an options is specified in the command line, the question is not prompted to the user, so you can generate a Docker Compose template with no user interaction.
e2e-test/generator-alfresco-docker-compose/generators/app/index.js
+20 -2
View File
@@ -56,7 +56,7 @@ module.exports = class extends Generator {
type: 'confirm',
name: 'protectSolr',
message: 'Would you like to protect the access to SOLR REST API?',
default: 'true'
default: true
},
{
whenFunction: response => response.httpMode == 'http',
@@ -69,6 +69,13 @@ module.exports = class extends Generator {
{ name: "Yes - two nodes in a master-master configuration", value: "master-master" }
]
},
{
whenFunction: response => response.acsVersion == '6.2',
type: 'confirm',
name: 'gzip',
message: 'Would you like to compress Get Content responses?',
default: false
},
// Enterprise only options
{
whenFunction: response => response.alfrescoVersion == 'enterprise' && !response.replication,
@@ -208,7 +215,8 @@ module.exports = class extends Generator {
searchPath: searchBasePath,
zeppelin: (this.props.zeppelin ? "true" : "false"),
sharding: (this.props.sharding ? "true" : "false"),
shardingMethod: (this.props.shardingMethod)
shardingMethod: (this.props.shardingMethod),
gzip: (this.props.gzip ? "true" : "false")
}
);
@@ -259,6 +267,16 @@ module.exports = class extends Generator {
)
}
// Empty addons directories.
['alfresco', 'share'].forEach(container => {
['jars', 'amps'].forEach(addonType => {
this.fs.copy(
this.templatePath('empty/empty'),
this.destinationPath(container + '/modules/' + addonType + '/empty')
);
});
});
// Copy Docker Image for Search applying configuration
this.fs.copyTpl(
this.templatePath(imagesDirectory + '/search'),
e2e-test/generator-alfresco-docker-compose/generators/app/templates/6.1/docker-compose-ce.yml
+1 -1
View File
@@ -23,7 +23,7 @@ services:
-Dsolr.host=<%=searchSolrHost%>
-Dsolr.port.ssl=8983
-Dsolr.secureComms=<%=secureComms%>
-Dsolr.base.url=/solr
-Dsolr.baseUrl=/solr
-Dindex.subsystem.name=solr6
-Dshare.host=localhost
-Dalfresco.port=8080
e2e-test/generator-alfresco-docker-compose/generators/app/templates/6.1/docker-compose-ee.yml
+1 -1
View File
@@ -22,7 +22,7 @@ services:
-Dsolr.host=<%=searchSolrHost%>
-Dsolr.port.ssl=8983
-Dsolr.secureComms=<%=secureComms%>
-Dsolr.base.url=/solr <% if (sharding == 'true') { %>
-Dsolr.baseUrl=/solr <% if (sharding == 'true') { %>
-Dsolr.useDynamicShardRegistration=true <% } %>
-Dindex.subsystem.name=solr6
-Dalfresco-pdf-renderer.url=http://alfresco-pdf-renderer:8090/
e2e-test/generator-alfresco-docker-compose/generators/app/templates/6.2/docker-compose-ce.yml
+4 -1
View File
@@ -13,6 +13,7 @@ services:
TRUSTSTORE_PASS: kT9X6oe68t
KEYSTORE_TYPE: JCEKS
KEYSTORE_PASS: kT9X6oe68t <% } %>
COMPRESS_CONTENT: "<%=gzip%>"
mem_limit: 1800m
environment:
JAVA_OPTS : "
@@ -23,7 +24,7 @@ services:
-Dsolr.host=<%=searchSolrHost%>
-Dsolr.port.ssl=8983
-Dsolr.secureComms=<%=secureComms%>
-Dsolr.base.url=/solr
-Dsolr.baseUrl=/solr
-Dindex.subsystem.name=solr6
-Dshare.host=localhost
-Dalfresco.port=8080
@@ -66,6 +67,7 @@ services:
KEYSTORE_TYPE: JCEKS <% } %> <% if (replication) { %>
ENABLE_MASTER: "true"
ENABLE_SLAVE: "false" <% } %>
COMPRESS_CONTENT: "<%=gzip%>"
mem_limit: 1200m
environment:
#Solr needs to know how to register itself with Alfresco
@@ -108,6 +110,7 @@ services:
ENABLE_MASTER: <% if (replication == 'master-master') { %>"true"<% } else { %>"false"<% } %>
ENABLE_SLAVE: <% if (replication == 'master-master') { %>"false"<% } else { %>"true"<% } %>
MASTER_HOST: solr6 <% } %>
COMPRESS_CONTENT: "<%=gzip%>"
mem_limit: 1200m
environment:
#Solr needs to know how to register itself with Alfresco
e2e-test/generator-alfresco-docker-compose/generators/app/templates/6.2/docker-compose-ee.yml
+1 -1
View File
@@ -22,7 +22,7 @@ services:
-Dsolr.host=<%=searchSolrHost%>
-Dsolr.port.ssl=8983
-Dsolr.secureComms=<%=secureComms%>
-Dsolr.base.url=/solr <% if (sharding == 'true') { %>
-Dsolr.baseUrl=/solr <% if (sharding == 'true') { %>
-Dsolr.useDynamicShardRegistration=true <% } %>
-Dindex.subsystem.name=solr6
e2e-test/generator-alfresco-docker-compose/generators/app/templates/empty/empty
View File
e2e-test/generator-alfresco-docker-compose/generators/app/templates/images/alfresco/Dockerfile
+22 -1
View File
@@ -25,6 +25,13 @@ VOLUME ["${ALF_DATA_DIR}/keystore"]
USER root
# Install modules and addons
RUN mkdir -p $TOMCAT_DIR/amps
COPY modules/amps/* $TOMCAT_DIR/amps/
COPY modules/jars/* $TOMCAT_DIR/webapps/alfresco/WEB-INF/lib/
RUN java -jar $TOMCAT_DIR/alfresco-mmt/alfresco-mmt*.jar install \
$TOMCAT_DIR/amps $TOMCAT_DIR/webapps/alfresco -directory -nobackup -force;
# Default value in "repository.properties" is "dir.keystore=classpath:alfresco/keystore"
RUN if [ "$SOLR_COMMS" == "https" ] ; then \
echo -e "\n\
@@ -40,13 +47,27 @@ RUN if [ "$SOLR_COMMS" == "https" ] ; then \
[[:space:]]\+<\/Engine>/\n\
<\/Engine>\n\
<Connector port=\"8443\" protocol=\"org.apache.coyote.http11.Http11Protocol\"\n\
connectionTimeout=\"20000\"\n\
SSLEnabled=\"true\" maxThreads=\"150\" scheme=\"https\"\n\
keystoreFile=\"\/usr\/local\/tomcat\/alf_data\/keystore\/ssl.keystore\"\n\
keystorePass=\"${KEYSTORE_PASS}\" keystoreType=\"${KEYSTORE_TYPE}\" secure=\"true\" connectionTimeout=\"240000\"\n\
keystorePass=\"${KEYSTORE_PASS}\" keystoreType=\"${KEYSTORE_TYPE}\" secure=\"true\"\n\
truststoreFile=\"\/usr\/local\/tomcat\/alf_data\/keystore\/ssl.truststore\"\n\
truststorePass=\"${TRUSTSTORE_PASS}\" truststoreType=\"${TRUSTSTORE_TYPE}\" clientAuth=\"want\" sslProtocol=\"TLS\">\n\
<\/Connector>/g" ${TOMCAT_DIR}/conf/server.xml; \
fi
# GZIP COMPRESSION
ARG COMPRESS_CONTENT
ENV COMPRESS_CONTENT $COMPRESS_CONTENT
RUN if [ "$COMPRESS_CONTENT" == "true" ] ; then \
sed -i "s/\
[[:space:]]\+connectionTimeout=\"20000\"/\n\
connectionTimeout=\"20000\"\n\
compression=\"on\"\n\
compressionMinSize=\"1\"\n\
/g" ${TOMCAT_DIR}/conf/server.xml; \
fi
# Copy custom content model to deployment folder
COPY model/* $TOMCAT_DIR/shared/classes/alfresco/extension/
e2e-test/generator-alfresco-docker-compose/generators/app/templates/images/search/Dockerfile
+8
View File
@@ -125,6 +125,14 @@ RUN if [ "$ENABLE_SHARDING" == "true" ] ; then \
fi; \
fi
# GZIP COMPRESSION
ARG COMPRESS_CONTENT
ENV COMPRESS_CONTENT $COMPRESS_CONTENT
RUN if [ "$COMPRESS_CONTENT" == "true" ] ; then \
sed -i '/^bash.*/i sed -i "'"s/solr.request.content.compress=false/solr.request.content.compress=true/g"'" ${DIST_DIR}/solrhome/templates/rerank/conf/solrcore.properties\n' \
${DIST_DIR}/solr/bin/search_config_setup.sh; \
fi
# Useless for 'none'/'http' communications with Alfresco
RUN mkdir ${DIST_DIR}/keystore \
&& chown -R solr:solr ${DIST_DIR}/keystore
e2e-test/generator-alfresco-docker-compose/generators/app/templates/images/share/Dockerfile
+7
View File
@@ -8,5 +8,12 @@ RUN sed -i '/Connector port="8080"/a scheme="https" secure="true"' /usr/local/to
sed -i '/Connector port="8080"/a proxyName="localhost" proxyPort="<%=port%>"' /usr/local/tomcat/conf/server.xml
<% } %>
# Install modules
RUN mkdir -p $TOMCAT_DIR/amps
COPY modules/amps/* $TOMCAT_DIR/amps/
COPY modules/jars/* $TOMCAT_DIR/webapps/share/WEB-INF/lib/
RUN java -jar $TOMCAT_DIR/alfresco-mmt/alfresco-mmt*.jar install \
$TOMCAT_DIR/amps $TOMCAT_DIR/webapps/share -directory -nobackup -force;
# Copy custom content forms to deployment folder
COPY model/* $TOMCAT_DIR/shared/classes/alfresco/web-extension/
e2e-test/pom.xml
+4 -5
View File
@@ -11,11 +11,10 @@
<name>Search Analytics E2E Tests</name>
<description>Test Project to test Search Service and Analytics Features on a complete setup of Alfresco, Share</description>
<properties>
<tas.rest.api.version>1.28</tas.rest.api.version>
<tas.rest.api.version>1.26</tas.rest.api.version>
<tas.rest.api.version>1.35</tas.rest.api.version>
<tas.cmis.api.version>1.13</tas.cmis.api.version>
<tas.utility.version>3.0.18</tas.utility.version>
<rm.version>3.3.0</rm.version>
<tas.utility.version>3.0.20</tas.utility.version>
<rm.version>3.3.0.1</rm.version>
<suiteXmlFile>src/test/resources/SearchSuite.xml</suiteXmlFile>
<test.exclude></test.exclude>
<test.include></test.include>
@@ -120,7 +119,7 @@
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<version>1.18.10</version>
<version>1.18.12</version>
<scope>test</scope>
</dependency>
<dependency>
e2e-test/src/main/java/org/alfresco/search/TestGroup.java
+7 -2
View File
@@ -23,7 +23,7 @@ public class TestGroup
// Used for TestRail test annotation
public static final String SEARCH = "search";
public static final String REST_API = "rest-api";
public static final String PREUPGRADE = "pre-upgrade";
public static final String POSTUPGRADE = "post-upgrade";
@@ -31,8 +31,13 @@ public class TestGroup
public static final String ASS_MASTER ="ASS_Master"; // Alfresco search services using master/stand alone mode
public static final String EXPLICIT_SHARDING ="Explicit_Sharding"; // Alfresco search services using sharded environment and explicit routing
public static final String SHARDING ="Sharding"; // Alfresco search services using sharded environment
public static final String ASS_SHARDING = "ASS_Sharding"; // Alfresco Search Services using Sharding
public static final String ASS_SHARDING_DB_ID_RANGE = "ASS_Sharding_DB_ID_RANGE"; // Alfresco Search Services using Sharding with DB_ID_RANGE
public static final String NOT_INSIGHT_ENGINE = "Not_InsightEngine"; // When Alfresco Insight Engine 1.0 isn't running
public static final String ACS_52n = "ACS_52n"; // Alfresco Content Services 5.2.n
public static final String ACS_60n = "ACS_60n"; // Alfresco Content Services 6.0 or above
public static final String ACS_61n = "ACS_61n"; // Alfresco Content Services 6.1 or above
e2e-test/src/test/java/org/alfresco/test/search/functional/AbstractE2EFunctionalTest.java
+52 -7
View File
@@ -22,6 +22,7 @@ import org.alfresco.dataprep.SiteService.Visibility;
import org.alfresco.rest.core.RestProperties;
import org.alfresco.rest.core.RestWrapper;
import org.alfresco.rest.model.RestRequestSpellcheckModel;
import org.alfresco.rest.search.Pagination;
import org.alfresco.rest.search.RestRequestHighlightModel;
import org.alfresco.rest.search.RestRequestQueryModel;
import org.alfresco.rest.search.SearchRequest;
@@ -55,7 +56,7 @@ import org.testng.annotations.BeforeSuite;
public abstract class AbstractE2EFunctionalTest extends AbstractTestNGSpringContextTests
{
/** The number of retries that a query will be tried before giving up. */
private static final int SEARCH_MAX_ATTEMPTS = 6;
protected static final int SEARCH_MAX_ATTEMPTS = 6;
private static final Logger LOGGER = LogFactory.getLogger();
@@ -107,6 +108,7 @@ public abstract class AbstractE2EFunctionalTest extends AbstractTestNGSpringCont
deployCustomModel("model/music-model.xml");
deployCustomModel("model/finance-model.xml");
deployCustomModel("model/sharding-content-model.xml");
}
@BeforeClass (alwaysRun = true)
@@ -393,10 +395,15 @@ public abstract class AbstractE2EFunctionalTest extends AbstractTestNGSpringCont
return restClient.authenticateUser(user).withSearchAPI().search(searchRequest);
}
protected SearchResponse queryAsUser(UserModel user, RestRequestQueryModel queryModel)
protected SearchResponse queryAsUser(UserModel user, RestRequestQueryModel queryModel, Pagination paging)
{
SearchRequest searchRequest = new SearchRequest();
searchRequest.setQuery(queryModel);
if (ofNullable(paging).isPresent())
{
searchRequest.setPaging(paging);
}
return restClient.authenticateUser(user).withSearchAPI().search(searchRequest);
}
@@ -436,7 +443,7 @@ public abstract class AbstractE2EFunctionalTest extends AbstractTestNGSpringCont
*/
protected SearchResponse testSearchQuery(String query, Integer expectedCount, SearchLanguage queryLanguage)
{
SearchResponse response = performSearch(testUser, query, queryLanguage);
SearchResponse response = performSearch(testUser, query, queryLanguage, getDefaultPagingOptions());
if (ofNullable(expectedCount).isPresent())
{
@@ -455,7 +462,7 @@ public abstract class AbstractE2EFunctionalTest extends AbstractTestNGSpringCont
*/
protected SearchResponse testSearchQueryOrdered(String query, List<String> expectedNames, SearchLanguage queryLanguage)
{
SearchResponse response = performSearch(testUser, query, queryLanguage);
SearchResponse response = performSearch(testUser, query, queryLanguage, getDefaultPagingOptions());
List<String> names = response.getEntries().stream().map(s -> s.getModel().getName()).collect(Collectors.toList());
@@ -474,7 +481,7 @@ public abstract class AbstractE2EFunctionalTest extends AbstractTestNGSpringCont
*/
protected SearchResponse testSearchQueryUnordered(String query, Set<String> expectedNames, SearchLanguage queryLanguage)
{
SearchResponse response = performSearch(testUser, query, queryLanguage);
SearchResponse response = performSearch(testUser, query, queryLanguage, getDefaultPagingOptions());
Set<String> names = response.getEntries().stream().map(s -> s.getModel().getName()).collect(Collectors.toSet());
@@ -483,7 +490,7 @@ public abstract class AbstractE2EFunctionalTest extends AbstractTestNGSpringCont
return response;
}
private SearchResponse performSearch(UserModel asUser, String query, SearchLanguage queryLanguage)
protected SearchResponse performSearch(UserModel asUser, String query, SearchLanguage queryLanguage, Pagination paging)
{
RestRequestQueryModel queryModel = new RestRequestQueryModel();
queryModel.setQuery(query);
@@ -498,11 +505,49 @@ public abstract class AbstractE2EFunctionalTest extends AbstractTestNGSpringCont
queryModel.setLanguage(queryLanguage.toString());
}
SearchResponse response = queryAsUser(asUser, queryModel);
SearchResponse response = queryAsUser(asUser, queryModel, paging);
return response;
}
/**
* Returns pagination object with alfresco default settings
* Sets skipCount = 0, maxItems = 100
*
* @return
*/
private Pagination getDefaultPagingOptions()
{
Pagination paging = new Pagination();
paging.setSkipCount(0);
paging.setMaxItems(100);
return paging;
}
/**
* Set the pagination options for the API query
* @param skipCount Integer
* @param maxItems Integer
* @return
*/
protected Pagination setPaging(Integer skipCount, Integer maxItems)
{
Pagination paging = new Pagination();
if (ofNullable(skipCount).isPresent())
{
paging.setSkipCount(skipCount);
}
if (ofNullable(maxItems).isPresent())
{
paging.setMaxItems(maxItems);
}
return paging;
}
/**
* Method to create and run a simple spellcheck query
* When a spellcheck query is run a user, the query inputed and the user query is inputted
e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/AbstractCmisE2ETest.java
+2 -2
View File
@@ -61,7 +61,7 @@ public abstract class AbstractCmisE2ETest extends AbstractE2EFunctionalTest
protected boolean waitForIndexing(String query, long expectedCountResults)
{
for (int searchCount = 1; searchCount <= 3; searchCount++)
for (int searchCount = 1; searchCount <= SEARCH_MAX_ATTEMPTS; searchCount++)
{
try
@@ -71,7 +71,7 @@ public abstract class AbstractCmisE2ETest extends AbstractE2EFunctionalTest
}
catch (AssertionError ae)
{
LOGGER.debug(ae.toString());
LOGGER.info(String.format("WaitForIndexing in Progress: %s", ae.toString()));
}
e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchByAspectTests.java
+3 -1
View File
@@ -7,6 +7,7 @@ import org.alfresco.utility.data.provider.XMLTestDataProvider;
import org.alfresco.utility.model.FileModel;
import org.alfresco.utility.model.FolderModel;
import org.alfresco.utility.model.QueryModel;
import org.testng.Assert;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
@@ -135,6 +136,7 @@ public class SolrSearchByAspectTests extends AbstractCmisE2ETest
.replace("NODE_REF[f1]", tasFolder1.getNodeRef())
.replace("NODE_REF[s1]", siteDoclibNodeRef);
cmisApi.authenticateUser(testUser).withQuery(currentQuery).assertResultsCount().equals(query.getResults());
cmisApi.authenticateUser(testUser);
Assert.assertTrue(waitForIndexing(currentQuery, query.getResults()), String.format("Result count not as expected for query: %s", currentQuery));
}
}
e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchByIdTests.java
+4 -2
View File
@@ -7,6 +7,7 @@ import org.alfresco.utility.data.provider.XMLTestDataProvider;
import org.alfresco.utility.model.FileModel;
import org.alfresco.utility.model.FolderModel;
import org.alfresco.utility.model.QueryModel;
import org.testng.Assert;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
@@ -87,7 +88,7 @@ public class SolrSearchByIdTests extends AbstractCmisE2ETest
@Test(dataProviderClass = XMLTestDataProvider.class, dataProvider = "getQueriesData")
@XMLDataConfig(file = "src/test/resources/testdata/search-by-id.xml")
public void executeSearchByAspect(QueryModel query) throws Exception
public void executeSearchById(QueryModel query) throws Exception
{
String currentQuery = query.getValue()
.replace("NODE_REF[siteId]", siteDoclibNodeRef)
@@ -96,6 +97,7 @@ public class SolrSearchByIdTests extends AbstractCmisE2ETest
.replace("NODE_REF[f1]", tasFolder1.getNodeRef())
.replace("NODE_REF[f1-1]", tasSubFolder1.getNodeRef());
cmisApi.authenticateUser(testUser).withQuery(currentQuery).assertResultsCount().equals(query.getResults());
cmisApi.authenticateUser(testUser);
Assert.assertTrue(waitForIndexing(currentQuery, query.getResults()), String.format("Result count not as expected for query: %s", currentQuery));
}
}
e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchByPathTests.java
+4 -1
View File
@@ -7,6 +7,7 @@ import org.alfresco.utility.data.provider.XMLTestDataProvider;
import org.alfresco.utility.model.QueryModel;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.testng.Assert;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
@@ -51,6 +52,8 @@ public class SolrSearchByPathTests extends AbstractCmisE2ETest
@XMLDataConfig(file = "src/test/resources/testdata/search-by-path.xml")
public void executeSearchByPathQueries(QueryModel query)
{
cmisApi.withQuery(query.getValue()).assertResultsCount().equals(query.getResults());
cmisApi.authenticateUser(testUser);
Assert.assertTrue(waitForIndexing(query.getValue(), query.getResults()), String.format("Result count not as expected for query: %s", query.getValue()));
}
}
e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchByPropertyTests.java
+3 -1
View File
@@ -10,6 +10,7 @@ import org.alfresco.utility.data.provider.XMLTestDataProvider;
import org.alfresco.utility.model.FileModel;
import org.alfresco.utility.model.FolderModel;
import org.alfresco.utility.model.QueryModel;
import org.testng.Assert;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
@@ -104,7 +105,8 @@ public class SolrSearchByPropertyTests extends AbstractCmisE2ETest
.addProperty("tas:IntPropertyC", 2223));
// wait for solr index
Utility.waitToLoopTime(getSolrWaitTimeInSeconds());
cmisApi.authenticateUser(testUser);
waitForIndexing("SELECT * FROM tas:document where cmis:name = 'testc3.txt'", 1);
}
@Test
e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchInFolderTests.java
+3 -1
View File
@@ -7,6 +7,7 @@ import org.alfresco.utility.model.FileModel;
import org.alfresco.utility.model.FileType;
import org.alfresco.utility.model.FolderModel;
import org.alfresco.utility.model.QueryModel;
import org.testng.Assert;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
@@ -55,6 +56,7 @@ public class SolrSearchInFolderTests extends AbstractCmisE2ETest
public void executeCMISQuery(QueryModel query) throws Exception
{
String currentQuery = String.format(query.getValue(), parentFolder.getNodeRef());
cmisApi.withQuery(currentQuery).assertResultsCount().equals(query.getResults());
cmisApi.authenticateUser(testUser);
Assert.assertTrue(waitForIndexing(currentQuery, query.getResults()), String.format("Result count not as expected for query: %s", currentQuery));
}
}
e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchInTreeTests.java
+3 -2
View File
@@ -7,6 +7,7 @@ import org.alfresco.utility.model.FileModel;
import org.alfresco.utility.model.FileType;
import org.alfresco.utility.model.FolderModel;
import org.alfresco.utility.model.QueryModel;
import org.testng.Assert;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
@@ -59,7 +60,7 @@ public class SolrSearchInTreeTests extends AbstractCmisE2ETest
public void executeCMISQuery(QueryModel query) throws Exception
{
String currentQuery = String.format(query.getValue(), parentFolder.getNodeRef());
cmisApi.withQuery(currentQuery)
.assertResultsCount().equals(query.getResults());
cmisApi.authenticateUser(testUser);
Assert.assertTrue(waitForIndexing(currentQuery, query.getResults()), String.format("Result count not as expected for query: %s", currentQuery));
}
}
e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/cmis/SolrSearchScoreQueryTests.java
+59 -88
View File
@@ -9,6 +9,7 @@ import org.alfresco.utility.data.provider.XMLTestDataProvider;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.testng.Assert;
import org.testng.annotations.AfterClass;
import org.testng.annotations.Test;
@@ -54,8 +55,8 @@ public class SolrSearchScoreQueryTests extends AbstractCmisE2ETest
this.testData = testData;
this.testData.createUsers(dataUser);
this.testData.createSitesStructure(dataSite, dataContent, dataUser);
cmisApi.authenticateUser(dataUser.getCurrentUser());
testUser = dataUser.getCurrentUser();
cmisApi.authenticateUser(testUser);
}
/**
@@ -70,68 +71,50 @@ public class SolrSearchScoreQueryTests extends AbstractCmisE2ETest
+ "FROM cmis:document "
+ "WHERE CONTAINS('Quidditch') "
+ "ORDER BY orderCriteria";
if (waitForIndexing(query, 3))
{
cmisApi
.withQuery(query)
.assertColumnIsOrdered().isOrderedAsc("orderCriteria");
}
else
{
throw new AssertionError("Wait for indexing has failed!");
}
Assert.assertTrue(waitForIndexing(query, 3), String.format("Result count not as expected for query: %s", query));
cmisApi.withQuery(query).assertColumnIsOrdered().isOrderedAsc("orderCriteria");
}
/**
* Verify that results are inverse ordered
* @throws Exception
*/
@Test(dependsOnMethods = "prepareDataForScoreSearch")
/**
* Verify that results are inverse ordered
*
* @throws Exception
*/
@Test(dependsOnMethods = "prepareDataForScoreSearch")
public void scoreQueryOrderedDesc() throws Exception
{
String query = "SELECT cmis:objectId, SCORE() AS orderCriteria "
String query = "SELECT cmis:objectId, SCORE() AS orderCriteria "
+ "FROM cmis:document "
+ "WHERE CONTAINS('Quidditch') "
+ "ORDER BY orderCriteria DESC";
if (waitForIndexing(query, 3))
{
cmisApi
.withQuery(query).assertColumnIsOrdered().isOrderedDesc("orderCriteria");
}
else
{
throw new AssertionError("Wait for indexing has failed!");
}
Assert.assertTrue(waitForIndexing(query, 3), String.format("Result count not as expected for query: %s", query));
cmisApi.withQuery(query).assertColumnIsOrdered().isOrderedDesc("orderCriteria");
}
/**
* Verify that all SCORE results are between 0 and 1
* @throws Exception
*/
@Test(groups = { TestGroup.ACS_62n }, dependsOnMethods = "prepareDataForScoreSearch")
/**
* Verify that all SCORE results are between 0 and 1
*
* @throws Exception
*/
@Test(groups = { TestGroup.ACS_62n }, dependsOnMethods = "prepareDataForScoreSearch")
public void scoreQueryInRange() throws Exception
{
String query = "SELECT cmis:objectId, SCORE() "
+ "FROM cmis:document "
+ "WHERE CONTAINS('Quidditch')";
if (waitForIndexing(query, 3))
{
cmisApi
.withQuery(query)
.assertColumnValuesRange().isReturningValuesInRange("SEARCH_SCORE", BigDecimal.ZERO, BigDecimal.ONE);
}
else
{
throw new AssertionError("Wait for indexing has failed!");
}
String query = "SELECT cmis:objectId, SCORE() "
+ "FROM cmis:document "
+ "WHERE CONTAINS('Quidditch')";
Assert.assertTrue(waitForIndexing(query, 3), String.format("Result count not as expected for query: %s", query));
cmisApi.withQuery(query).assertColumnValuesRange().isReturningValuesInRange("SEARCH_SCORE", BigDecimal.ZERO, BigDecimal.ONE);
}
/**
@@ -143,45 +126,33 @@ public class SolrSearchScoreQueryTests extends AbstractCmisE2ETest
{
String query = "SELECT cmis:objectId, SCORE() AS orderCriteria "
+ "FROM cmis:document "
+ "WHERE CONTAINS('Quidditch')";
if (waitForIndexing(query, 3))
{
cmisApi
.withQuery(query)
.assertColumnValuesRange().isReturningValuesInRange("orderCriteria", BigDecimal.ZERO, BigDecimal.ONE);
}
else
{
throw new AssertionError("Wait for indexing has failed!");
}
+ "FROM cmis:document "
+ "WHERE CONTAINS('Quidditch')";
Assert.assertTrue(waitForIndexing(query, 3), String.format("Result count not as expected for query: %s", query));
cmisApi.withQuery(query).assertColumnValuesRange().isReturningValuesInRange("orderCriteria", BigDecimal.ZERO, BigDecimal.ONE);
}
/**
* Verify that SCORE is valid name for an alias
* Currently only supported with double quotes
* @throws Exception
*/
@Test(dependsOnMethods = "prepareDataForScoreSearch")
/**
* Verify that SCORE is valid name for an alias
* Currently only supported with double quotes
*
* @throws Exception
*/
@Test(dependsOnMethods = "prepareDataForScoreSearch")
public void scoreQueryScoreAsAlias() throws Exception
{
String query = "SELECT cmis:objectId, SCORE() AS \"score\" "
+ "FROM cmis:document "
+ "WHERE CONTAINS('Quidditch')";
if (waitForIndexing(query, 3))
{
cmisApi
.withQuery(query).assertResultsCount().equals(3);
}
else
{
throw new AssertionError("Wait for indexing has failed!");
}
String query = "SELECT cmis:objectId, SCORE() AS \"score\" "
+ "FROM cmis:document "
+ "WHERE CONTAINS('Quidditch')";
Assert.assertTrue(waitForIndexing(query, 3), String.format("Result count not as expected for query: %s", query));
cmisApi.withQuery(query).assertResultsCount().equals(3);
}
}
e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SearchAFTSInFieldTest.java
+2 -2
View File
@@ -118,13 +118,13 @@ public class SearchAFTSInFieldTest extends AbstractE2EFunctionalTest
boolean fileFound = isContentInSearchResults(query, file2.getName(), true);
Assert.assertTrue(fileFound, "File Not found for query: " + query);
testSearchQuery(query, 1, SearchLanguage.AFTS);
testSearchQuery(query, 2, SearchLanguage.AFTS);
query = fieldName + ":'" + file2.getName() + "\'";
fileFound = isContentInSearchResults(query, file2.getName(), true);
Assert.assertTrue(fileFound, "File Not found for query: " + query);
testSearchQuery(query, 1, SearchLanguage.AFTS);
testSearchQuery(query, 2, SearchLanguage.AFTS);
});
}
e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SearchPermissionsTest.java
+222
View File
@@ -0,0 +1,222 @@
/*
* Copyright (C) 2020 Alfresco Software Limited.
*
* This file is part of Alfresco
*
* Alfresco is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Alfresco is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
*/
package org.alfresco.test.search.functional.searchServices.search;
import javax.json.Json;
import javax.json.JsonObject;
import org.alfresco.dataprep.SiteService.Visibility;
import org.alfresco.rest.search.SearchResponse;
import org.alfresco.test.search.functional.AbstractE2EFunctionalTest;
import org.alfresco.utility.constants.UserRole;
import org.alfresco.utility.data.RandomData;
import org.alfresco.utility.model.FileModel;
import org.alfresco.utility.model.FileType;
import org.alfresco.utility.model.FolderModel;
import org.alfresco.utility.model.SiteModel;
import org.alfresco.utility.model.UserModel;
import org.testng.Assert;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
/**
* Search end point Public API test with Permission checks
*
* @author Meenal Bhave
*/
public class SearchPermissionsTest extends AbstractE2EFunctionalTest
{
private FileModel file1, file2;
private FolderModel parentFolder, folder1, folder2;
private UserModel testUser1, testUser2, testUser3;
@BeforeClass(alwaysRun = true)
public void dataPreparation() throws Exception
{
/*
* Create the following file structure in the same Site : In addition to the preconditions created in dataPreparation
* |- permGrandParent
* |-- permChild1
* |------ permFile1
* |-- permChild2
* |------ permFile2
* |-- permChild3 (Later: In test 2)
*/
parentFolder = new FolderModel("permGrandParent");
dataContent.usingUser(testUser).usingSite(testSite).createFolder(parentFolder);
folder1 = dataContent.usingUser(testUser).usingSite(testSite).usingResource(parentFolder).createFolderCmisApi("permChild1");
folder2 = dataContent.usingUser(testUser).usingSite(testSite).usingResource(parentFolder).createFolderCmisApi("permChild2");
file1 = new FileModel("permFile1", FileType.TEXT_PLAIN, "File1 with inherited permissions");
file2 = new FileModel("permFile2", FileType.TEXT_PLAIN, "File2 with inherited permissions");
// Create test users
testUser1 = dataUser.createRandomTestUser("UserSiteMember");
testUser2 = dataUser.createRandomTestUser("UserNotASiteMemeber");
testUser3 = dataUser.createRandomTestUser("UserWithoutContentAccess");
dataUser.addUserToSite(testUser1, testSite, UserRole.SiteCollaborator);
dataContent.usingUser(testUser).usingSite(testSite).usingResource(folder1).createContent(file1);
dataContent.usingUser(testUser).usingSite(testSite).usingResource(folder2).createContent(file2);
// Deny testUser1 - access to file1
JsonObject userPermission = Json
.createObjectBuilder()
.add("permissions",
Json.createObjectBuilder()
.add("isInheritanceEnabled", false)
.add("locallySet",
Json.createObjectBuilder()
.add("authorityId", testUser1.getUsername())
.add("name", "SiteCollaborator")
.add("accessStatus", "DENIED")
)).build();
String putBody = userPermission.toString();
restClient.authenticateUser(testUser).withCoreAPI().usingNode(file1).updateNode(putBody);
// Allow testUser2 - access to file2, user is not a site member
userPermission = Json
.createObjectBuilder()
.add("permissions",
Json.createObjectBuilder()
.add("isInheritanceEnabled", false)
.add("locallySet",
Json.createObjectBuilder()
.add("authorityId", testUser2.getUsername())
.add("name", "SiteContributor")
.add("accessStatus", "ALLOWED")
)).build();
putBody = userPermission.toString();
restClient.authenticateUser(testUser).withCoreAPI().usingNode(file2).updateNode(putBody);
waitForContentIndexing(file2.getContent(), true);
}
@Test(priority = 1)
public void searchResultsRespectInheritedPermissions()
{
/**
* Private Site Folder Structure available for this test
* |- permGrandParent
* |-- permChild1
* |------ permFile1 (inheritance disabled, deny permission to testUser1)
* |-- permChild2
* |------ permFile2 (inheritance disabled, allow permission to testUser2)
*/
// Search as testUser: expect all: 5 results: When user is a Site Manager
SearchResponse response = queryAsUser(testUser, "cm:name:perm*");
int resultCount = response.getPagination().getCount();
Assert.assertEquals(resultCount, 5, "Unexpected Result count for testUser: Expected 5, received: " + resultCount);
// Search as testUser1: expect 3 results: when user is a site member but without permission to a content
response = queryAsUser(testUser1, "cm:name:perm*");
resultCount = response.getPagination().getCount();
Assert.assertEquals(resultCount, 3, "Unexpected Result count for testUser1: Expected 3, received: " + resultCount);
// Search as testUser2: expect 1 result: When user isn't a site member but has granular permissions to a content
response = queryAsUser(testUser2, "cm:name:perm*");
resultCount = response.getPagination().getCount();
Assert.assertEquals(resultCount, 1, "Unexpected Result count for testUser2: Expected 1, received: " + resultCount);
// Search as testUser3: expect none: 0 results: When user isn't a site member / does not have granular permissions to the content
response = queryAsUser(testUser3, "cm:name:perm*");
resultCount = response.getPagination().getCount();
Assert.assertEquals(resultCount, 0, "Unexpected Result count for testUser3: Expected 0, received: " + resultCount);
}
@Test(priority = 2)
public void searchResultsRespectInheritedPermissionsDisabled() throws Exception
{
// Create folder
FolderModel folder3 = dataContent.usingUser(testUser).usingSite(testSite).usingResource(parentFolder).createFolderCmisApi("permChild3");
// Turn off inherited permissions for folder3
JsonObject userPermission = Json.createObjectBuilder().add("permissions", Json.createObjectBuilder().add("isInheritanceEnabled", false)).build();
String putBody = userPermission.toString();
restClient.authenticateUser(testUser).withCoreAPI().usingNode(folder3).updateNode(putBody);
// Wait for indexing
waitForIndexing(folder3.getName(), true);
/**
* Private Site Folder Structure available now for this test
* |- permGrandParent
* |-- permChild1
* |------ permFile1 (inheritance disabled, deny permission to testUser1)
* |-- permChild2
* |------ permFile2 (inheritance disabled, allow permission to testUser2)
* |-- permChild3 (inheritance disabled)
*/
// Search as testUser: expect all: 6 results: When user is a Site Manager
SearchResponse response = queryAsUser(testUser, "cm:name:perm*");
int resultCount = response.getPagination().getCount();
Assert.assertEquals(resultCount, 6, "Unexpected Result count for testUser: Expected 6, received: " + resultCount);
// Search as testUser1: expect 3 results: when user is a site member but without permission to a content
response = queryAsUser(testUser1, "cm:name:perm*");
resultCount = response.getPagination().getCount();
Assert.assertEquals(resultCount, 3, "Unexpected Result count for testUser1: Expected 3, received: " + resultCount);
// Search as testUser2: expect 1 result: When user isn't a site member but has granular permissions to a content
response = queryAsUser(testUser2, "cm:name:perm*");
resultCount = response.getPagination().getCount();
Assert.assertEquals(resultCount, 1, "Unexpected Result count for testUser2: Expected 1, received: " + resultCount);
// Search as testUser3: expect none: 0 results: When user isn't a site member / does not have granular permissions to the content
response = queryAsUser(testUser3, "cm:name:perm*");
resultCount = response.getPagination().getCount();
Assert.assertEquals(resultCount, 0, "Unexpected Result count for testUser3: Expected 0, received: " + resultCount);
}
@Test(priority = 3)
public void searchResultsOnChangingSiteVisibility() throws Exception
{
// Create Site
SiteModel testPermissionsSite = new SiteModel(RandomData.getRandomName("SiteSearchPermissions"));
testPermissionsSite.setVisibility(Visibility.PUBLIC);
dataSite.usingUser(adminUserModel).createSite(testPermissionsSite);
// Add Users to the site
dataUser.addUserToSite(testUser, testPermissionsSite, UserRole.SiteCollaborator);
// Create a folder
String folderName = "Folder" + unique_searchString;
FolderModel folder = dataContent.usingUser(testUser).usingSite(testPermissionsSite).createFolderCmisApi(folderName);
// Query
Assert.assertTrue(waitForIndexing(folder.getName(), true), "Folder isn't yet indexed");
// Edit Site Visibility
testPermissionsSite.setDescription("PrivateSite".concat(testPermissionsSite.getDescription()));
testPermissionsSite.setVisibility(Visibility.PRIVATE);
restClient.authenticateUser(adminUserModel).withCoreAPI().usingSite(testPermissionsSite).updateSite(testPermissionsSite);
// Wait for indexing
Assert.assertTrue(waitForIndexing("description:" + testPermissionsSite.getDescription(), true), "New Site Description isn't yet been indexed");
// Query
SearchResponse response = queryAsUser(testUser, folderName);
int resultCount = response.getPagination().getCount();
Assert.assertEquals(resultCount, 1, "Unexpected Result count for testUser: Expected 1, received: " + resultCount);
}
}
e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SearchQueryPaginationTest.java
+258
View File
@@ -0,0 +1,258 @@
/*
* Copyright 2019 Alfresco Software, Ltd. All rights reserved.
* License rights for this program may be obtained from Alfresco Software, Ltd.
* pursuant to a written agreement and any use of this program without such an
* agreement is prohibited.
*/
package org.alfresco.test.search.functional.searchServices.search;
import org.alfresco.rest.search.SearchResponse;
import org.alfresco.test.search.functional.searchServices.cmis.AbstractCmisE2ETest;
import org.alfresco.utility.model.FileModel;
import org.alfresco.utility.model.FileType;
import org.alfresco.utility.model.FolderModel;
import org.alfresco.utility.model.UserModel;
import org.testng.Assert;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
/**
* The purpose of this test is to test search query pagination using cmis and afts query
*
* @author Meenal Bhave
*/
public class SearchQueryPaginationTest extends AbstractCmisE2ETest
{
private UserModel testUser2;
private FolderModel testFolder ;
private FileModel testFile;
@BeforeClass(alwaysRun = true)
public void dataPreparation() throws Exception
{
// Create testUser2: This user does not have access to the testSite
testUser2 = dataUser.createRandomTestUser("testUser2");
// Create a new folder and 10 files inside the folder
testFolder = dataContent.usingUser(testUser).usingSite(testSite).createFolder();
for (int i = 0; i < 10; i++)
{
testFile = new FileModel(i + "-File.txt", unique_searchString, "", FileType.TEXT_PLAIN);
dataContent.usingUser(testUser).usingSite(testSite).usingResource(testFolder).createContent(testFile);
}
waitForMetadataIndexing(testFile.getName(), true);
}
@Test(priority = 1)
public void testCmisSearchWithPagination()
{
// Search for the cmis documents using cmis query
String query = "select * from cmis:document";
// Set skipCount = 0, maxItems = 1000
SearchResponse response = performSearch(testUser, query, SearchLanguage.CMIS, setPaging(0, 1000));
// Get getTotalItems, Expect hasModeItems = false
Integer totalDocuments = response.getPagination().getTotalItems();
Assert.assertFalse(response.getPagination().isHasMoreItems(), "Incorrect: hasMoreItems");
// Set skipCount = totalDocument, maxItems = 1000
response = performSearch(testUser, query, SearchLanguage.CMIS, setPaging(totalDocuments, 1000));
// Check getTotalItems is the same, hasMoreItems = false
Assert.assertEquals(response.getPagination().getTotalItems(), totalDocuments, "Total Document Count doesn't match");
Assert.assertFalse(response.getPagination().isHasMoreItems(), "Incorrect: hasMoreItems");
// Set skipCount = 0, maxItems = totalDocuments/2
response = performSearch(testUser, query, SearchLanguage.CMIS, setPaging(0, totalDocuments/2));
// Check getTotalItems is the same, hasMoreItems = True
Assert.assertEquals(response.getPagination().getTotalItems(), totalDocuments, "Total Document Count doesn't match");
Assert.assertTrue(response.getPagination().isHasMoreItems(), "Incorrect: hasMoreItems");
// Set skipCount = 1, maxItems = totalDocuments-2
response = performSearch(testUser, query, SearchLanguage.CMIS, setPaging(1, totalDocuments-2));
// Check getTotalItems is the same, hasMoreItems = True
Assert.assertEquals(response.getPagination().getTotalItems(), totalDocuments, "Total Document Count doesn't match");
Assert.assertTrue(response.getPagination().isHasMoreItems(), "Incorrect: hasMoreItems");
// Set skipCount = totalDocuments, maxItems = totalDocuments/2
response = performSearch(testUser, query, SearchLanguage.CMIS, setPaging(totalDocuments, totalDocuments/2));
// Check getTotalItems is the same, hasMoreItems = false
Assert.assertEquals(response.getPagination().getTotalItems(), totalDocuments, "Total Document Count doesn't match");
Assert.assertFalse(response.getPagination().isHasMoreItems(), "Incorrect: hasMoreItems");
// Set skipCount = 0, maxItems = totalDocuments
response = performSearch(testUser, query, SearchLanguage.CMIS, setPaging(0, totalDocuments));
// Check getTotalItems is the same, hasMoreItems = false
Assert.assertEquals(response.getPagination().getTotalItems(), totalDocuments, "Total Document Count doesn't match");
Assert.assertFalse(response.getPagination().isHasMoreItems(), "Incorrect: hasMoreItems");
// Set skipCount = totalDocuments/2+1, maxItems = totalDocuments
response = performSearch(testUser, query, SearchLanguage.CMIS, setPaging(totalDocuments/2+1, totalDocuments));
// Check getTotalItems is the same, hasMoreItems = false
Assert.assertEquals(response.getPagination().getTotalItems(), totalDocuments, "Total Document Count doesn't match");
Assert.assertFalse(response.getPagination().isHasMoreItems(), "Incorrect: hasMoreItems");
// Set skipCount = totalDocuments, maxItems = totalDocuments
response = performSearch(testUser, query, SearchLanguage.CMIS, setPaging(totalDocuments, totalDocuments));
// Check getTotalItems is the same, hasMoreItems = false
Assert.assertEquals(response.getPagination().getTotalItems(), totalDocuments, "Total Document Count doesn't match");
Assert.assertFalse(response.getPagination().isHasMoreItems(), "Incorrect: hasMoreItems");
// Set skipCount = 0, maxItems = 0
response = performSearch(testUser, query, SearchLanguage.CMIS, setPaging(0, 0));
Assert.assertTrue(response.isEmpty(), "Empty Response, Error is expected when maxItems is <= 0");
// Set skipCount = totalDocuments, maxItems = 0
response = performSearch(testUser, query, SearchLanguage.CMIS, setPaging(totalDocuments, 0));
Assert.assertTrue(response.isEmpty(), "Empty Response, Error is expected when maxItems <= 0");
// Set skipCount = -1, maxItems = 1
response = performSearch(testUser, query, SearchLanguage.CMIS, setPaging(-1, 1));
Assert.assertTrue(response.isEmpty(), "Empty Response, Error is expected when skipCount < 0");
}
@Test(priority = 2)
public void testPagination()
{
// Search for the files under the testFolder using cmis query
String parentId = testFolder.getNodeRefWithoutVersion();
String query = "select * from cmis:document where IN_FOLDER('" + parentId + "')";
// Set skipCount = 0, maxItems = 100
SearchResponse response = performSearch(testUser, query, SearchLanguage.CMIS, setPaging(0, 100));
// Check getTotalItems = 10, Expect hasModeItems = false
testPaginationDetails(response, 10, 0, 100);
Assert.assertFalse(response.getPagination().isHasMoreItems(), "Incorrect: hasMoreItems");
// Set skipCount = 0, maxItems = 10: Expect hasModeItems = false
response = performSearch(testUser, query, SearchLanguage.CMIS, setPaging(0, 10));
testPaginationDetails(response, 10, 0, 10);
Assert.assertFalse(response.getPagination().isHasMoreItems(), "Incorrect: hasMoreItems");
// Set skipCount = 0, maxItems = 5: Expect hasModeItems = true
response = performSearch(testUser, query, SearchLanguage.CMIS, setPaging(0, 5));
testPaginationDetails(response, 10, 0, 5);
Assert.assertTrue(response.getPagination().isHasMoreItems(), "Incorrect: hasMoreItems");
// Set skipCount = 2, maxItems = 10: Expect hasModeItems = false
response = performSearch(testUser, query, SearchLanguage.CMIS, setPaging(2, 10));
testPaginationDetails(response, 10, 2, 10);
Assert.assertFalse(response.getPagination().isHasMoreItems(), "Incorrect: hasMoreItems");
// Set skipCount = 2, maxItems = 7: Expect hasModeItems = true
response = performSearch(testUser, query, SearchLanguage.CMIS, setPaging(2, 7));
testPaginationDetails(response, 10, 2, 7);
Assert.assertTrue(response.getPagination().isHasMoreItems(), "Incorrect: hasMoreItems");
// Set skipCount = 2, maxItems = 8: Expect hasModeItems = false
response = performSearch(testUser, query, SearchLanguage.CMIS, setPaging(2, 8));
testPaginationDetails(response, 10, 2, 8);
Assert.assertFalse(response.getPagination().isHasMoreItems(), "Incorrect: hasMoreItems");
}
@Test(priority = 3)
public void testPaginationRespectsACLs()
{
// Search for the files under the testFolder using cmis query
String parentId = testFolder.getNodeRefWithoutVersion();
String query = "select * from cmis:document where IN_FOLDER('" + parentId + "')";
// Set skipCount = 0, maxItems = 100
SearchResponse response = performSearch(testUser2, query, SearchLanguage.CMIS, setPaging(0, 100));
// Get getTotalItems, Expect hasModeItems = false
testPaginationDetails(response, 0, 0, 100);
Assert.assertFalse(response.getPagination().isHasMoreItems(), "Incorrect: hasMoreItems");
// Set skipCount = 1, maxItems = 1
response = performSearch(testUser2, query, SearchLanguage.CMIS, setPaging(1, 1));
// Get getTotalItems, Expect hasModeItems = false
testPaginationDetails(response, 0, 1, 1);
Assert.assertFalse(response.getPagination().isHasMoreItems(), "Incorrect: hasMoreItems");
}
@Test(priority = 4)
public void testSearchApiPagination()
{
// Search for the files with specific title
String query = "cm:title:'" + unique_searchString + "'";
// Set skipCount = 0, maxItems = 100
SearchResponse response = performSearch(testUser, query, SearchLanguage.AFTS, setPaging(0, 100));
// Get getTotalItems, Expect hasModeItems = false
testPaginationDetails(response, 10, 0, 100);
Assert.assertFalse(response.getPagination().isHasMoreItems(), "Incorrect: hasMoreItems");
// Set skipCount = 1, maxItems = 1
response = performSearch(testUser, query, SearchLanguage.AFTS, setPaging(1, 1));
// Get getTotalItems, Expect hasModeItems = true
testPaginationDetails(response, 10, 1, 1);
Assert.assertTrue(response.getPagination().isHasMoreItems(), "Incorrect: hasMoreItems");
// Set skipCount = 9, maxItems = 10
response = performSearch(testUser, query, SearchLanguage.AFTS, setPaging(9, 10));
// Get getTotalItems, Expect hasModeItems = false
testPaginationDetails(response, 10, 9, 10);
Assert.assertFalse(response.getPagination().isHasMoreItems(), "Incorrect: hasMoreItems");
// Set skipCount = 10, maxItems = 10
response = performSearch(testUser, query, SearchLanguage.AFTS, setPaging(10, 10));
// Get getTotalItems, Expect hasModeItems = false
testPaginationDetails(response, 10, 10, 10);
Assert.assertFalse(response.getPagination().isHasMoreItems(), "Incorrect: hasMoreItems");
}
@Test(priority = 5)
public void testSearchApiPaginationRespectsACLs()
{
// Search for the files with specific title
String query = "cm:title:'" + unique_searchString + "'";
// Set skipCount = 0, maxItems = 100
SearchResponse response = performSearch(testUser2, query, SearchLanguage.AFTS, setPaging(0, 100));
// Get getTotalItems, Expect hasModeItems = false
testPaginationDetails(response, 0, 0, 100);
Assert.assertFalse(response.getPagination().isHasMoreItems(), "Incorrect: hasMoreItems");
// Set skipCount = 1, maxItems = 1
response = performSearch(testUser2, query, SearchLanguage.AFTS, setPaging(1, 1));
// Get getTotalItems, Expect hasModeItems = false
testPaginationDetails(response, 0, 1, 1);
Assert.assertFalse(response.getPagination().isHasMoreItems(), "Incorrect: hasMoreItems");
}
private void testPaginationDetails(SearchResponse response, int expectedTotalCount, int skipCount, int maxItems)
{
Assert.assertEquals(response.getPagination().getTotalItems().intValue(), expectedTotalCount, "Unexpected document count");
Assert.assertEquals(response.getPagination().getSkipCount(), skipCount, "Unexpected skip count returned");
Assert.assertEquals(response.getPagination().getMaxItems(), maxItems, "Unexpected maxItems returned");
// count = (total-skipCount) < maxItems ? total-skipCount: maxItems
int expectedCount = (expectedTotalCount < skipCount) ? 0 : (expectedTotalCount-skipCount) < maxItems ? expectedTotalCount-skipCount : maxItems;
Assert.assertEquals(response.getPagination().getCount(), expectedCount, "Unexpected document count");
}
}
e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/SearchSpellCheckTest.java
+30 -24
View File
@@ -317,8 +317,11 @@ public class SearchSpellCheckTest extends AbstractSearchServicesE2ETest
Assert.assertTrue(isContentInSearchResponse(response, file2.getName()), "Expected file not returned in the search results: " + file.getName());
testSearchSpellcheckResponse(response, "searchInsteadFor", "eklipse");
// Add Solr Query, to check the suggestions on each shard
restClient.authenticateUser(testUser).withParams("spellcheck.q=eclipsess&spellcheck=on").withSolrAPI().getSelectQuery();
// Incorrect spelling with no field for file2
response = SearchSpellcheckQuery(testUser, "eclipses", "eclipses");
response = SearchSpellcheckQuery(testUser, "eclipsess", "eclipsess");
// Matching Result, Spellcheck = searchInsteadFor: eklipses
Assert.assertTrue(isContentInSearchResponse(response, file2.getName()), "Expected file not returned in the search results: " + file.getName());
@@ -422,12 +425,12 @@ public class SearchSpellCheckTest extends AbstractSearchServicesE2ETest
getDataUser().addUserToSite(testUser2, testSite2, UserRole.SiteCollaborator);
// Add file <spacebar> to testSite
FileModel file1 = new FileModel("spacebar", "", "", FileType.TEXT_PLAIN, "spacebar");
FileModel file1 = new FileModel("superbar", "", "", FileType.TEXT_PLAIN, "superbar");
dataContent.usingUser(testUser).usingSite(testSite).createContent(file1);
// Add file <spacecar> to testSite, testSite2
FileModel file2 = new FileModel("spacecar", "", "", FileType.TEXT_PLAIN, "spacecar");
// Add file <supercar> to testSite, testSite2
FileModel file2 = new FileModel("supercar", "", "", FileType.TEXT_PLAIN, "supercar");
dataContent.usingUser(testUser).usingSite(testSite).createContent(file2);
dataContent.usingUser(testUser).usingSite(testSite2).createContent(file2);
@@ -436,59 +439,62 @@ public class SearchSpellCheckTest extends AbstractSearchServicesE2ETest
// Checks for User 2
// Incorrect spelling with no field
SearchResponse response = SearchSpellcheckQuery(testUser, "spaceber", "spaceber");
SearchResponse response = SearchSpellcheckQuery(testUser, "superber", "superber");
// Matching Result, Spellcheck = searchInsteadFor: spacebar: alphabetical
// Matching Result, Spellcheck = searchInsteadFor: superbar: alphabetical
Assert.assertTrue(isContentInSearchResponse(response, file1.getName()), "Expected file not returned in the search results: " + file2.getName());
testSearchSpellcheckResponse(response, "searchInsteadFor", "spacebar");
testSearchSpellcheckResponse(response, "searchInsteadFor", "superbar");
// Correct spelling with no field
response = SearchSpellcheckQuery(testUser, "spacebar", "spacebar");
response = SearchSpellcheckQuery(testUser, "superbar", "superbar");
// Matching Result, Spellcheck = searchInsteadFor: spacebar
// Matching Result, Spellcheck = searchInsteadFor: superbar
Assert.assertTrue(isContentInSearchResponse(response, file1.getName()), "Expected file not returned in the search results: " + file1.getName());
testSearchSpellcheckResponse(response, "didYouMean", "spacecar");
testSearchSpellcheckResponse(response, "didYouMean", "supercar");
// Incorrect spelling with no field
response = SearchSpellcheckQuery(testUser, "spacebra", "spacebra");
response = SearchSpellcheckQuery(testUser, "superbra", "superbra");
// Matching Result, Spellcheck = searchInsteadFor: spacebar
// Matching Result, Spellcheck = searchInsteadFor: superbar
Assert.assertTrue(isContentInSearchResponse(response, file1.getName()), "Expected file not returned in the search results: " + file1.getName());
testSearchSpellcheckResponse(response, "searchInsteadFor", "spacebar");
testSearchSpellcheckResponse(response, "searchInsteadFor", "superbar");
// Correct spelling with no field
response = SearchSpellcheckQuery(testUser, "spacecar", "spacecar");
response = SearchSpellcheckQuery(testUser, "supercar", "supercar");
// Matching Result, Spellcheck Not returned
Assert.assertTrue(isContentInSearchResponse(response, file2.getName()), "Expected file not returned in the search results: " + file2.getName());
testSearchSpellcheckResponse(response, null, null);
// Add Solr Query, to check the suggestions on the shard
restClient.authenticateUser(testUser).withParams("spellcheck.q=spacebur&spellcheck=on").withSolrAPI().getSelectQuery();
// Checks for User 2
// Incorrect spelling for files created no field
response = SearchSpellcheckQuery(testUser2, "spaceber", "spaceber");
response = SearchSpellcheckQuery(testUser2, "superbur", "superbur");
// Matching Result, Spellcheck = searchInsteadFor: spacecar
// Matching Result, Spellcheck = searchInsteadFor: supercar
Assert.assertTrue(isContentInSearchResponse(response, file2.getName()), "Expected file not returned in the search results: " + file2.getName());
testSearchSpellcheckResponse(response, "searchInsteadFor", "spacecar");
testSearchSpellcheckResponse(response, "searchInsteadFor", "supercar");
// correct spelling no field
response = SearchSpellcheckQuery(testUser2, "spacebar", "spacebar");
response = SearchSpellcheckQuery(testUser2, "superbar", "superbar");
// Matching Result, Spellcheck = searchInsteadFor: spacecar
// Matching Result, Spellcheck = searchInsteadFor: supercar
Assert.assertFalse(isContentInSearchResponse(response, file1.getName()), "Expected file not returned in the search results: " + file1.getName());
Assert.assertTrue(isContentInSearchResponse(response, file2.getName()), "Expected file not returned in the search results: " + file2.getName());
testSearchSpellcheckResponse(response, "searchInsteadFor", "spacecar");
testSearchSpellcheckResponse(response, "searchInsteadFor", "supercar");
// Incorrect spelling no field
response = SearchSpellcheckQuery(testUser2, "spacecra", "spacecra");
response = SearchSpellcheckQuery(testUser2, "supercra", "supercra");
// Matching Result, Spellcheck = searchInsteadFor: spacebar
// Matching Result, Spellcheck = searchInsteadFor: superbar
Assert.assertFalse(isContentInSearchResponse(response, file1.getName()), "Expected file not returned in the search results: " + file1.getName());
Assert.assertTrue(isContentInSearchResponse(response, file2.getName()), "Expected file not returned in the search results: " + file2.getName());
testSearchSpellcheckResponse(response, "searchInsteadFor", "spacecar");
testSearchSpellcheckResponse(response, "searchInsteadFor", "supercar");
// Correct spelling no field
response = SearchSpellcheckQuery(testUser2, "spacecar", "spacecar");
response = SearchSpellcheckQuery(testUser2, "supercar", "supercar");
// Matching Result, Spellcheck not returned
Assert.assertFalse(isContentInSearchResponse(response, file1.getName()), "Expected file not returned in the search results: " + file1.getName());
e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/ShardInfoTest.java
+85 -2
View File
@@ -20,9 +20,13 @@ import static org.testng.Assert.assertNotNull;
import static org.testng.Assert.assertTrue;
import java.util.Arrays;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.google.common.collect.Sets;
import org.alfresco.rest.search.RestInstanceModel;
import org.alfresco.rest.search.RestShardInfoModel;
@@ -58,7 +62,9 @@ public class ShardInfoTest extends AbstractE2EFunctionalTest
RestShardInfoModel model = shardInfoModel.getModel();
assertEquals(model.getTemplate(), "rerank");
assertEquals(model.getMode(), "MASTER");
assertEquals(model.getShardMethod(), "DB_ID");
List<String> shardingMethods = Arrays.asList("DB_ID", "DB_ID_RANGE", "EXPLICIT_ID", "ACL_ID", "MOD_ACL_ID", "DATE", "PROPERTY");
String shardingMethod = model.getShardMethod();
assertTrue(shardingMethods.contains(shardingMethod), "Unexpected Sharding Method Found: " + shardingMethod);
assertTrue(model.getHasContent());
assertTrue(stores.contains(model.getStores()));
@@ -75,7 +81,7 @@ public class ShardInfoTest extends AbstractE2EFunctionalTest
assertTrue(baseUrls.contains(instance.getBaseUrl()));
// TODO: Ideally Solr Host and Port should be Parameterised
assertEquals(instance.getHost(), "search");
assertNotNull(instance.getHost(), "The solr host is not present");
assertEquals(instance.getPort().intValue(), 8983);
assertEquals(instance.getState(), "ACTIVE");
assertEquals(instance.getMode(), "MASTER");
@@ -101,6 +107,7 @@ public class ShardInfoTest extends AbstractE2EFunctionalTest
RestShardInfoModel model = shardInfoModel.getModel();
assertEquals(model.getTemplate(), "rerank");
assertEquals(model.getShardMethod(), "DB_ID");
assertEquals(model.getMode(), "MIXED");
assertTrue(model.getHasContent());
assertTrue(stores.contains(model.getStores()));
@@ -126,4 +133,80 @@ public class ShardInfoTest extends AbstractE2EFunctionalTest
restClient.authenticateUser(dataUser.createRandomTestUser()).withShardInfoAPI().getInfo();
restClient.assertStatusCodeIs(HttpStatus.INTERNAL_SERVER_ERROR);
}
/**
* This is a test to check that sharding is correctly working on bamboo and locally.
* Include test group 'sharding' on bamboo to enable this test to run
* @throws JsonProcessingException
*/
@Test(groups = { TestGroup.ACS_60n, TestGroup.SHARDING })
public void getShardInfoWith2OrMoreShards() throws JsonProcessingException
{
RestShardInfoModelCollection info = restClient.authenticateUser(dataUser.getAdminUser()).withShardInfoAPI().getInfo();
restClient.assertStatusCodeIs(HttpStatus.OK);
info.assertThat().entriesListIsNotEmpty();
assertEquals(info.getPagination().getTotalItems().intValue(), 2, "Pagination is: " + info.getPagination().getTotalItems().intValue() + " not expected value of 2");
Set<String> stores = Sets.newHashSet("workspace://SpacesStore", "archive://SpacesStore");
List<String> baseUrls = Arrays.asList("/solr/alfresco", "/solr/archive");
List<RestShardInfoModel> entries = info.getEntries();
Set<String> actualStores = entries.stream().map(shardInfoModel -> shardInfoModel.getModel().getStores()).collect(Collectors.toSet());
assertEquals(actualStores, stores, "The number of stores do not match the expected number of stores");
for (RestShardInfoModel shardInfoModel : entries)
{
RestShardInfoModel model = shardInfoModel.getModel();
assertEquals(model.getTemplate(), "rerank", "Template is not rerank, template found: "+ model.getTemplate());
assertEquals(model.getMode(), "MASTER", "Mode is not MASTER, mode found: "+ model.getMode());
assertTrue(model.getHasContent(), "There is no content on the shards");
assertTrue(model.getNumberOfShards()>=2, "Number of shards is not equal to or greater than 2");
List<String> shardingMethods = Arrays.asList("DB_ID", "DB_ID_RANGE", "EXPLICIT_ID", "ACL_ID", "MOD_ACL_ID", "DATE", "PROPERTY");
String shardingMethod = model.getShardMethod();
assertTrue(shardingMethods.contains(shardingMethod), "Unexpected Sharding Method Found: " + shardingMethod);
List<RestShardModel> shards = model.getShards();
assertNotNull(shards, "There are no shards present");
for (RestShardModel shardInstance : shards)
{
List<RestInstanceModel> instanceList = shardInstance.getInstances();
for (RestInstanceModel instanceX : instanceList)
{
assertTrue(baseUrls.contains(instanceX.getBaseUrl()), "The baseUrl is not present, baseUrl found is: " + instanceX.getBaseUrl());
assertEquals(instanceX.getState(), "ACTIVE", "Shard state is not ACTIVE, shard state is: " + instanceX.getState());
assertNotNull(instanceX.getPort(), "There is not port found for the instance");
assertEquals(instanceX.getMode(), "MASTER", "Mode is not MASTER, mode found: "+ instanceX.getMode());
assertTrue(instanceX.getTransactionsRemaining() >= 0, "Transactions remaining is not more than 0, transactions remaining: " + instanceX.getTransactionsRemaining());
String shardParams = (instanceX).getShardParams();
switch (shardingMethod)
{
case "MOD_ACL_ID":
break;
case "ACL_ID":
break;
case "DB_ID":
break;
case "DB_ID_RANGE":
assertTrue(shardParams.contains("shard.range="), "Shard Parameters Not as expected for the Shard Method: DB_ID_RANGE");
break;
case "DATE":
assertTrue(shardParams.contains("shard.key="), "Shard Parameters Not as expected for the Shard Method: DATE");
assertTrue(shardParams.contains("shard.date.grouping="), "Shard Parameters Not as expected for the Shard Method: DATE");
break;
case "PROPERTY":
assertTrue(shardParams.contains("shard.key="), "Shard Parameters Not as expected for the Shard Method: PROPERTY");
assertTrue(shardParams.contains("shard.regex="), "Shard Parameters Not as expected for the Shard Method: PROPERTY");
break;
case "EXPLICIT_ID":
assertTrue(shardParams.contains("shard.key="), "Shard Parameters Not as expected for the Shard Method: EXPLICIT_ID");
break;
default:
throw new AssertionError("Shard Method Not expected: " + model.getShardMethod());
}
}
}
}
}
}
e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/StatsSearchTest.java
+3 -1
View File
@@ -322,7 +322,9 @@ public class StatsSearchTest extends AbstractSearchServicesE2ETest
response.getContext().assertThat().field("facets").isNotEmpty();
RestGenericFacetResponseModel facetResponseModel = response.getContext().getFacets().get(0);
facetResponseModel.assertThat().field("type").is("stats");
facetResponseModel.assertThat().field("label").is(label);
// TODO : Change back to facetResponseModel.assertThat().field("label").is(label);
// when https://issues.alfresco.com/jira/browse/SEARCH-2125 is fixed
facetResponseModel.assertThat().field("label").contains(label);
RestGenericBucketModel bucket = facetResponseModel.getBuckets().get(0);
List<RestGenericMetricModel> metrics = bucket.getMetrics();
assertEquals(metrics.size(),metricsCount);
e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/tracker/CascadingTrackerIntegrationTest.java → e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/search/tracker/CascadingIntegrationTest.java
+1 -1
View File
@@ -31,7 +31,7 @@ import static org.testng.Assert.assertTrue;
* @author Alessandro Benedetti
* @author Meenal Bhave
*/
public class CascadingTrackerIntegrationTest extends AbstractE2EFunctionalTest
public class CascadingIntegrationTest extends AbstractE2EFunctionalTest
{
@Autowired
protected DataContent dataContent;
e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/solr/admin/SolrE2eAdminTest.java
+973
View File
@@ -0,0 +1,973 @@
/*
* Copyright (C) 2020 Alfresco Software Limited.
* This file is part of Alfresco
* Alfresco is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
* Alfresco is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
* You should have received a copy of the GNU Lesser General Public License
* along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
*/
package org.alfresco.test.search.functional.searchServices.solr.admin;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import org.alfresco.rest.core.RestResponse;
import org.alfresco.search.TestGroup;
import org.alfresco.test.search.functional.AbstractE2EFunctionalTest;
import org.springframework.context.annotation.Configuration;
import org.testng.Assert;
import org.testng.annotations.Test;
/**
* End to end tests for SOLR Admin actions REST API, available from:
*
* http://<server>:<port>/solr/admin/cores?action=(actionName)
*
* @author aborroy
*
*/
@Configuration
public class SolrE2eAdminTest extends AbstractE2EFunctionalTest
{
// SOLR default response status codes (returned in responseHeader.status)
private static final Integer SOLR_RESPONSE_STATUS_OK = 0;
// Alfresco SOLR action response status identifiers
private static final String ACTION_RESPONSE_REPORT = "report";
// Default Alfresco SOLR Core Names
private static final List<String> DEFAULT_CORE_NAMES = new ArrayList<>(List.of("alfresco", "archive"));
/**
* Check that SOLR Response Header contains a Query Time (qtime) and status equals to 0
* @param response SOLR REST API Response in JSON
*/
private void checkResponseStatusOk(RestResponse response)
{
Integer qtime = response.getResponse().body().jsonPath().get("responseHeader.QTime");
Assert.assertTrue(qtime >= 0, "Expeted responseHeader.QTime to be a positive number");
Integer status = response.getResponse().body().jsonPath().get("responseHeader.status");
Assert.assertEquals(status, SOLR_RESPONSE_STATUS_OK, "Expected " + SOLR_RESPONSE_STATUS_OK + " in responseHeader.status,");
}
/**
* Node Report for every core.
* @throws Exception
*/
@Test(priority = 1)
public void testNodeReport() throws Exception
{
Integer nodeid = 200;
RestResponse response = restClient.withParams("nodeid=" + nodeid).withSolrAdminAPI().getAction("nodeReport");
checkResponseStatusOk(response);
DEFAULT_CORE_NAMES.forEach(core -> {
Integer reportNodeid = response.getResponse().body().jsonPath().get(ACTION_RESPONSE_REPORT + "." + core + ".'Node DBID'");
Assert.assertEquals(reportNodeid, nodeid, "Expected " + nodeid + " in " + ACTION_RESPONSE_REPORT + "." + core + ".'Node DBID',");
});
}
/**
* Node Report for an specific core.
* @throws Exception
*/
@Test(priority = 2)
public void testNodeReportCore() throws Exception
{
final Integer nodeid = 200;
DEFAULT_CORE_NAMES.forEach(core -> {
try
{
RestResponse response = restClient.withParams("nodeid=" + nodeid, "core=" + core).withSolrAdminAPI().getAction("nodeReport");
checkResponseStatusOk(response);
Integer reportNodeid = response.getResponse().body().jsonPath().get(ACTION_RESPONSE_REPORT + "." + core + ".'Node DBID'");
Assert.assertEquals(reportNodeid, nodeid, "Expected " + nodeid + " in " + ACTION_RESPONSE_REPORT + "." + core + ".'Node DBID',");
}
catch (Exception e)
{
throw new RuntimeException(e);
}
});
}
/**
* Node Report requires "nodeid" parameter.
* This test will return an error as we are missing to pass the parameter.
* @throws Exception
*/
@Test(priority = 3)
public void testNodeReportError() throws Exception
{
RestResponse response = restClient.withSolrAdminAPI().getAction("nodeReport");
checkResponseStatusOk(response);
String reportError = response.getResponse().body().jsonPath().get(ACTION_RESPONSE_REPORT + ".error");
Assert.assertEquals(reportError, "No nodeid parameter set.", "Unexpected message in " + ACTION_RESPONSE_REPORT + ".error,");
}
/**
* ACL Report for every core.
* @throws Exception
*/
@Test(priority = 4)
public void testAclReport() throws Exception
{
Integer aclid = 1;
RestResponse response = restClient.withParams("aclid=" + aclid).withSolrAdminAPI().getAction("aclReport");
checkResponseStatusOk(response);
DEFAULT_CORE_NAMES.forEach(core -> {
Integer reportAclid = response.getResponse().body().jsonPath().get(ACTION_RESPONSE_REPORT + "." + core + ".'Acl Id'");
Assert.assertEquals(reportAclid, aclid, "Expected " + aclid + " in " + ACTION_RESPONSE_REPORT + "." + core + ".'Acl Id',");
});
}
/**
* ACL Report for an specific core.
* @throws Exception
*/
@Test(priority = 5)
public void testAclReportCore() throws Exception
{
final Integer aclid = 1;
DEFAULT_CORE_NAMES.forEach(core -> {
try
{
RestResponse response = restClient.withParams("aclid=" + aclid, "core=" + core).withSolrAdminAPI().getAction("aclReport");
checkResponseStatusOk(response);
Integer reportAclid = response.getResponse().body().jsonPath().get(ACTION_RESPONSE_REPORT + "." + core + ".'Acl Id'");
Assert.assertEquals(reportAclid, aclid, "Expected " + aclid + " in " + ACTION_RESPONSE_REPORT + "." + core + ".'Acl Id',");
}
catch (Exception e)
{
throw new RuntimeException(e);
}
});
}
/**
* ACL Report requires "aclid" parameter.
* This test will fail as we are missing to pass the parameter.
* @throws Exception
*/
@Test(priority = 6)
public void testAclReportError() throws Exception
{
RestResponse response = restClient.withSolrAdminAPI().getAction("aclReport");
checkResponseStatusOk(response);
String reportError = response.getResponse().body().jsonPath().get(ACTION_RESPONSE_REPORT + ".error");
Assert.assertEquals(reportError, "No aclid parameter set.", "Unexpected message in " + ACTION_RESPONSE_REPORT + ".error,");
}
/**
* TX Report for every core.
* @throws Exception
*/
@Test(priority = 7)
public void testTxReport() throws Exception
{
Integer txid = 1;
RestResponse response = restClient.withParams("txid=" + txid).withSolrAdminAPI().getAction("txReport");
checkResponseStatusOk(response);
DEFAULT_CORE_NAMES.forEach(core -> {
Integer reportTxid = response.getResponse().body().jsonPath().get(ACTION_RESPONSE_REPORT + "." + core + ".TXID");
Assert.assertEquals(reportTxid, txid, "Expected " + txid + " in " + ACTION_RESPONSE_REPORT + "." + core + ".TXID,");
});
}
/**
* TX Report for an specific core.
* @throws Exception
*/
@Test(priority = 8)
public void testTxReportCore() throws Exception
{
final Integer txid = 1;
DEFAULT_CORE_NAMES.forEach(core -> {
try
{
RestResponse response = restClient.withParams("coreName=" + core, "txid=" + txid).withSolrAdminAPI().getAction("txReport");
checkResponseStatusOk(response);
Integer reportTxid = response.getResponse().body().jsonPath().get(ACTION_RESPONSE_REPORT + "." + core + ".TXID");
Assert.assertEquals(reportTxid, txid, "Expected " + txid + " in " + ACTION_RESPONSE_REPORT + "." + core + ".TXID,");
}
catch (Exception e)
{
throw new RuntimeException(e);
}
});
}
/**
* Transaction report requires "txid" parameter.
* This test will fail as we are missing to pass the parameter.
* @throws Exception
*/
@Test(priority = 9)
public void testTxReportError() throws Exception
{
String coreName = "alfresco";
RestResponse response = restClient.withParams("coreName=" + coreName).withSolrAdminAPI().getAction("txReport");
checkResponseStatusOk(response);
String reportError = response.getResponse().body().jsonPath().get(ACTION_RESPONSE_REPORT + ".error");
Assert.assertEquals(reportError, "No txid parameter set.", "Unexpected message in " + ACTION_RESPONSE_REPORT + ".error,");
}
/**
* ACL TX Report for every core.
* @throws Exception
*/
@Test(priority = 10)
public void testAclTxReport() throws Exception
{
Integer acltxid = 1;
RestResponse response = restClient.withParams("acltxid=" + acltxid).withSolrAdminAPI().getAction("aclTxReport");
checkResponseStatusOk(response);
DEFAULT_CORE_NAMES.forEach(core -> {
Integer reportAcltxidCount = response.getResponse().body().jsonPath().get(ACTION_RESPONSE_REPORT + "." + core + ".aclTxDbAclCount");
Assert.assertEquals(reportAcltxidCount, Integer.valueOf(2), "Expected 2 in " + ACTION_RESPONSE_REPORT + "." + core + ".aclTxDbAclCount,");
});
}
/**
* ACL TX Report for specific core.
* @throws Exception
*/
@Test(priority = 11)
public void testAclTxReportCore() throws Exception
{
final Integer acltxid = 1;
DEFAULT_CORE_NAMES.forEach(core -> {
try
{
RestResponse response = restClient.withParams("acltxid=" + acltxid, "core=" + core).withSolrAdminAPI().getAction("aclTxReport");
checkResponseStatusOk(response);
Integer reportAcltxidCount = response.getResponse().body().jsonPath().get(ACTION_RESPONSE_REPORT + "." + core + ".aclTxDbAclCount");
Assert.assertEquals(reportAcltxidCount, Integer.valueOf(2), "Expected 2 in " + ACTION_RESPONSE_REPORT + "." + core + ".aclTxDbAclCount,");
}
catch (Exception e)
{
throw new RuntimeException(e);
}
});
}
/**
* AclTx report requires "acltxid" parameter.
* This test will fail as we are missing to pass the parameter.
* @throws Exception
*/
@Test(priority = 12)
public void testAclTxReportError() throws Exception
{
RestResponse response = restClient.withSolrAdminAPI().getAction("aclTxReport");
checkResponseStatusOk(response);
String reportError = response.getResponse().body().jsonPath().get(ACTION_RESPONSE_REPORT + ".error");
Assert.assertEquals(reportError, "No acltxid parameter set.", "Unexpected message in " + ACTION_RESPONSE_REPORT + ".error,");
}
/**
* Report for every core.
* @throws Exception
*/
@Test(priority = 13)
public void testReport() throws Exception
{
RestResponse response = restClient.withSolrAdminAPI().getAction(ACTION_RESPONSE_REPORT);
checkResponseStatusOk(response);
DEFAULT_CORE_NAMES.forEach(core -> {
Integer reportTxCount = response.getResponse().body().jsonPath().get(ACTION_RESPONSE_REPORT + "." + core + ".'DB transaction count'");
Assert.assertTrue(reportTxCount > 0, "Expecting a positive integer in " + ACTION_RESPONSE_REPORT + "." + core + ".'DB transaction count',");
});
}
/**
* Report for specific core.
* @throws Exception
*/
@Test(priority = 14)
public void testReportCore() throws Exception
{
DEFAULT_CORE_NAMES.forEach(core -> {
try
{
RestResponse response = restClient.withParams("coreName=" + core).withSolrAdminAPI().getAction(ACTION_RESPONSE_REPORT);
checkResponseStatusOk(response);
Integer reportTxCount = response.getResponse().body().jsonPath().get(ACTION_RESPONSE_REPORT + "." + core + ".'DB transaction count'");
Assert.assertTrue(reportTxCount > 0, "Expecting a positive integer in " + ACTION_RESPONSE_REPORT + "." + core + ".'DB transaction count',");
}
catch (Exception e)
{
throw new RuntimeException(e);
}
});
}
/**
* Report using params.
* @throws Exception
*/
@Test(priority = 15)
public void testReportWithParams() throws Exception
{
Long fromTime = 0l;
Long toTime = 0l;
RestResponse response = restClient.withParams("fromTime=" + fromTime, "toTime=" + toTime).withSolrAdminAPI()
.getAction(ACTION_RESPONSE_REPORT);
checkResponseStatusOk(response);
DEFAULT_CORE_NAMES.forEach(core -> {
Integer reportTxCount = response.getResponse().body().jsonPath().get(ACTION_RESPONSE_REPORT + "." + core + ".'DB transaction count'");
Assert.assertTrue(reportTxCount == 0, "Expecting 0 in " + ACTION_RESPONSE_REPORT + "." + core + ".'DB transaction count',");
});
}
/**
* Summary for every core.
* @throws Exception
*/
@Test(priority = 16)
public void testSummary() throws Exception
{
RestResponse response = restClient.withSolrAdminAPI().getAction("summary");
checkResponseStatusOk(response);
DEFAULT_CORE_NAMES.forEach(core -> {
Integer reportTxCount = response.getResponse().body().jsonPath().get("Summary." + core + ".'Alfresco Transactions in Index'");
Assert.assertTrue(reportTxCount > 0, "Expecting a positive integer in Summary." + core + ".'Alfresco Transactions in Index',");
});
}
/**
* Summary for specific core.
* @throws Exception
*/
@Test(priority = 17)
public void testSummaryCore() throws Exception
{
DEFAULT_CORE_NAMES.forEach(core -> {
try
{
RestResponse response = restClient.withParams("core=" + core).withSolrAdminAPI().getAction("summary");
checkResponseStatusOk(response);
Integer reportTxCount = response.getResponse().body().jsonPath().get("Summary." + core + ".'Alfresco Transactions in Index'");
Assert.assertTrue(reportTxCount > 0, "Expecting a positive integer in Summary." + core + ".'Alfresco Transactions in Index',");
}
catch (Exception e)
{
throw new RuntimeException(e);
}
});
}
/**
* Check every core.
* @throws Exception
*/
@Test(priority = 18)
public void testCheck() throws Exception
{
RestResponse response = restClient.withSolrAdminAPI().getAction("check");
checkResponseStatusOk(response);
String actionStatus = response.getResponse().body().jsonPath().get("action.status");
Assert.assertEquals(actionStatus, "success");
}
/**
* Check specific core.
* @throws Exception
*/
@Test(priority = 19)
public void testCheckCore() throws Exception
{
DEFAULT_CORE_NAMES.forEach(core -> {
try
{
RestResponse response = restClient.withParams("core=" + core).withSolrAdminAPI().getAction("check");
checkResponseStatusOk(response);
String actionStatus = response.getResponse().body().jsonPath().get("action.status");
Assert.assertEquals(actionStatus, "success");
}
catch (Exception e)
{
throw new RuntimeException(e);
}
});
}
/**
* This action only applies to DB_ID_RANGE Sharding method.
* This test verifies expected result when using another deployment
* @throws Exception
*/
@Test(priority = 20)
public void testRangeCheck() throws Exception
{
String coreName = "alfresco";
RestResponse response = restClient.withParams("coreName=" + coreName).withSolrAdminAPI().getAction("rangeCheck");
checkResponseStatusOk(response);
Integer expand = response.getResponse().body().jsonPath().get("expand");
Assert.assertEquals(expand, Integer.valueOf(-1), "Expansion is not allowed when not using Shard DB_ID_RANGE method,");
}
/**
* When using DB_ID_RANGE Sharding method, expand param is including a number of nodes to be extended.
* @throws Exception
*/
@Test(priority = 21, groups = { TestGroup.ASS_SHARDING_DB_ID_RANGE })
public void testRangeCheckSharding() throws Exception
{
String coreName = "alfresco";
RestResponse response = restClient.withParams("coreName=" + coreName).withSolrAdminAPI().getAction("rangeCheck");
checkResponseStatusOk(response);
Integer expand = response.getResponse().body().jsonPath().get("expand");
Assert.assertNotEquals(expand, Integer.valueOf(-1), "Expansion is a positive number when using Shard DB_ID_RANGE method,");
}
/**
* This action only applies to DB_ID_RANGE Sharding method.
* This test verifies expected result when using another deployment
* @throws Exception
*/
@Test(priority = 22)
public void testExpand() throws Exception
{
String coreName = "alfresco";
String add = "1000";
RestResponse response = restClient.withParams("coreName=" + coreName, "add=" + add).withSolrAdminAPI().getAction("expand");
checkResponseStatusOk(response);
// This action only applies to DB_ID_RANGE Sharding method
Integer expand = response.getResponse().body().jsonPath().get("expand");
Assert.assertEquals(expand, Integer.valueOf(-1), "Expansion is not allowed when not using Shard DB_ID_RANGE method,");
}
/**
* When using DB_ID_RANGE Sharding method, expand param is including a number of nodes extended.
* @throws Exception
*/
@Test(priority = 23, groups = { TestGroup.ASS_SHARDING_DB_ID_RANGE })
public void testExpandSharding() throws Exception
{
String coreName = "alfresco";
String add = "1000";
RestResponse response = restClient.withParams("coreName=" + coreName, "add=" + add).withSolrAdminAPI().getAction("expand");
checkResponseStatusOk(response);
// This action only applies to DB_ID_RANGE Sharding method
Integer expand = response.getResponse().body().jsonPath().get("expand");
Assert.assertNotEquals(expand, Integer.valueOf(-1), "Expansion is a positive number when using Shard DB_ID_RANGE method,");
}
/**
* Purge TX in every core.
* @throws Exception
*/
@Test(priority = 24)
public void testPurge() throws Exception
{
Integer txid = 1;
RestResponse response = restClient.withParams("txid=" + txid).withSolrAdminAPI().getAction("purge");
checkResponseStatusOk(response);
String actionStatus = response.getResponse().body().jsonPath().get("action.status");
Assert.assertEquals(actionStatus, "scheduled");
}
/**
* Purge TX in specific core.
* @throws Exception
*/
@Test(priority = 25)
public void testPurgeCore() throws Exception
{
final Integer txid = 1;
DEFAULT_CORE_NAMES.forEach(core -> {
try
{
RestResponse response = restClient.withParams("core=" + core, "txid=" + txid).withSolrAdminAPI().getAction("purge");
checkResponseStatusOk(response);
String actionStatus = response.getResponse().body().jsonPath().get("action.status");
Assert.assertEquals(actionStatus, "scheduled");
}
catch (Exception e)
{
throw new RuntimeException(e);
}
});
}
/**
* Purge with no params produces an empty response.
* @throws Exception
*/
@Test(priority = 26)
public void testPurgeEmpty() throws Exception
{
RestResponse response = restClient.withSolrAdminAPI().getAction("purge");
checkResponseStatusOk(response);
String actionStatus = response.getResponse().body().jsonPath().get("action.status");
Assert.assertEquals(actionStatus, "scheduled");
}
/**
* FIX for every core.
* @throws Exception
*/
@Test(priority = 27, dependsOnMethods = "testPurge")
public void testFix() throws Exception
{
RestResponse response = restClient.withSolrAdminAPI().getAction("fix");
checkResponseStatusOk(response);
DEFAULT_CORE_NAMES.forEach(core -> {
List<String> txToReindex = response.getResponse().body().jsonPath().get("action." + core +".txToReindex");
Assert.assertTrue(txToReindex.size() >= 0, "Expected a list of transactions (or empty list) to be reindexed,");
List<String> aclToReindex = response.getResponse().body().jsonPath().get("action." + core + ".aclChangeSetToReindex");
Assert.assertTrue(aclToReindex.size() >= 0, "Expected a list of ACLs (or empty list) to be reindexed,");
});
String actionStatus = response.getResponse().body().jsonPath().get("action.status");
Assert.assertEquals(actionStatus, "scheduled");
}
/**
* FIX for specific core.
* @throws Exception
*/
@Test(priority = 28)
public void testFixCore() throws Exception
{
DEFAULT_CORE_NAMES.forEach(core -> {
try
{
RestResponse response = restClient.withParams("core=" + core).withSolrAdminAPI().getAction("fix");
checkResponseStatusOk(response);
List<String> txToReindex = response.getResponse().body().jsonPath().get("action." + core +".txToReindex");
Assert.assertTrue(txToReindex.size() >= 0, "Expected a list of transactions (or empty list) to be reindexed,");
List<String> aclToReindex = response.getResponse().body().jsonPath().get("action." + core + ".aclChangeSetToReindex");
Assert.assertTrue(aclToReindex.size() >= 0, "Expected a list of ACLs (or empty list) to be reindexed,");
String actionStatus = response.getResponse().body().jsonPath().get("action.status");
Assert.assertEquals(actionStatus, "scheduled");
}
catch (Exception e)
{
throw new RuntimeException(e);
}
});
}
/**
* REINDEX for every core.
* @throws Exception
*/
@Test(priority = 29)
public void testReindex() throws Exception
{
Integer txid = 1;
RestResponse response = restClient.withParams("txid=" + txid).withSolrAdminAPI().getAction("reindex");
checkResponseStatusOk(response);
String actionStatus = response.getResponse().body().jsonPath().get("action.status");
Assert.assertEquals(actionStatus, "scheduled");
}
/**
* REINDEX for specific core.
* @throws Exception
*/
@Test(priority = 30)
public void testReindexCore() throws Exception
{
Integer txid = 1;
DEFAULT_CORE_NAMES.forEach(core -> {
try
{
RestResponse response = restClient.withParams("core=" + core, "txid=" + txid).withSolrAdminAPI().getAction("reindex");
checkResponseStatusOk(response);
String actionStatus = response.getResponse().body().jsonPath().get("action.status");
Assert.assertEquals(actionStatus, "scheduled");
}
catch (Exception e)
{
throw new RuntimeException(e);
}
});
}
/**
* RETRY for every core.
* @throws Exception
*/
@Test(priority = 31)
public void testRetry() throws Exception
{
RestResponse response = restClient.withSolrAdminAPI().getAction("retry");
checkResponseStatusOk(response);
String actionStatus = response.getResponse().body().jsonPath().get("action.status");
Assert.assertEquals(actionStatus, "scheduled");
DEFAULT_CORE_NAMES.forEach(core -> {
List<String> errorNodeList = response.getResponse().body().jsonPath().get("action." + core);
Assert.assertEquals(errorNodeList, Arrays.asList(), "Expected no error nodes,");
});
}
/**
* RETRY for specific core.
* @throws Exception
*/
@Test(priority = 32)
public void testRetryCore() throws Exception
{
DEFAULT_CORE_NAMES.forEach(core -> {
try
{
RestResponse response = restClient.withParams("core=" + core).withSolrAdminAPI().getAction("retry");
checkResponseStatusOk(response);
String actionStatus = response.getResponse().body().jsonPath().get("action.status");
Assert.assertEquals(actionStatus, "scheduled");
List<String> errorNodeList = response.getResponse().body().jsonPath().get("action." + core);
Assert.assertEquals(errorNodeList, Arrays.asList(), "Expected no error nodes,");
}
catch (Exception e)
{
throw new RuntimeException(e);
}
});
}
/**
* INDEX for every core.
* @throws Exception
*/
@Test(priority = 33)
public void testIndex() throws Exception
{
Integer txid = 1;
RestResponse response = restClient.withParams("txid=" + txid).withSolrAdminAPI().getAction("index");
checkResponseStatusOk(response);
String actionStatus = response.getResponse().body().jsonPath().get("action.status");
Assert.assertEquals(actionStatus, "scheduled");
}
/**
* INDEX for specific core.
* @throws Exception
*/
@Test(priority = 34)
public void testIndexCore() throws Exception
{
final Integer txid = 1;
DEFAULT_CORE_NAMES.forEach(core -> {
try
{
RestResponse response = restClient.withParams("core=" + core, "txid=" + txid).withSolrAdminAPI().getAction("index");
checkResponseStatusOk(response);
String actionStatus = response.getResponse().body().jsonPath().get("action.status");
Assert.assertEquals(actionStatus, "scheduled");
}
catch (Exception e)
{
throw new RuntimeException(e);
}
});
}
/**
* Reloads default log4j properties into memory.
* @throws Exception
*/
@Test(priority = 35)
public void testLog4J() throws Exception
{
RestResponse response = restClient.withSolrAdminAPI().getAction("log4j");
checkResponseStatusOk(response);
String actionStatus = response.getResponse().body().jsonPath().get("action.status");
Assert.assertEquals(actionStatus, "success");
}
/**
* This REST API call will fail as the specified resource to reload doesn't exist.
* @throws Exception
*/
@Test(priority = 36)
public void testLog4JError() throws Exception
{
RestResponse response = restClient.withParams("resource=log4j-unexisting.properties").withSolrAdminAPI().getAction("log4j");
checkResponseStatusOk(response);
String actionStatus = response.getResponse().body().jsonPath().get("action.status");
Assert.assertEquals(actionStatus, "error");
}
/**
* This test will fail if it's executed twice
* @throws Exception
*/
@Test(priority = 37)
public void testNewCore() throws Exception
{
String core = "newCore";
String storeRef = "workspace://SpacesStore";
String template = "rerank";
RestResponse response = restClient.withParams("coreName=" + core, "storeRef=" + storeRef, "template=" + template)
.withSolrAdminAPI().getAction("newCore");
checkResponseStatusOk(response);
String actionStatus = response.getResponse().body().jsonPath().get("action.status");
Assert.assertEquals(actionStatus, "success");
String actionCore = response.getResponse().body().jsonPath().get("action.core");
Assert.assertEquals(actionCore, core, "Created core name is expected in action.core,");
}
/**
* When creating a core that already exists, this action fails.
* @throws Exception
*/
@Test(priority = 38)
public void testNewCoreError() throws Exception
{
String core = "alfresco";
String template = "rerank";
RestResponse response = restClient.withParams("coreName=" + core, "template=" + template)
.withSolrAdminAPI().getAction("newCore");
checkResponseStatusOk(response);
String actionStatus = response.getResponse().body().jsonPath().get("action.status");
Assert.assertEquals(actionStatus, "error");
}
/**
* Reloads core configuration in memory.
* @throws Exception
*/
@Test(priority = 39)
public void testUpdateCore() throws Exception
{
String core = "alfresco";
RestResponse response = restClient.withParams("coreName=" + core).withSolrAdminAPI().getAction("updateCore");
checkResponseStatusOk(response);
String actionStatus = response.getResponse().body().jsonPath().get("action.status");
Assert.assertEquals(actionStatus, "success");
}
/**
* When updating a core that doesn't exist, this action fails.
* @throws Exception
*/
@Test(priority = 40)
public void testUpdateCoreError() throws Exception
{
String core = "nonExistingCore";
RestResponse response = restClient.withParams("coreName=" + core).withSolrAdminAPI().getAction("updateCore");
checkResponseStatusOk(response);
String actionStatus = response.getResponse().body().jsonPath().get("action.status");
Assert.assertEquals(actionStatus, "error");
}
/**
* This test updates "shared.properties" memory loading for every SOLR core.
* @throws Exception
*/
@Test(priority = 41)
public void testUpdateShared() throws Exception
{
RestResponse response = restClient.withSolrAdminAPI().getAction("updateShared");
checkResponseStatusOk(response);
String actionStatus = response.getResponse().body().jsonPath().get("action.status");
Assert.assertEquals(actionStatus, "success");
}
/**
* This test will fail if it's executed twice
* @throws Exception
*/
@Test(priority = 42)
public void testNewDefaultCore() throws Exception
{
String core = "newDefaultCore";
String storeRef = "workspace://SpacesStore";
String template = "rerank";
RestResponse response = restClient
.withParams("coreName=" + core, "storeRef=" + storeRef, "template=" + template)
.withSolrAdminAPI().getAction("newDefaultIndex");
checkResponseStatusOk(response);
String actionStatus = response.getResponse().body().jsonPath().get("action.status");
Assert.assertEquals(actionStatus, "success");
String actionCore = response.getResponse().body().jsonPath().get("action.core");
Assert.assertEquals(actionCore, core, "Created core name is expected in action.core,");
}
/**
* When creating a core that already exists, this action fails.
* @throws Exception
*/
@Test(priority = 43)
public void testNewDefaultCoreError() throws Exception
{
String core = "alfresco";
String template = "rerank";
RestResponse response = restClient.withParams("coreName=" + core, "template=" + template)
.withSolrAdminAPI().getAction("newDefaultIndex");
checkResponseStatusOk(response);
String actionStatus = response.getResponse().body().jsonPath().get("action.status");
Assert.assertEquals(actionStatus, "error");
}
/**
* This test has to be executed after "testNewCore" test, otherwise it will fail
*/
@Test(priority = 99, dependsOnMethods = ("testNewCore"))
public void testRemoveCore() throws Exception
{
String core = "newCore";
String storeRef = "workspace://SpacesStore";
RestResponse response = restClient
.withParams("coreName=" + core, "storeRef=" + storeRef)
.withSolrAdminAPI().getAction("removeCore");
checkResponseStatusOk(response);
String actionStatus = response.getResponse().body().jsonPath().get("action.status");
Assert.assertEquals(actionStatus, "success");
}
}
e2e-test/src/test/resources/SearchSuite.xml
+14
View File
@@ -20,7 +20,21 @@
<exclude name="org.alfresco.test.search.functional.searchServices.search.rm"/>
</package>
</packages>
<!-- Despite this class is included in Search Services package, needs to be excluded in order to be executed as the last one -->
<classes>
<class name="org.alfresco.test.search.functional.searchServices.solr.admin.SolrE2eAdminTest">
<methods>
<exclude name=".*" />
</methods>
</class>
</classes>
</test>
<!-- This is deliberately scheduled at the end of the test suite because it messes with the cores and might break other tests -->
<test name="Admin">
<classes>
<class name="org.alfresco.test.search.functional.searchServices.solr.admin.SolrE2eAdminTest" />
</classes>
</test>
</suite>
search-services/README.md
+13
View File
@@ -433,11 +433,24 @@ cd packaging/target/docker-resources/6.x
docker-compose up
```
##Docker Master-Slave setup
We have seperate docker compose file for slave. To setup Master slave setup
`docker-compose -f docker-compose.yml -f ./master-slave/docker-compose.slave.yml up`
The slave running behind the nginx load balancer under 8084, so we can spin up multiple slaves with the same port. To deploy multiple slaves
`docker-compose -f docker-compose.yml -f ./master-slave/docker-compose.slave.yml up --scale search_slave=2`
This will start up Alfresco, Postgres, Share and SearchServices. You can access the applications using the following URLs:
* Alfresco: http://localhost:8081/alfresco
* Share: http://localhost:8082/share
* Solr: http://localhost:8083/solr
* Solr-slave: http://localhost:8084/solr
Note: Once deployed and all services up and running, goto <http://localhost:8081/alfresco/s/enterprise/admin/admin-searchservice?m=admin-console.success>. Scroll down and click save button.
This is the bug which currently working on [SEARCH-2085](https://issues.alfresco.com/jira/browse/SEARCH-2085)
If you start version 5.x instead you can also access the API Explorer:
search-services/alfresco-search/doc/architecture/decisions/0007-message-driven-content-tracker.md
+1 -1
View File
@@ -4,7 +4,7 @@ Date: 09/04/2019
## Status
Approved
~~Approved~~ Postponed, see [version 2](0009-message-driven-content-tracker-next-gen.md)
## Context
search-services/alfresco-search/doc/architecture/decisions/0009-message-driven-content-tracker-next-gen.md
+28
View File
@@ -0,0 +1,28 @@
# Message Driven Content Tracker v2
Date: 05/02/2020
## Status
WIP
## Context
This is a second iteration of content tracking via message bus design. See [previous version](0007-message-driven-content-tracker.md).
New content tracker implementation will be based on new Search Services architecture (SS v2.0 or next gen). Main context behind this decision is almost the same as for v1 - get more throughput by leveraging new Transform Service.
## Decision
The decision is based on [version 1](0007-message-driven-content-tracker.md). The main differences are:
* Shared File Store may not be the right option as it is only available for Enterprise. Alternatively the URL to content can point to other locations. (TBC)
* The change in behaviour requires a major release of Search Services, most likely version 2.0.
* The changes in Content Repository will be available from version 6.3.
* The synchronous transformation APIs will remain functional until 7.0.
Details of the architecture to be clarified (WIP).
## Consequences
Additional latency will be introduced due to the extra calls when using the Transform Service. Also the transformation capabilities are much more limited.
This design highlights a major difference in behaviour, which requires a major release of Search Services.
search-services/alfresco-search/doc/architecture/solr-content-store-removal/00001-solr-content-store-removal.md
+436
View File
@@ -0,0 +1,436 @@
## SolrContentStore Removal
### Status
![Completeness Badge](https://img.shields.io/badge/Document_Level-Completed-green.svg?style=flat-square)
### Context
SearchServices is a set of Alfresco specific customisations built on top of Apache Solr, a highly reliable, scalable and
fault tolerant search platform.
Apache Solr provides efficient search services on a given set of data composed by atomic units of work called "documents".
Data managed in Alfresco that needs to be "searchable" must be sent to Solr for "indexing". In the indexing phase Solr
stores the incoming data and organises it in an immutable data structure called "Inverted Index" plus some additional data
structures needed for other complementary services offered by the platform (e.g. highlighting, storage, more like this).
The following picture illustrates the content of the Solr data directory
![Apache Solr Data Directory](solr.data.dir.png)
SearchServices adds a complementary and auxiliary data organisation structure, based on filesystem, called **Solr Content Store**.
The following picture illustrates the content and the structure of the Solr Content Store.
![SolrContentStore](solrcontentstore.png)
### What is the SolrContentStore?
The _SolrContentStore_ is a logical extension of the Apache Solr Index used by SearchServices for maintaining a verbatim copy of
each incoming data. It is a local folder organised by tenant which contains all input documents indexed in Solr.
Within that folder, each file
- is organised hierarchically, under a root folder called "contentstore", and per tenant
- represents a single document sent to Solr for indexing
- is compressed (.gz) and serialised. Specifically, it consists of the serialised form of a _SolrInputDocument_ instance,
the Java class used by Solr for representing an incoming document that is going to be indexed.
Data that needs to be indexed is retrieved from Alfresco (_Node_ is the composite class representing the main Alfresco
Domain Object) and then
- each _Node_ instance is converted to a _SolrInputDocument_ instance
- each _SolrInputDocument_ instance is compressed, serialised and then stored in the content store
- each _SolrInputDocument_ instance is sent to Solr
![SolrContentStore](stored_content_in_searchservices.png)
With that flow in mind, at a given time T, the main difference between a document D in the content store and in the Solr
index is that:
- the content store file represents a verbatim copy of the _SolrInputDocument_ created starting from the corresponding _Node_
- it can be easily individuated because it corresponds to a single file in the content store; for comparison the Solr document definition
doesn't have a "single" representation in the filesystem because it has been passed through the text analysis process.
### Apache Solr Domain Model
In order to understand the reason why the content store approach has been adopted until SearchServices 1.4.x, we need to
briefly describe how Solr manages the fields of the managed documents.
In Solr, the configuration file where fields are declared and configured is called "schema.xml". Each field can have
different attributes that define
- how it is internally organised
- what search features are enabled (for that specific field)
In this context we are interested in two specific attributes: "indexed" and "stored". A field in Solr schema can be declared
as "stored" and/or "indexed":
- if the field is indexed (i.e. indexed="true") that means search features are enabled for that field (i.e. search, faceting, sorting)
- if the field is stored (i.e. stored="true") the verbatim copy of the incoming field value is stored and can be returned as part of search results.
In the SearchServices 1.4.x schema:
- all fields are marked as indexed: this is quite obvious because we want to enable search features on them
- 99% of fields are marked as **non stored**: this because SearchServices **retrieves the stored content from the Solr Content Store**
There are actually only three fields marked as stored: id, DBID and _version_. The last one is a Solr internal field used for some
features like atomic updates and optimistic locking (both of them are not used in SearchServices 1.4.x).
### When the Solr Content Store is used
As described above, SearchServices doesn't make use of Solr storage capabilities, so the rule is: the Solr
Content Store is involved on each interaction which requires the stored content. That includes:
- **Fields retrieval**: Solr stored only DBID, id and version fields; in search results we want to be able to retrieve
also other fields (e.g. name, title, LID, PATH)
- **Highlighting**: highlighted snippets are built using the fields stored value(s)
- **Clustering**: runtime clusters generation use the fields stored value(s)
- **Fingerprint**: the Fingerprint (custom) SearchComponent returns the (stored value of the) MINHASH field computed from the text content associated
with a given document
- **Text Content Management**: this is strictly connected with how the _ContentTracker_ works. See this [ADR](../trackers/00001-content-tracker.md) for a detailed explanation about the text content lifecycle in SearchServices.
### Read/Write Path on Solr Content Store
Every time a search request involves one of the points listed in the previous section we need to interact
- with the Solr index
- with the Solr Content Store
The Solr Content Store interaction can have two purposes:
- **Read only**: we need to read the stored fields associated with one or more documents
- **Read/Write**: we need to read and update the document definition (i.e. some field has been updated)
The two execution paths require additional I/O and CPU work on top of what Solr already normally does; Specifically:
The **Read Path** consists of the following steps (remember, this needs to be done for each match produced by a query):
- Locate the .gz file corresponding to a given DBID
- Uncompress the .gz file
- Deserialise the file in a _SolrInputDocument_ instance
- Use the fields values in the instance in order to perform the required task (e.g. fields retrieval, highlighting)
A first important thing about the flow above: it's not possible to load in memory only the fields we need.
Every time the document D is needed (even if our interaction requires just one field) the whole document definition is
- located (file seek)
- uncompressed
- deserialised
- read
Such capability is instead possible using Lucene: the IndexSearcher class can load a partial document definition which
contains only the fields actually needed. For example, if we want to highlight search terms in two fields, let's say
"name" and "title"
- the _AlfrescoHighlighter_ loads the whole document in memory
- the _SolrHighlighter_ loads only those two fields
This can make a relevant difference if we are in a context where the fields cardinality for each document is high, or if
we have one or more big fields (not needed) with a lot of text content.
The **Write Path** is even worse because it adds the following steps to the list above:
- Update the _SolrInputDocument_ instance with updated values
- Delete the old compressed file in the filesystem
- Serialise the updated _SolrInputDocument_ instance
- Compress the serialised image
- Write a new .gz file
### Solr Content Store Removal Benefits
#### Use as much as possible Solr built-in capabilities
The main reason why an open source platform is chosen as underlying framework is its popularity. That means a lot of
advantages in terms of
- community and non-commercial support
- product improvements with short iterations (e.g. enhancements, bug fixing)
Although the underlying reasons for introducing a customisation could be perfectly acceptable, it's important to keep in
mind that increasing such customisation level necessarily creates a gap, a distance from the open source product.
From one side, the customisation allows to implement some functional requirement not covered by the open source version,
on the other side the same customisation won't have the required support from the community.
The initial approach to this task consisted of a verification [Spike](https://issues.alfresco.com/jira/browse/SEARCH-1669) where
we investigated the pros and cons of having/removing the _SolrContentStore_.
Summarised, the output has been in favour of the removal, because the Solr storage capabilities are definitely more efficient
than the approach adopted in the _SolrContentStore_.
#### Less Solr customisations
This is a direct consequence of the preceding point. As you can read below, when we describe the major components affected
by the removal task, some customised components (e.g. Clustering) have been removed while others (e.g. Highlighter)
have been simplified a lot, leveraging the Solr built-in capabilities as much as possible.
### Only Solr data files
SearchServices no longer has to manage external files or folders. In SearchServices 1.4.x the content store required a
relevant effort for [customising]((https://issues.alfresco.com/jira/browse/SEARCH-1669)) the built-in Solr Replication
mechanism that doesn't take in account the Alfresco SolrContentStore.
![Solr Replication](replication.png)
Note that such customisation has been removed in this branch and it has been replaced by the built-in Solr Replication Handler:
the whole stored content management has been centralised in Solr; as consequence of that, the Read/Write paths described above
are no longer valid.
### Better compression
Compressing at single document level is not very efficient due to the small amount of data available. Moving this task
to the Solr level can deliver very good results for two main reasons:
- data cardinality is higher, so that means the compression algorithm can work with more representative and efficient stats
- data compression and index organisation is one area where the Solr community dedicated and dedicates a considerable amount of effort
### Less, more efficient I/O and CPU (compress/decompress) resources usage
This is again related with the Read/Write paths we described above: once the _SolrContentStore_ has been removed, we do not have to
deal with external files and folders and the read, write, compress, uncompress, serialise, deserialise tasks will be no longer needed.
### Better OS Page Cache usage
The OS Page Cache is used for storing and caching files required by the application processes running on a given machine.
In an ideal context the OS would put the entire Solr index in the page cache so every further read operation won't require any disk seek.
Unfortunately, the cache size is usually smaller than that, so a certain amount of time is spent by the OS in order to load/unload the
requested files.
In a context like that, the fewer files we have to manage, the better: having a component like the content store
which requires a relevant amount of I/O operations, it means a significant impact on the hardware resources (e.g. disk, cpu)
and a less efficient usage of the OS Page cache (e.g. the OS could unload the Solr datafiles for working with Solr content store files).
## Major Changes
This section provides a high-level description of the components / area that have been affected by the SolrContentStore removal.
### Solr Schema
Jira Ticket: [SEARCH-1707](https://issues.alfresco.com/jira/browse/SEARCH-1707)
The Solr schema (schema.xml) includes the following changes:
- **stored fields**: every field is marked as stored. Since this is something we'd want to apply to all fields, the stored
attribute has been defined at field type level.
- **cleanup and new field types**: there are several new field types that declare the default values applied to a field.
The naming is quite intuitive (e.g. "long" is a single value numeric field, "longs" is for multiValued numeric fields).
This change allowed clearer field definitions (i.e. field definitions that don't override default values are very short and concise)
![Field Types](schema_field_types.png)
- **comments and examples**: sometimes it is very hard to understand the purpose of a field and what is its runtime content.
For each field the current schema provides a description about its intent and one or more examples.
![Comments in schema](field_content_example.png)
### Highlighting
Jira Ticket: [SEARCH-1693](https://issues.alfresco.com/jira/browse/SEARCH-1693)
Before the content store removal, the _AlfrescoSolrHighlighter_ class was a custom "copy" of the _DefaultSolrHighlighter_.
Instead of extending the Solr component, at time of writing that class had been
- copied
- renamed _AlfrescoSolrHighlighter_
- customised
As consequence of that, the class was a mix of Alfresco and Solr code. Specifically, the custom code (and this is valid for all the customised
components mentioned in this document) was there mainly for two reasons:
- SolrContentStore interaction: every time the component needed to access to the stored content of a document
- Field renaming/mapping between Alfresco and Solr: for example a "cm_name" or "name" Alfresco field in the highlighting
request needs to be translated in the corresponding Solr field (e.g. text@s___t@....@name)
The new _AlfrescoSolrHighlighter_
- removes any interactions with the content store
- extends the _DefaultSolrHighlighter_
- consists of 95% Alfresco specific logic (mainly related with the field mapping/renaming). Each time it needs to execute
the highlighting logic, it delegates the Solr superclass.
- 5% of the code is still copied from the superclass. This is because sometimes it has't been possible to decorate
Solr methods from the superclass (see _getSpanQueryScorer_ or _getHighlighter_ methods)
The field mapping/renaming didn't allow to remove completely the custom component. However, the refactoring described above could be
a first step for externalising (in an intermediate REST layer) that logic. Once that was done, the custom highlighter could be removed and replaced with
the plain Solr built-in component.
### Clustering
Jira Ticket: [SEARCH-1688](https://issues.alfresco.com/jira/browse/SEARCH-1688)
The _AlfrescoClusteringComponent_ has been removed because it was a raw copy of the corresponding Solr component
and the only customisation was related with the content store interaction.
### Fingerprint
Jira Ticket: [SEARCH-1694](https://issues.alfresco.com/jira/browse/SEARCH-1694)
Two components have been affected by the content store removal:
- the [_Solr4QueryParser_](https://issues.alfresco.com/jira/browse/SEARCH-1694?focusedCommentId=622599&page=com.atlassian.jira.plugin.system.issuetabpanels%3Acomment-tabpanel#comment-622599): the component which is in charge of parsing incoming queries (FINGERPRINT queries in this case)
- _FingerprintComponent_: this is a custom _SearchComponent_ which accepts in input a node identifier and returns a response consisting of the corresponding fingerprint (i.e. the MINHASH multivalued field). Note that the MINHASH value(s) is not computed on the fly. Instead it is computed at index time when the text context is indexed.
### CachedDocTransformer
Jira Ticket: [SEARCH-1689](https://issues.alfresco.com/jira/browse/SEARCH-1689)
We said above that before the content store removal we had only three stored fields: DBID, id and version.
If that could be perfectly reasonable from a "search" execution perspective because we didn't need stored fields at all in the
matching and scoring phases, that becomes a problem when we have to return the search results to the caller:
- a search client would probably need some other field like a title, a name.
- we couldn't use Solr for retrieving those fields/values because we didn't store them
The only place where we had the stored content was the _SolrContentStore_, but Solr didn't know how to interact with it.
For this reason Alfresco introduced a custom _DocTransformer_. A _DocTransformer_ is an extension point provided by Solr for
introducing a custom transformation logic at document level. Once the search has been executed, for each matching document
the transformer is invoked and it can manipulate it.
This was one of the customisations strictly tied with the content store. Even after the content store removal, the doc
transformer is still there because the field mapping/renaming executed at document/field level is crucial for decoupling
the Alfresco Data Model from the Solr schema.
The _DocTransformer_ could be referred using the "[cached]" mnemonic code which no longer communicate the new purpose.
For that reason a new alias "[fmap]" has been introduced. The old "[cached]" code is still working but will be deprecated.
The same consideration we did for the highlighter is valid for this component as well: if the field mapping/renaming is
moved outside Solr, this component could be easily removed.
### DateFormatDocTransformer
Jira Ticket: [SEARCH-2044](https://issues.alfresco.com/jira/browse/SEARCH-2044)
This is a new _DocTransfomer_ introduced for maintaining the retro-compatibility with date/datetime fields management
in InsightEngine.
The InsightEngine SQL interface uses a hybrid language for expressing queries. Specifically, while the most part of the
query language is a plain standard SQL, everything related with date/datetime fields or expressions follows the Solr semantic.
For example, the Solr DateMath expressions can be used in SQL queries:
- select cm_created_month, count(*) as ct from alfresco where cm_owner = 'jimmy' and cm_created >= **'NOW/MONTH-6MONTHS'** group by cm_created_month
- select cm_created_year, count(*) as ct from alfresco where cm_owner = 'morton' and cm_created >= **'NOW/YEAR-4YEARS'** group by cm_created_year
Those expressions are not valid in SQL, so we must force the Calcite parser behaviour in order to consider them as "opaque" values.
In other words, everything related with date/datetime fields/expressions are considered (opaque) strings and aren't parsed by the
Calcite SQLParser: they are directly forwarded to Solr.
A _SolrInputDocument_ instance in the content store was composed by a set of fields whose values were exclusively a string
or list of strings. After the content store removal SearchServices retrieves the stored content from Solr, and if a field
is declared as having a Date or DateTime field type, Solr will return its value as a _java.util.Date_.
The _DateFormatDocTransformer_ is a simple transformer which replaces the Date value of such fields with the corresponding UTC
string representation.
### SolrInformationServer
Jira Ticket: [SEARCH-1702](https://issues.alfresco.com/jira/browse/SEARCH-1702)
The _SolrInformationServer_ is a kind of Mediator/Facade between SearchServices and the underlying search platform. This is
huge class which contains all methods for manipulating the index. Those methods are mainly called by the trackers
subsystem.
It had a strong connection/interaction with the content store because it represents the central point where the three
different representations of the same data are managed, manipulated, updated or deleted, and finally indexed. The three
different representations are:
- the incoming Node representing new or updated data which will create the "updated" version of document D
- the document D in the content store
- the document D in the Solr index
A first big change which affected the _SolrInformationServer_ has been the removal of all interactions with the content store.
#### Atomic Updates
An important change has been the introduction of partial/atomic updates.
Imagine an update path:
- an incoming Node arrives. It contains data that needs to be updated
- the _Node_ is not the exact copy of the Solr document. For example there are some fields that have been computed at indexing time (e.g MINHASH)
- the _SolrContentStore_ contains the exact copy of the last version of that Solr document which has been previously sent to index
- that document is loaded from the content store
- it merged/updated with the data in the incoming _Node_
- the entry on the content store is overwritten
- the updated document is then sent to Solr
Without the _SolrContentStore_ that path is no longer possible and it will be simplified a lot with the introduction of Atomic Updates.
Atomic Updates are a way to execute indexing commands on the client side using an “update” semantic, by applying/indexing
a document which represents a partial state of a domain object (the incoming _Node_, in our case).
One of the main reason why the _SolrInformationServer_ code has been widely changed is related with the Atomic Updates
introduction. More information about this change can be found [here](https://sease.io/2020/01/apache-solr-atomic-updates-polymorphic-approach.html)
Note that enabling the atomic updates requires also a major change in the configuration: the **UpdateLog**
must be enabled in order to make sure the updates are always applied to the latest version of the indexed document.
##### Dirty Text Content Detection
Another change introduced in _SolrInformationServer_ class is related with how SearchServices (specifically
the _ContentTracker_) detects the documents whose text content needs to be updated.
Previously, we had a field in the Solr schema called **FTSSTATUS** that could have the following domain:
- **Clean**: the text content of the document is in sync, no update is needed
- **New**: the document has been just created, it has to be updated with the corresponding text content
- **Dirty**: the text content of the document changed, the new content needs to be retrieved and the document updated
After the content store removal, the FTSSTATUS field has been removed. This because the field value was set depending on
the document state in the content store:
- if the incoming node didn't have a corresponding entry in the content store, then it was set to **New**
- if the incoming node had a corresponding entry in the content store a DOCID field value was compared between the node and the stored document. In case
the two values were different then the FTSSTATUS was set to **Dirty**
- Once the _ContentTracker_ updated the document with the new text content, the FTSSTATUS was set to **Clean**
We no longer have the content store, so the comparison above cannot be done. For example, when a _Node_ arrives we
cannot know if that corresponds to an existing document or if it is the first time we see it.
We could request that information to Solr but that would mean one query for each incoming _Node_, and that wouldn't be efficient.
The new approach uses two fields:
- **LATEST_APPLIED_CONTENT_VERSION_ID**: it corresponds to the identifier of the latest applied content property id
(content@s__docid@* or content@m__docid@*). It can be null (i.e. the incoming node doesn't have a value for that property,
even if it requires content indexing)
- **LAST_INCOMING_CONTENT_VERSION_ID**: If the field has the same value of the previous one (or it is equal to _SolrInformationServer.CONTENT_UPDATED_MARKER_),
then the content is supposed to be in sync. Otherwise, if the value is different, it is not _SolrInformationServer.CONTENT_UPDATED_MARKER_
or it is _SolrInformationServer.CONTENT_OUTDATED_MARKER_ the content is intended as outdated and therefore it will
be selected (later) by the _ContentTracker_.
### AlfrescoReplicationHandler
This set of components, [introduced in SearchServices 1.4.x](https://issues.alfresco.com/jira/browse/SEARCH-1850) for including the content store in the Solr replication mechanism, has been removed
because we no longer have any external folder/file to be synced between master and slave(s). As consequence of that
the built-in Solr ReplicationHandler is used.
### Content Store Package and Tests
Jira Tickets: [SEARCH-1692](https://issues.alfresco.com/jira/browse/SEARCH-1692),[SEARCH-2025](https://issues.alfresco.com/jira/browse/SEARCH-2025)
Once the content store references have been removed from the components listed in the sections above, the _org.alfresco.solr.content_
package has been completely removed.
## Code Metrics
This section provides some metrics about the lines of code of the components affected by the content store removal task,
before and after the change.
The table compares the classes size (the lines of code) in this branch with the same class in the master branch.
| Class | Master Branch | Content Store Removal Branch | Gain |
| --------------------------|:----------:|------------------------------------|------------:|
|org.apache.solr.handler.component.AlfrescoSolrHighlighter |728 |504|-224|
|org.alfresco.solr.SolrInformationServer |3979 |3898|-88|
|org.apache.solr.handler.component.AlfrescoClusteringComponent |400 |0|-400|
|org.alfresco.solr.transformer.AlfrescoFieldMapperTransformer* |197 |130|-67|
|org.alfresco.solr.handler.AlfrescoIndexFetcher |2762 |0|-2762|
|org.alfresco.solr.handler.AlfrescoReplicationHandler |2377 |0|-2377|
|org.alfresco.solr.handler.OldBackupDirectory |377 |0|-377|
|org.alfresco.solr.handler.Snapshooter |139 |0|-139|
|org.alfresco.solr.component.FingerPrintComponent |130 |131|+1|
|org.alfresco.solr.content.AccessMode |106 |0|-106|
|org.alfresco.solr.content.InitialisableAccessMode |33 |0|-33|
|org.alfresco.solr.content.SolrContentStore |707 |0|-707|
|org.alfresco.solr.content.SolrContentUrlBuilder |213 |0|-213|
|org.alfresco.solr.content.SolrFileContentReader |265 |0|-265|
|org.alfresco.solr.content.SolrFileContentWriter |277 |0|-277|
|org.alfresco.solr.content.SolrContentStoreChangeSetTest |291 |0|-291|
|org.alfresco.solr.content.SolrContentStoreTest |424 |0|-424|
|org.alfresco.solr.content.SolrContentUrlBuilderTest |217 |0|-217|
|org.alfresco.solr.content.SolrContentWriterTest |86 |0|-86|
|org.alfresco.solr.handler.ContentStoreReplicationIT |286 |0|-286|
| | | | |
| TOTAL |13944|4633|-9311|
search-services/alfresco-search/doc/architecture/solr-content-store-removal/field_content_example.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 138 KiB

search-services/alfresco-search/doc/architecture/solr-content-store-removal/replication.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 797 KiB

search-services/alfresco-search/doc/architecture/solr-content-store-removal/schema_field_types.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 298 KiB

search-services/alfresco-search/doc/architecture/solr-content-store-removal/solr.data.dir.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 95 KiB

search-services/alfresco-search/doc/architecture/solr-content-store-removal/solrcontentstore.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 383 KiB

search-services/alfresco-search/doc/architecture/solr-content-store-removal/stored_content_in_searchservices.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 162 KiB

search-services/alfresco-search/doc/architecture/solr-schema/00001-solr-schema.md
+363
View File
@@ -0,0 +1,363 @@
## Solr Schema (Re)Design
### Status
![Completeness Badge](https://img.shields.io/badge/Document_Level-Completed-green.svg?style=flat-square)
### Context
This document details the Solr schema improvements implemented on top of the changes described in the [SolrContentStore removal ADR](../solr-content-store-removal/00001-solr-content-store-removal.md).
The starting context is the Solr Schema as described in [SEARCH-1707](https://issues.alfresco.com/jira/browse/SEARCH-1707).
Other than additional comments, examples and readability improvements, the big change introduced consisted of all
fields having the "stored" flag set to true. That was needed for replacing the _SolrContentStore_ storage in favour of
Solr built-in capabilities.
The SearchServices Solr schema is composed by group of fields executing a different text analysis but sharing the
same identical content. For example, the following group:
<!-- Note: prior to this task all these fields were stored -->
<dynamicField name="text@s__l_@*" type="alfrescoFieldType" omitNorms="true" />
<dynamicField name="text@s__lt@*" type="alfrescoFieldType" omitNorms="false" />
<dynamicField name="text@s___t@*" type="text___" />
<dynamicField name="text@s__sort@*" type="alfrescoCollatableTextFieldType" />
<dynamicField name="text@sd___@*" type="identifier" stored="true" docValues="true" />
is related to single valued text attributes. For instance, a "name" attribute at indexing time uses that group
in order to generate the following fields:
text@s__l_@{http://www.alfresco.org/model/content/1.0}name
text@s__lt@{http://www.alfresco.org/model/content/1.0}name
text@s___t@{http://www.alfresco.org/model/content/1.0}name
text@s__sort@{http://www.alfresco.org/model/content/1.0}name
text@sd___@{http://www.alfresco.org/model/content/1.0}name
The logical step on top of this refactoring has been to reduce the number of stored fields
in order to
- **avoid duplicated content** as much as possible
- **improve** the indexing process
- **gain** disk space
### Solr Schema Fields Logical Grouping
Four main groups have been identified:
1. fields starting with "text@s" (s stands for single valued)
<dynamicField name="text@s____@*" type="identifier" />
<dynamicField name="text@s__l_@*" type="alfrescoFieldType" omitNorms="true" />
<dynamicField name="text@s__lt@*" type="alfrescoFieldType" omitNorms="false" />
<dynamicField name="text@s___t@*" type="text___" />
<dynamicField name="text@s__sort@*" type="alfrescoCollatableTextFieldType" />
<dynamicField name="text@sd___@*" type="identifier" stored="true" docValues="true" />
2. fields starting with "text@m" (multivalued)
<dynamicField name="text@m____@*" type="identifiers" />
<dynamicField name="text@m__l_@*" type="alfrescoFieldType" multiValued="true" />
<dynamicField name="text@m__lt@*" type="alfrescoFieldType" multiValued="true" />
<dynamicField name="text@m___t@*" type="text___" multiValued="true" />
<dynamicField name="text@md___@*" type="identifiers" stored="true" docValues="true" />
3. fields starting with "mltext@m"
<dynamicField name="mltext@m____@*" type="identifiers" />
<dynamicField name="mltext@m__l_@*" type="alfrescoFieldType" multiValued="true" />
<dynamicField name="mltext@m__lt@*" type="alfrescoFieldType" multiValued="true" />
<dynamicField name="mltext@m___t@*" type="text___" multiValued="true" />
<dynamicField name="mltext@m__sort@*" type="alfrescoCollatableMLTextFieldType" />
4. fields starting with "content@s"
<dynamicField name="content@s____@*" type="identifier" termPositions="false" />
<dynamicField name="content@s__l_@*" type="alfrescoFieldType" termPositions="false" />
<dynamicField name="content@s__lt@*" type="alfrescoFieldType" />
<dynamicField name="content@s___t@*" type="text___" />
Note SearchServices schema includes also other fields (e.g. static fields, primitive fields): they haven't been involved
in this refactoring.
The starting points of the investigation have been:
- fields listed above are *all* marked as stored: it means their content is verbatim retained/copied in the index, therefore generating a lot of redundancy
- fields belonging to the same group have the same identical content (again, a lot of redundancy)
After switching to Solr storage capabilities, we were been able to remove a huge set of customisations, but at the same
time we realised we were storing the same content several times.
In other words, the amount of data required for "representing" a whole document inevitably required a high redundancy degree,
because the same text content was associated to all fields belonging to a give group.
We decided to adopt an incremental approach for gradually reducing as much as possible the number of stored fields in the schema.
#### Iteration #1: one single stored field per group
In a first iteration we've created one stored field for each group, and then we've used the copyField directive
to have Solr copying the field value on all other indexed (but this time not stored) fields:
<!-- Group 1: TEXT "S" -->
<dynamicField name="text@s_stored@*" type="identifier" stored="true" indexed="false/>
<dynamicField name="text@s____@*" type="identifier" stored="false"/>
<dynamicField name="text@s__l_@*" type="alfrescoFieldType" stored="false" />
<dynamicField name="text@s__lt@*" type="alfrescoFieldType" stored="false" />
<dynamicField name="text@s___t@*" type="text___" stored="false" />
<dynamicField name="text@s__sort@*" type="alfrescoCollatableTextFieldType" />
<dynamicField name="text@sd___@*" type="identifier" stored="true" docValues="true" />
<copyField source="text@s_stored@*" dest="text@s____@*"/>
<copyField source="text@s_stored@*" dest="text@s__l_@*"/>
<copyField source="text@s_stored@*" dest="text@s__lt@*"/>
<copyField source="text@s_stored@*" dest="text@s___t@*"/>
<copyField source="text@s_stored@*" dest="text@s__sort@*"/>
<!-- Groups 2 and 3 follows the same logic... -->
...
<!-- Group 4: MLTEXT -->
<dynamicField name="mltext@m_stored@*" type="identifiers" stored="true" indexed="false/>
<dynamicField name="mltext@m____@*" type="identifier" multiValued="true" stored="false"/>
<dynamicField name="mltext@m__l_@*" type="alfrescoFieldType" omitNorms="true" multiValued="true" stored="false"/>
<dynamicField name="mltext@m__lt@*" type="alfrescoFieldType" omitNorms="true" multiValued="true" stored="false"/>
<dynamicField name="mltext@m___t@*" type="text___" multiValued="true" stored="false" />
<dynamicField name="mltext@m__sort@*" type="alfrescoCollatableMLTextFieldType" />
<copyField source="mltext@m_stored@*" dest="mltext@s____@*"/>
<copyField source="mltext@m_stored@*" dest="mltext@m__l_@*"/>
<copyField source="mltext@m_stored@*" dest="mltext@m__lt@*"/>
<copyField source="mltext@m_stored@*" dest="mltext@m___t@*"/>
The indexing process is simpler than before, because the _SolrInputDocument_ instance needs to provide a value only for
those stored fields, and then before indexing Solr takes care about copying their value on the other fields of the same group.
The drawback of this approach is: the stored field has one value which must contain all the information needed for creating the other fields.
Although that could sound trivial, the different cardinality between the source and the destinations could lead some issue related
with the different usage of the target fields.
For example, the value of the stored field is:
_"Hello I'm a title"_
some target fields (e.g. text@s____@*, text@s__t@*) just need to be filled with that literal value, while some others
(e.g. text@s__lt@*, text@s__l_@*) require an additional information, specifically a locale marker which consists of
- a starting delimiter char \u0000
- the locale language code
- a closing delimiter char \u0000
This marker has to be set at the beginning of the field value, so in the example above the value those fields expect is:
_"\u0000en\u0000Hello I'm a title"_
Since it's not possible to interfere with the copy field directive in order to inject such prefix at index time, a different
approach has been adopted: the stored field **always** contains the prefix marker; fields that don't need that prefix
must provide an analyzer which strips it out. This is the main reason why you will find the following CharFilter inside
the text analysis of a lot of existing field types:
<charFilter class="solr.PatternReplaceCharFilterFactory" pattern="(#0;.*#0;)" replacement=""/>
As a side note, there are some fields that cannot use the copy field approach. Specifically
- fields used for **sorting** (mltext@m___sort): they have a special value set on _SolrInformationServer_ side which cannot be
rebuilt by Solr by simply copying the value of the stored field
- fields with **docValues** enabled: mainly used for faceting, they cannot have the locale prefix marker but having docValues
enabled, they cannot have a TextField as a type. That means we cannot define an analyzer for removing that unwanted prefix
before the field gets indexed.
Those field have been marked as stored or docValues and they are sent in the documents as usual, without relying on the copyField
directive.
#### Iteration #2: disjoint set of stored fields
The approach of the first iteration worked, but it had the following drawback:
- we always had one stored field per group, even if that group was not used (that could be possible, depending on the Alfresco Data Model)
- each stored field was always copied on all the indexed fields belonging to its group, regardless their usage (for example, we
don't need the cross-locale version for a specific field)
So, we gained disk space in terms of stored content, but a different type of redundancy was introduced, for creating the
inverted index for all copied fields.
In order to overcome this new type of redundancy a set of "disjoint" stored fields has been introduced.
"Disjoint" means, within a group,
- we've introduced several stored fields, one per possible usage scenario
- each stored fields is copied only on those indexed fields that map the usage scenario
These fields are automatically generated by a [python procedure](../../../generator-solr-config/generate-solr-config.py)
and included in the solrconfig.xml using the xi:include directive.
Here's an extract of the [included fragment](../../../src/main/resources/solr/instance/templates/rerank/conf/generated_copy_fields.xml). :
<!-- Ex. 1: the field value is used only for suggestions -->
<dynamicField name="text@s_stored_____s@*" type="localePrefixedField" />
<copyField source="text@s_stored_____s@*" dest="suggest" />
...
<!-- Ex. 2: the field value is used for suggestions and cross-locale searches -->
<dynamicField name="text@s_stored_t___s@*" type="localePrefixedField" />
<copyField source="text@s_stored_t___s@*" dest="text@s__lt@*" />
<copyField source="text@s_stored_t___s@*" dest="suggest" />
...
<dynamicField name="text@s_stored_t_c_s@*" type="localePrefixedField" />
<copyField source="text@s_stored_t_c_s@*" dest="text@s__lt@*" />
<copyField source="text@s_stored_t_c_s@*" dest="text@s___t@*" />
<copyField source="text@s_stored_t_c_s@*" dest="suggest" />
...
<dynamicField name="text@s_stored_tscss@*" type="localePrefixedField" />
<copyField source="text@s_stored_tscss@*" dest="text@s__lt@*" />
<copyField source="text@s_stored_tscss@*" dest="text@s___t@*" />
<copyField source="text@s_stored_tscss@*" dest="text@s__l_@*" />
<copyField source="text@s_stored_tscss@*" dest="text@s__sort@*" />
<copyField source="text@s_stored_tscss@*" dest="text@s____@*" />
<copyField source="text@s_stored_tscss@*" dest="suggest" />
As you can see we are several stored fields that, depending on the target usage, are copied across different searchable
fields. The naming adopted for the new stored fields follows the existing approach already in use for "text" and "mltext" fields.
Specifically, the stored field name is structured as follows:
field_type_name@(s|m)_stored(t|_)(s|_)(c|_)(s|_)(s|_)@local_name
where
- field type name can be "text" or "mltext"
- (s|m): single value or multivalued field
- "_stored": static part which denotes a stored field
- (t|_) tokenised usage (t) or not (\_)
- (s|_) string usage (s) or not (\_)
- (c|_) cross locale usage (c) or not (\_)
- (s|_) sort usage (s) or not (\_)
- (s|_) suggestable usage (s) or not (\_)
## Major changes
The previous sections gave a high-level "functional" overview of the most relevant changes in the Solr schema.
However, there are also some other implementation details that need to be described for a better understanding of all
components that participate in the new Solr data model.
### Stored Fields Fragment Generator
The [entire set](../../../src/main/resources/solr/instance/templates/rerank/conf/generated_copy_fields.xml)
of disjoint stored fields is quite huge. Instead of manually creating all those fields, a [python procedure](../../../generator-solr-config/generate-solr-config.py)
generates the XML fragment that is included at the end of the [schema.xml](../../../src/main/resources/solr/instance/templates/rerank/conf/schema.xml)
<xi:include href="generated_copy_fields.xml" xmlns:xi="http://www.w3.org/2001/XInclude">
<xi:fallback/>
</xi:include>
The procedure is a very simple script which can be executed without any parameter at all
python generate-solr-config.py
It generates a
[generated_copy_fields.xml file](../../../src/main/resources/solr/instance/templates/rerank/conf/generated_copy_fields.xml)
within the execution folder. You can generate that file once and them move it into the rerank template configuration folder.
The execution is manual, it hasn't been bound at build time because there's no need to re-execute it for every build.
### Custom StrField
We've already mentioned that stored fields contain a language marker at the beginning of their value. That is needed because
a stored field will be used at indexing time (by means of copyField directive) for feeding the other searchable (and not stored)
fields. A target field could need that prefix (that is the case of all fields having AlfrescoFieldType as type) or not.
In this latter case we could be in one of the following scenarios:
- the field type is a TextField (or a subclass): it must provide in its analyzer a char/token filter which removes the prefix
- the field type is a StrField: it's not possible to define a text analysis for these fields, so in order to overcome
that limit the _org.alfresco.solr.StripLocaleStrField_ custom type has been introduced. It is a subclass of Solr _StrField_ which removes the locale prefix
from the value that will be stored.
<fieldType
name="stripLocaleStrField"
class="org.alfresco.solr.StripLocaleStrField"
indexed="true"
stored="false"/>
...
<dynamicField name="text@s____@*" type="stripLocaleStrField"/>
This is the type used for untokenised and indexed fields.
### Custom TextField for stored fields
Stored fields are used, as the name suggests
- for retaining a verbatim copy of the original text content
- at field retrieval (i.e. query response) time
- for some complementary search features like Highlighting and More Like This (not yet implemented)
Usually, the field type of a stored field is a _StrField_ (or one subclass), that is: a field type which doesn't provide any
text analysis. It is untokenised and its value is managed as a whole single token.
The Solr Highlighting search feature uses the stored value for computing the highlight snippets: it needs to analyze the
stored content using the index analyzer defined for the field type. If the field type doesn't provide any tokenisation,
the highlighting won't return any snippet.
This is the main reason why in SearchServices stored fields cannot be "StrField": a special TextField field type is needed,
it must be able to provide the proper text analysis depending on the language prefix put at the beginning of the field value.
Let's see an example. Suppose we have a simple document, with just one field:
{
"description": "We were discussing about an interesting topic"
}
The user requests to highlight the term "discuss". He would expect "discussing" to be highlighted:
_"we were **discussing** about an interesting topic"_
In other words, the highlighting process is expected to take care about the field language during the tokenisation.
Solr built-in TextField type is not able to switch the text analysis at runtime, so the
_org.alfresco.solr.schema.highlight.LanguagePrefixedTextField_
has been introduced in order to fulfill that requirement. The field type analyses the locale marker and determines the
analyzer that will be used looking at:
- a field type called "highlighted_text_" + locale (e.g. highlighted_text_en)
- if it's not found, then it checks if a field type called "text_" + locale is defined (e.g. text_en)
- if it cannot be found, it uses the cross locale field type "text___"
The runtime computed analyzer makes sure the text analysis (for the highlighting) will always match the locale information
within the field value.
### Highlighter
The Alfresco Highlighter is a customisation of the Solr Default Highlighter. The most relevant changes that affected this
component has been described in the _SolrContentStore_ removal [ADR](../solr-content-store-removal/00001-solr-content-store-removal.md).
In this context it is important to underline the stored fields management described above widely reduced its complexity:
prior to that, for each field interested in the highlight process, the Highlighter executed in the worst case two requests:
- a first request using the cross-locale field
- a second request using the locale-specific field
Since the new data model provides a single stored version of each field, the Highlighter executes only a single request
using that field.
## Closing Remarks
A precise measure of the performance improvements introduced is out of the scope of this task because it would require a proper
benchmark infrastructure. This section is more a summary which lists the major relevant impacts that should be taken in
account when a comparison/benchmark will be done between the "pre" and "post" SolrContentStore SearchServices versions.
### Query time
- **field retrieval**: a user requested field is quickly associated to the (single) corresponding stored field
- **highlighting**: instead of doubling the highlighting process (first cross-locale and then locale specific fields) there's just one execution
which targets the unique stored field (per attribute)
### Indexing time
In general we expect
- a higher indexing throughput
- less disk space required for storing the index datafiles:
These are the points that contribute to the list above:
- **atomic updates** reduce the amount of data sent to Solr on partial updates
- **the amount of data sent** at indexing time is smaller: only the stored fields are sent
- **the merge logic** (in case of document update) has been centralised in Solr
- **the amount of stored data in the index is smaller**: there's just one stored field per Alfresco attribute. This can be very important in
those scenarios where the same field has more than one "usage" (e.g. cross-language, language specific and suggestions).
Prior to that we had one copy of the same stored content for each usage, while now there's always one single stored field.
search-services/alfresco-search/generator-solr-config/generate-solr-config.py
+129
View File
@@ -0,0 +1,129 @@
# Copyright (C) 2020 Alfresco Software Limited.
# This file is part of Alfresco
#
# Alfresco is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Alfresco is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
import itertools
tokenized = "tokenized"
string = "string"
sortable = "sortable"
suggestable = "suggestable"
cross_locale = "cross-locale"
output_file = "generated_copy_fields.xml"
def find_subsets(s, n):
return [set(i) for x in range(1, n+1) for i in itertools.combinations(s, x) ]
def get_copy_field_xml(source, destination):
return '<copyField source="' + source + '" dest="' + destination+ '" />'
def get_dynamic_field_xml(field, field_type):
postfix = ""
if field_type in ("text", "content"):
postfix = '" type="localePrefixedField" />'
else:
postfix = '" type="localePrefixedField" multiValued="true" />'
return '<dynamicField name="'+ field + postfix
def get_field_prefix(field_type):
if field_type == "text":
return "text@s_"
elif field_type == "mltext":
return "mltext@m_"
elif field_type == "content":
return "content@s_"
else:
return "text@m_"
def generate_fields(field_type, tokenized, string, cross_locale, sortable, suggestable):
prefix = get_field_prefix(field_type)
field = prefix + "stored_" + ("t" if tokenized else "_") + ("s" if string else "_") + ("c" if cross_locale else "_") + ("s" if sortable else "_" ) + ("s" if suggestable else "_") + "@*"
generated_fields = []
generated_fields.append(get_dynamic_field_xml(field, field_type))
if tokenized:
generated_fields.append(get_copy_field_xml(field, create_tokenized(prefix)))
if cross_locale:
generated_fields.append(get_copy_field_xml(field, create_tokenized_cross_locale(prefix)))
if string:
generated_fields.append(get_copy_field_xml(field, create_non_tokenized(prefix)))
if sortable:
generated_fields.append(get_copy_field_xml(field, create_sortable(prefix)))
if cross_locale:
generated_fields.append(get_copy_field_xml(field, create_non_tokenized_cross_locale(prefix)))
if suggestable:
generated_fields.append(get_copy_field_xml(field, "suggest"))
return generated_fields
def create_tokenized(prefix):
return prefix + "_lt" + "@*"
def create_tokenized_cross_locale(prefix):
return prefix + "__t" + "@*"
def create_non_tokenized(prefix):
return prefix + "_l_" + "@*"
def create_non_tokenized_cross_locale(prefix):
return prefix + "___" + "@*"
def create_sortable(prefix):
return prefix + "_sort" + "@*"
def generate_text(file):
for t in ("text", "mltext", "content", "multivalue-text"):
s = {tokenized, string, cross_locale, sortable, suggestable} if t == "text" else {tokenized, string, cross_locale, suggestable}
for s in find_subsets(s, 5):
generated = generate_fields(t, tokenized in s, string in s, cross_locale in s, sortable in s, suggestable in s)
file.writelines(["%s\n" % item for item in generated])
file.write("\n")
file.write("\n")
def main():
file = open(output_file, "w")
file.write('<fields>\n')
generate_text(file)
file.write('</fields>')
file.close()
main()
search-services/alfresco-search/pom.xml
+12 -2
View File
@@ -96,6 +96,11 @@
<version>2.3.2</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
<version>3.9</version>
</dependency>
<dependency>
<groupId>org.apache.cxf</groupId>
<artifactId>cxf-core</artifactId>
@@ -146,6 +151,11 @@
<artifactId>cxf-rt-wsdl</artifactId>
<version>${cxf.version}</version>
</dependency>
<dependency>
<groupId>xpp3</groupId>
<artifactId>xpp3</artifactId>
<version>1.1.4c</version>
</dependency>
<!-- Test dependencies -->
<dependency>
@@ -158,7 +168,7 @@
<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
<version>3.2.4</version>
<version>3.3.0</version>
<scope>test</scope>
</dependency>
@@ -171,7 +181,7 @@
<dependency>
<groupId>com.carrotsearch.randomizedtesting</groupId>
<artifactId>randomizedtesting-runner</artifactId>
<version>2.7.6</version>
<version>2.7.7</version>
<scope>test</scope>
</dependency>
<dependency>
search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoCollatableTextFieldType.java
+1 -4
View File
@@ -80,7 +80,6 @@ public class AlfrescoCollatableTextFieldType extends StrField
public static class TextSortFieldComparatorSource extends FieldComparatorSource
{
/*
* (non-Javadoc)
* @see org.apache.lucene.search.FieldComparatorSource#newComparator(java.lang.String, int, int, boolean)
@@ -90,11 +89,9 @@ public class AlfrescoCollatableTextFieldType extends StrField
{
return new TextSortFieldComparator(numHits, fieldname, I18NUtil.getLocale());
}
}
/*
* Adapted from org.apache.lucene.search.FieldComparator.TermValComparator<T>
*/
@@ -174,7 +171,7 @@ public class AlfrescoCollatableTextFieldType extends StrField
else if (withLocale.startsWith("\u0000"))
{
String[] parts = withLocale.split("\u0000");
return parts[1];
return parts[2];
}
else
{
search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoCoreAdminHandler.java
+719 -187
View File
File diff suppressed because it is too large Load Diff
search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoSolrDataModel.java
+191 -28
View File
@@ -63,7 +63,6 @@ import org.alfresco.repo.search.impl.parsers.AlfrescoFunctionEvaluationContext;
import org.alfresco.repo.search.impl.parsers.FTSParser;
import org.alfresco.repo.search.impl.parsers.FTSQueryParser;
import org.alfresco.repo.search.impl.querymodel.Constraint;
import org.alfresco.repo.search.impl.querymodel.Ordering;
import org.alfresco.repo.search.impl.querymodel.QueryModelFactory;
import org.alfresco.repo.search.impl.querymodel.QueryOptions.Connective;
import org.alfresco.repo.search.impl.querymodel.impl.lucene.LuceneQueryBuilder;
@@ -106,6 +105,8 @@ import org.slf4j.LoggerFactory;
import org.springframework.beans.BeansException;
import org.springframework.context.support.FileSystemXmlApplicationContext;
import static java.util.Optional.ofNullable;
/**
* @author Andy
*
@@ -117,6 +118,13 @@ public class AlfrescoSolrDataModel implements QueryConstants
public String tenant;
public Long aclId;
public Long dbId;
public Map<String, Object> optionalBag = new HashMap<>();
public void setProperty(String name, Object value)
{
optionalBag.put(name, value);
}
}
public enum FieldUse
@@ -145,7 +153,6 @@ public class AlfrescoSolrDataModel implements QueryConstants
}
public static final String CONTENT_S_LOCALE_PREFIX = "content@s__locale@";
public static final String CONTENT_M_LOCALE_PREFIX = "content@m__locale@";
static final String SHARED_PROPERTIES = "shared.properties";
protected final static Logger log = LoggerFactory.getLogger(AlfrescoSolrDataModel.class);
@@ -523,7 +530,7 @@ public class AlfrescoSolrDataModel implements QueryConstants
}
catch (IOException e)
{
log.info("Failed to read shared properties fat " + propertiesFile.getAbsolutePath());
log.info("Failed to read shared properties at " + propertiesFile.getAbsolutePath());
}
return props;
@@ -769,23 +776,26 @@ public class AlfrescoSolrDataModel implements QueryConstants
private void addHighlightSearchFields( PropertyDefinition propertyDefinition , IndexedField indexedField)
{
if ((propertyDefinition.getIndexTokenisationMode() == IndexTokenisationMode.TRUE)
|| (propertyDefinition.getIndexTokenisationMode() == IndexTokenisationMode.BOTH))
QName propertyName = propertyDefinition.getName();
QName propertyDataTypeQName = propertyDefinition.getDataType().getName();
String fieldName;
if(propertyDataTypeQName.equals(DataTypeDefinition.MLTEXT))
{
if(crossLocaleSearchDataTypes.contains(propertyDefinition.getDataType().getName()) || crossLocaleSearchProperties.contains(propertyDefinition.getName()))
{
indexedField.addField(getFieldForText(false, true, false, propertyDefinition), false, false);
indexedField.addField(getFieldForText(true, true, false, propertyDefinition), false, false);
}
else
{
indexedField.addField(getFieldForText(true, true, false, propertyDefinition), false, false);
}
fieldName = getStoredMLTextField(propertyName);
}
else if(propertyDataTypeQName.equals(DataTypeDefinition.CONTENT))
{
fieldName = getStoredContentField(propertyName);
}
else
{
indexedField.addField(getFieldForText(false, false, false, propertyDefinition), false, false);
fieldName = getStoredTextField(propertyName);
}
FieldInstance field = new FieldInstance(fieldName, false, false);
indexedField.getFields().add(field);
}
/*
@@ -929,6 +939,94 @@ public class AlfrescoSolrDataModel implements QueryConstants
}
}
public String getStoredTextField(QName propertyQName)
{
PropertyDefinition propertyDefinition = getPropertyDefinition(propertyQName);
StringBuilder sb = new StringBuilder();
sb.append("text@" + (propertyDefinition.isMultiValued()? "m" : "s") + "_stored_");
sb.append((propertyDefinition.getIndexTokenisationMode() == IndexTokenisationMode.TRUE ||
propertyDefinition.getIndexTokenisationMode() == IndexTokenisationMode.BOTH)? "t" : "_");
sb.append((propertyDefinition.getIndexTokenisationMode() == IndexTokenisationMode.FALSE ||
propertyDefinition.getIndexTokenisationMode() == IndexTokenisationMode.BOTH ||
isIdentifierTextProperty(propertyDefinition.getName()))? "s" : "_");
sb.append((crossLocaleSearchDataTypes.contains(propertyDefinition.getDataType().getName()) ||
crossLocaleSearchProperties.contains(propertyDefinition.getName())) ? "c" : "_");
sb.append((propertyDefinition.getIndexTokenisationMode() == IndexTokenisationMode.FALSE ||
propertyDefinition.getIndexTokenisationMode() == IndexTokenisationMode.BOTH ||
isIdentifierTextProperty(propertyDefinition.getName())) && !propertyDefinition.isMultiValued()? "s" : "_");
sb.append(isSuggestable(propertyQName)? "s": "_");
sb.append("@");
sb.append(propertyDefinition.getName().toString());
return sb.toString();
}
public String getStoredMLTextField(QName propertyQName)
{
PropertyDefinition propertyDefinition = getPropertyDefinition(propertyQName);
StringBuilder sb = new StringBuilder();
sb.append("mltext@m_stored_");
sb.append((propertyDefinition.getIndexTokenisationMode() == IndexTokenisationMode.TRUE ||
propertyDefinition.getIndexTokenisationMode() == IndexTokenisationMode.BOTH)? "t" : "_");
sb.append((propertyDefinition.getIndexTokenisationMode() == IndexTokenisationMode.FALSE ||
propertyDefinition.getIndexTokenisationMode() == IndexTokenisationMode.BOTH ||
isIdentifierTextProperty(propertyDefinition.getName()))? "s" : "_");
sb.append((crossLocaleSearchDataTypes.contains(propertyDefinition.getDataType().getName()) ||
crossLocaleSearchProperties.contains(propertyDefinition.getName())) ? "c" : "_");
sb.append("_");
sb.append(isSuggestable(propertyQName)? "s": "_");
sb.append("@");
sb.append(propertyDefinition.getName().toString());
return sb.toString();
}
public String getStoredContentField(QName propertyQName)
{
PropertyDefinition propertyDefinition = getPropertyDefinition(propertyQName);
StringBuilder sb = new StringBuilder();
sb.append("content@s_stored_");
sb.append((propertyDefinition.getIndexTokenisationMode() == IndexTokenisationMode.TRUE ||
propertyDefinition.getIndexTokenisationMode() == IndexTokenisationMode.BOTH)? "t" : "_");
sb.append((propertyDefinition.getIndexTokenisationMode() == IndexTokenisationMode.FALSE ||
propertyDefinition.getIndexTokenisationMode() == IndexTokenisationMode.BOTH ||
isIdentifierTextProperty(propertyDefinition.getName()))? "s" : "_");
sb.append((crossLocaleSearchDataTypes.contains(propertyDefinition.getDataType().getName()) ||
crossLocaleSearchProperties.contains(propertyDefinition.getName())) ? "c" : "_");
sb.append("_");
sb.append(isSuggestable(propertyQName)? "s": "_");
sb.append("@");
sb.append(propertyDefinition.getName().toString());
return sb.toString();
}
/**
* Get all the field names into which we must copy the source data
*
@@ -969,7 +1067,6 @@ public class AlfrescoSolrDataModel implements QueryConstants
{
indexedField.addField(getFieldForText(true, false, false, propertyDefinition), true, false);
indexedField.addField(getFieldForText(false, false, false, propertyDefinition), false, false);
}
if(dataTypeDefinition.getName().equals(DataTypeDefinition.TEXT))
@@ -1021,18 +1118,18 @@ public class AlfrescoSolrDataModel implements QueryConstants
return identifierProperties.contains(propertyQName);
}
private boolean isTextField(PropertyDefinition propertyDefinition)
public boolean isTextField(PropertyDefinition propertyDefinition)
{
QName propertyDataTypeQName = propertyDefinition.getDataType().getName();
if(propertyDataTypeQName.equals(DataTypeDefinition.MLTEXT))
{
return true;
}
else if(propertyDataTypeQName.equals(DataTypeDefinition.CONTENT))
{
return true;
}
else return propertyDataTypeQName.equals(DataTypeDefinition.TEXT);
return ofNullable(propertyDefinition)
.map(PropertyDefinition::getDataType)
.map(DataTypeDefinition::getName)
.map(name ->
name.equals(DataTypeDefinition.MLTEXT)
||
name.equals(DataTypeDefinition.CONTENT)
||
name.equals(DataTypeDefinition.TEXT))
.orElse(false);
}
private boolean isSuggestable(QName propertyQName)
@@ -1447,7 +1544,7 @@ public class AlfrescoSolrDataModel implements QueryConstants
Constraint constraint = FTSQueryParser.buildFTS(searchParameters.getQuery(), factory, functionContext, null, null, mode,
searchParameters.getDefaultFTSOperator() == org.alfresco.service.cmr.search.SearchParameters.Operator.OR ? Connective.OR : Connective.AND,
searchParameters.getQueryTemplates(), searchParameters.getDefaultFieldName(), rerankPhase);
org.alfresco.repo.search.impl.querymodel.Query queryModelQuery = factory.createQuery(null, null, constraint, new ArrayList<Ordering>());
org.alfresco.repo.search.impl.querymodel.Query queryModelQuery = factory.createQuery(null, null, constraint, new ArrayList<>());
@SuppressWarnings("unchecked")
LuceneQueryBuilder<Query, Sort, ParseException> builder = (LuceneQueryBuilder<Query, Sort, ParseException>) queryModelQuery;
@@ -1490,6 +1587,66 @@ public class AlfrescoSolrDataModel implements QueryConstants
return mapProperty(potentialProperty, fieldUse, req, 0);
}
/**
*
* return the stored field associated to potentialProperty parameter
*/
public String mapStoredProperty(String potentialProperty, SolrQueryRequest req)
{
if(potentialProperty.equals("asc") || potentialProperty.equals("desc") || potentialProperty.equals("_docid_"))
{
return potentialProperty;
}
if(potentialProperty.equalsIgnoreCase("score") || potentialProperty.equalsIgnoreCase("SEARCH_SCORE"))
{
return "score";
}
AlfrescoFunctionEvaluationContext functionContext =
new AlfrescoSolr4FunctionEvaluationContext(
getNamespaceDAO(),
getDictionaryService(CMISStrictDictionaryService.DEFAULT),
NamespaceService.CONTENT_MODEL_1_0_URI,
req.getSchema());
Pair<String, String> fieldNameAndEnding = QueryParserUtils.extractFieldNameAndEnding(potentialProperty);
String luceneField = functionContext.getLuceneFieldName(fieldNameAndEnding.getFirst());
PropertyDefinition propertyDef = getPropertyDefinition(fieldNameAndEnding.getFirst());
//Retry scan using luceneField.
if(propertyDef == null)
{
if(luceneField.contains("@"))
{
int index = luceneField.lastIndexOf("@");
propertyDef = getPropertyDefinition(luceneField.substring(index +1));
}
}
if (propertyDef == null || propertyDef.getName() == null){
return mapNonPropertyFields(luceneField);
}
if (propertyDef.getName().equals(DataTypeDefinition.TEXT))
{
return getStoredTextField(propertyDef.getName());
}
else if (propertyDef.getName().equals(DataTypeDefinition.MLTEXT))
{
return getStoredMLTextField(propertyDef.getName());
}
else if (propertyDef.getName().equals(DataTypeDefinition.CONTENT))
{
return getStoredContentField(propertyDef.getName());
}
else
{
return mapAlfrescoField(FieldUse.FTS, 0, fieldNameAndEnding, luceneField, propertyDef);
}
}
public String mapProperty(String potentialProperty, FieldUse fieldUse, SolrQueryRequest req, int position)
{
if(potentialProperty.equals("asc") || potentialProperty.equals("desc") || potentialProperty.equals("_docid_"))
@@ -1527,6 +1684,12 @@ public class AlfrescoSolrDataModel implements QueryConstants
propertyDef = getPropertyDefinition(luceneField.substring(index +1));
}
}
String solrSortField;
solrSortField = mapAlfrescoField(fieldUse, position, fieldNameAndEnding, luceneField, propertyDef);
return solrSortField;
}
private String mapAlfrescoField(FieldUse fieldUse, int position, Pair<String, String> fieldNameAndEnding, String luceneField, PropertyDefinition propertyDef) {
String solrSortField;
if(propertyDef != null)
{
search-services/alfresco-search/src/main/java/org/alfresco/solr/HandlerReportHelper.java
+3 -3
View File
@@ -227,7 +227,7 @@ class HandlerReportHelper
ihr.add("Last indexed transaction commit date", CachingDateFormat.getDateFormat().format(lastTxDate));
ihr.add("Last TX id before holes", metaState.getLastIndexedTxIdBeforeHoles());
srv.addFTSStatusCounts(ihr);
srv.addContentOutdatedAndUpdatedCounts(ihr);
return ihr;
}
@@ -275,7 +275,7 @@ class HandlerReportHelper
NamedList<Object> ftsSummary = new SimpleOrderedMap<>();
long remainingContentTimeMillis = 0;
srv.addFTSStatusCounts(ftsSummary);
srv.addContentOutdatedAndUpdatedCounts(ftsSummary);
long cleanCount =
ofNullable(ftsSummary.get("Node count with FTSStatus Clean"))
.map(Number.class::cast)
@@ -422,7 +422,7 @@ class HandlerReportHelper
NamedList<Object> ftsSummary = new SimpleOrderedMap<>();
long remainingContentTimeMillis = 0;
srv.addFTSStatusCounts(ftsSummary);
srv.addContentOutdatedAndUpdatedCounts(ftsSummary);
long cleanCount =
ofNullable(ftsSummary.get("Node count with FTSStatus Clean"))
.map(Number.class::cast)
search-services/alfresco-search/src/main/java/org/alfresco/solr/InformationServer.java
+18 -9
View File
@@ -43,14 +43,14 @@ import org.json.JSONException;
/**
* This is the interface to the information server, whether it be Solr or some other search server.
* @author Ahmed Owian
*
* @author Ahmed Owian
*/
public interface InformationServer extends InformationServerCollectionProvider
{
public static final String PROP_PREFIX_PARENT_TYPE = "alfresco.metadata.ignore.datatype.";
String PROP_PREFIX_PARENT_TYPE = "alfresco.metadata.ignore.datatype.";
public static final String PROP_PREFIX_PARENT_ASPECT = "alfresco.metadata.ignore.aspect.";
String PROP_PREFIX_PARENT_ASPECT = "alfresco.metadata.ignore.aspect.";
void rollback() throws IOException;
@@ -88,7 +88,7 @@ public interface InformationServer extends InformationServerCollectionProvider
void indexNode(Node node, boolean overwrite) throws IOException, AuthenticationException, JSONException;
void indexNodes(List<Node> nodes, boolean overwrite, boolean cascade) throws IOException, AuthenticationException, JSONException;
void indexNodes(List<Node> nodes, boolean overwrite) throws IOException, AuthenticationException, JSONException;
void cascadeNodes(List<NodeMetaData> nodes, boolean overwrite) throws IOException, AuthenticationException, JSONException;
@@ -118,9 +118,9 @@ public interface InformationServer extends InformationServerCollectionProvider
boolean isInIndex(String id) throws IOException;
public void setCleanContentTxnFloor(long cleanContentTxnFloor);
void setCleanContentTxnFloor(long cleanContentTxnFloor);
public void setCleanCascadeTxnFloor(long cleanCascadeTxnFloor);
void setCleanCascadeTxnFloor(long cleanCascadeTxnFloor);
Set<Long> getErrorDocIds() throws IOException;
@@ -150,11 +150,15 @@ public interface InformationServer extends InformationServerCollectionProvider
List<TenantAclIdDbId> getDocsWithUncleanContent(int start, int rows) throws IOException;
void updateContentToIndexAndCache(long dbId, String tenant) throws Exception;
void updateContent(TenantAclIdDbId docRef) throws Exception;
void addCommonNodeReportInfo(NodeReport nodeReport);
void addFTSStatusCounts(NamedList<Object> ihr);
/**
* Adds to the input report container (a {@link NamedList}) the counts of nodes/documents whose content is
* outdated and updated (i.e. in synch with the CMS).
*/
void addContentOutdatedAndUpdatedCounts(NamedList<Object> ihr);
IndexHealthReport reportAclTransactionsInIndex(Long minAclTxId, IOpenBitSet aclTxIdsInDb, long maxAclTxId);
@@ -180,5 +184,10 @@ public interface InformationServer extends InformationServerCollectionProvider
String getBaseUrl();
void flushContentStore() throws IOException;
/**
* Check if cascade tracking is enabled.
*
* @return true if cascade tracking is enabled (note that this is the default behaviour if not specified in the properties file).
*/
boolean cascadeTrackingEnabled();
}
search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java
+1084 -1068
View File
File diff suppressed because it is too large Load Diff
search-services/alfresco-search/src/main/java/org/alfresco/solr/StripLocaleStrField.java
+43
View File
@@ -0,0 +1,43 @@
/*
* Copyright (C) 2020 Alfresco Software Limited.
*
* This file is part of Alfresco
*
* Alfresco is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Alfresco is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
*/
package org.alfresco.solr;
import static java.util.Optional.ofNullable;
import org.apache.lucene.index.IndexableField;
import org.apache.solr.schema.SchemaField;
import org.apache.solr.schema.StrField;
import java.util.List;
/**
* A {@link StrField} subclass which removes the locale language marker at the beginning of its value.
*/
public class StripLocaleStrField extends StrField
{
@Override
public List<IndexableField> createFields(SchemaField field, Object value, float boost)
{
Object newValue =
ofNullable(value).map(String.class::cast)
.map(v -> v.replaceFirst("\\x{0000}.*\\x{0000}", ""))
.orElse(null);
return super.createFields(field, newValue, boost);
}
}
search-services/alfresco-search/src/main/java/org/alfresco/solr/component/FingerPrintComponent.java
+67 -75
View File
@@ -19,100 +19,103 @@
package org.alfresco.solr.component;
import java.io.IOException;
import java.util.*;
import static org.alfresco.repo.search.adaptor.lucene.QueryConstants.FIELD_LID;
import org.alfresco.repo.search.adaptor.lucene.QueryConstants;
import org.alfresco.repo.tenant.TenantService;
import org.alfresco.solr.AlfrescoCoreAdminHandler;
import org.alfresco.solr.AlfrescoSolrDataModel;
import org.alfresco.solr.SolrInformationServer;
import org.alfresco.solr.content.SolrContentStore;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.LegacyNumericRangeQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.TopDocs;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.SolrInputField;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.core.CoreContainer;
import org.apache.solr.core.SolrCore;
import org.apache.solr.handler.component.*;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.handler.component.ResponseBuilder;
import org.apache.solr.handler.component.SearchComponent;
import org.apache.solr.response.DocsStreamer;
import org.apache.solr.schema.IndexSchema;
import org.apache.solr.schema.SchemaField;
import org.apache.solr.search.SolrIndexSearcher;
import org.apache.solr.util.plugin.SolrCoreAware;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
/**
* @author Joel Bernstein
* @since 5.2
*/
public class FingerPrintComponent extends SearchComponent implements SolrCoreAware
public class FingerPrintComponent extends SearchComponent
{
public static final String COMPONENT_NAME = "fingerprint";
public void inform(SolrCore core) {
@Override
public void prepare(ResponseBuilder responseBuilder)
{
}
public void prepare(ResponseBuilder responseBuilder) {
}
public void process(ResponseBuilder responseBuilder) throws IOException {
if(!responseBuilder.req.getParams().getBool(FingerPrintComponent.COMPONENT_NAME, false)) {
@Override
@SuppressWarnings("unchecked")
public void process(ResponseBuilder responseBuilder) throws IOException
{
if(!responseBuilder.req.getParams().getBool(FingerPrintComponent.COMPONENT_NAME, false))
{
return;
}
SolrContentStore solrContentStore = getContentStore(responseBuilder.req);
NamedList response = responseBuilder.rsp.getValues();
String id = responseBuilder.req.getParams().get("id");
NamedList<Object> response = responseBuilder.rsp.getValues();
IndexSchema schema = responseBuilder.req.getSchema();
SolrIndexSearcher searcher = responseBuilder.req.getSearcher();
long dbid = fetchDBID(id, responseBuilder.req.getSearcher());
if(dbid == -1 && isNumber(id) ) {
dbid = Long.parseLong(id);
Query q;
// Make a distinction:
// if id is a number is taken as DBID, otherwise as LID
if(isNumber(id))
{
long dbid = Long.parseLong(id);
q = LegacyNumericRangeQuery.newLongRange("DBID", dbid, dbid + 1, true, false);
}
else
{
String query = id.startsWith("workspace") ? id : "workspace://SpacesStore/"+id;
q = new TermQuery(new Term(FIELD_LID, query));
}
NamedList fingerPrint = new NamedList();
if(dbid > -1) {
SolrInputDocument solrDoc = solrContentStore.retrieveDocFromSolrContentStore(AlfrescoSolrDataModel.getTenantId(TenantService.DEFAULT_DOMAIN), dbid);
if (solrDoc != null) {
SolrInputField mh = solrDoc.getField("MINHASH");
if (mh != null) {
Collection col = mh.getValues();
List l = new ArrayList();
l.addAll(col);
fingerPrint.add("MINHASH", l);
}
TopDocs docs = searcher.search(q, 1);
Set<String> fields = new HashSet<>();
fields.add("MINHASH");
NamedList<Object> fingerPrint = new NamedList<>();
List<Object> values = new ArrayList<>();
if(docs.totalHits == 1)
{
ScoreDoc scoreDoc = docs.scoreDocs[0];
Document doc = searcher.doc(scoreDoc.doc, fields);
IndexableField[] minHashes = doc.getFields("MINHASH");
for (IndexableField minHash : minHashes)
{
SchemaField sf = schema.getFieldOrNull(minHash.name());
Object value = DocsStreamer.getValue(sf, minHash);
values.add(value);
fingerPrint.add("MINHASH", values);
}
}
response.add("fingerprint", fingerPrint);
}
private long fetchDBID(String UUID, SolrIndexSearcher searcher) throws IOException {
String query = "workspace://SpacesStore/"+UUID;
TermQuery q = new TermQuery(new Term(QueryConstants.FIELD_LID, query));
TopDocs docs = searcher.search(q, 1);
Set<String> fields = new HashSet();
fields.add(QueryConstants.FIELD_DBID);
if(docs.totalHits == 1) {
ScoreDoc scoreDoc = docs.scoreDocs[0];
Document doc = searcher.doc(scoreDoc.doc, fields);
IndexableField dbidField = doc.getField(QueryConstants.FIELD_DBID);
return dbidField.numericValue().longValue();
}
return -1;
}
private boolean isNumber(String s) {
for(int i=0; i<s.length(); i++) {
if(!Character.isDigit(s.charAt(i))) {
private boolean isNumber(String s)
{
for(int i=0; i<s.length(); i++)
{
if(!Character.isDigit(s.charAt(i)))
{
return false;
}
}
@@ -120,20 +123,9 @@ public class FingerPrintComponent extends SearchComponent implements SolrCoreAwa
return true;
}
private SolrContentStore getContentStore(SolrQueryRequest req)
@Override
public String getDescription()
{
if(req.getSearcher() != null)
{
CoreContainer coreContainer = req.getSearcher().getCore().getCoreContainer();
AlfrescoCoreAdminHandler coreAdminHandler = (AlfrescoCoreAdminHandler) coreContainer.getMultiCoreHandler();
SolrInformationServer srv = (SolrInformationServer) coreAdminHandler.getInformationServers().get(req.getSearcher().getCore().getName());
return srv.getSolrContentStore();
}
return null;
}
public String getDescription() {
return null;
}
}
}
search-services/alfresco-search/src/main/java/org/alfresco/solr/component/RewriteFieldListComponent.java
+144
View File
@@ -0,0 +1,144 @@
/*
* Copyright (C) 2020 Alfresco Software Limited.
*
* This file is part of Alfresco
*
* Alfresco is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Alfresco is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
*/
package org.alfresco.solr.component;
import org.alfresco.solr.AlfrescoSolrDataModel;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.handler.component.ResponseBuilder;
import org.apache.solr.handler.component.SearchComponent;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.search.SolrReturnFields;
import java.io.IOException;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import static java.util.Optional.ofNullable;
import static org.alfresco.solr.AlfrescoSolrDataModel.FieldUse.FACET;
import static org.alfresco.solr.AlfrescoSolrDataModel.FieldUse.FTS;
import static org.alfresco.solr.AlfrescoSolrDataModel.FieldUse.ID;
import static org.alfresco.solr.AlfrescoSolrDataModel.FieldUse.SORT;
/**
* @Author elia
*/
/**
* Transform the fieldlist depending on the use of cached transformer:
* [cached] -> add to the field list the translations of the fiels to the internal schema notation
* otherwise -> modify the field list in order to contains a subset of the following fields:
* id, DBID, _version_ and score
*/
public class RewriteFieldListComponent extends SearchComponent {
private boolean checkParamsHaveBeenRewritten(SolrParams params)
{
return (params.get("originalFl") != null);
}
private void transformFieldList(SolrQueryRequest req)
{
Set<String> fieldListSet = new HashSet<>();
Set<String> defaultNonCachedFields = Set.of("id","DBID", "_version_");
Set<String> allowedNonCachedFields = new HashSet<>(defaultNonCachedFields);
allowedNonCachedFields.add("score");
SolrReturnFields solrReturnFields = new SolrReturnFields(req);
String originalFieldList = req.getParams().get("fl");
boolean cacheTransformer = ofNullable(solrReturnFields.getTransformer())
.map(t -> t.getName())
.map(name -> name.contains("fmap"))
.orElse(false);
ModifiableSolrParams params = new ModifiableSolrParams(req.getParams());
// In case cache transformer is no set, we need to modify the field list in order return
// only id, DBID and _version_ fields
if (!cacheTransformer){
if (!solrReturnFields.wantsAllFields())
{
fieldListSet.addAll(solrReturnFields.getLuceneFieldNames()
.stream()
.filter(field -> allowedNonCachedFields.contains(field))
.collect(Collectors.toSet()));
}
if (fieldListSet.isEmpty())
{
fieldListSet.addAll(defaultNonCachedFields);
}
params.set("fl", fieldListSet.stream().collect(Collectors.joining(",")));
}
else
{
if (solrReturnFields.wantsAllFields() || solrReturnFields.hasPatternMatching())
{
fieldListSet.add("*");
}
else
{
fieldListSet.addAll(solrReturnFields.getLuceneFieldNames().stream()
.map( field -> AlfrescoSolrDataModel.getInstance()
.mapStoredProperty(field, req))
.filter(schemaFieldName -> schemaFieldName != null)
.map(schemaFieldName -> schemaFieldName.chars()
.mapToObj(c -> (char) c)
.map(c -> Character.isJavaIdentifierPart(c)? c : '?')
.map(Object::toString)
.collect(Collectors.joining()))
.collect(Collectors.toSet()));
}
params.add("fl", fieldListSet.stream().collect(Collectors.joining(",")));
}
// This is added for filtering the fields in the cached transformer.
params.set("originalFl", originalFieldList);
req.setParams(params);
}
@Override
public void prepare(ResponseBuilder responseBuilder) {
SolrQueryRequest req = responseBuilder.req;
if (checkParamsHaveBeenRewritten(req.getParams()))
return;
transformFieldList(req);
}
@Override
public void process(ResponseBuilder responseBuilder)
{
}
@Override
public String getDescription()
{
return null;
}
}
search-services/alfresco-search/src/main/java/org/alfresco/solr/content/AccessMode.java
-106
View File
@@ -1,106 +0,0 @@
/*
* Copyright (C) 2005-2019 Alfresco Software Limited.
*
* This file is part of Alfresco
*
* Alfresco is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Alfresco is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
*/
package org.alfresco.solr.content;
import org.alfresco.solr.client.NodeMetaData;
import org.apache.solr.common.SolrInputDocument;
import java.io.Closeable;
import java.io.IOException;
import java.util.List;
import java.util.Map;
/**
* Behavioural interface which indicates the type of content store access mode for the owning node.
* A replication interaction is composed at least by 2 roles: a master and a slave.
*
* Despite the same interface, the behaviour of the content store management changes depending on the node kind:
*
* <ul>
* <li>Master Node: READ + WRITE + Changes tracking. Writes and changes tracking are a direct consequence of the "indexing" nature of the master node.</li>
* <li>Slave Node: READ ONLY (i.e. never write: changes are applied on the master and replicated on slaves)</li>
* </ul>
*
* Important: the owning entity *is not* the SolrCore instance: the Content store is shared across all cores of
* A node can transit from Read to Write mode
*
* @author Andrea Gazzarini
* @since 1.5
*/
interface AccessMode extends Closeable
{
/**
* Returns the last persisted content store version.
*
* @return the last persisted content store version, SolrContentStore#NO_VERSION_AVAILABLE in case the version isn't available.
*/
long getLastCommittedVersion();
/**
* Persists the last committed version on the hosting node.
* Note that this is tipically valid only on slave node, because the master already manages the content store version on
* a persistent storage, so it doesn't need to call this method.
*
* @param version the last committed content store version.
*/
void setLastCommittedVersion(long version);
Map<String, List<Map<String, Object>>> getChanges(long version);
/**
* Stores a {@link SolrInputDocument} into Alfresco solr content store.
*
* @param tenant the owning tenant.
* @param dbId the document DBID
* @param doc the document itself.
*/
void storeDocOnSolrContentStore(String tenant, long dbId, SolrInputDocument doc);
/**
* Stores a {@link SolrInputDocument} into Alfresco solr content store.
*
* @param nodeMetaData the node metadata.
* @param doc the document itself.
*/
void storeDocOnSolrContentStore(NodeMetaData nodeMetaData, SolrInputDocument doc);
/**
* Removes a node from the content store.
*
* @param nodeMetaData the node metadata.
*/
void removeDocFromContentStore(NodeMetaData nodeMetaData);
/**
* Flushes pending changesets.
*
* @throws IOException in case of I/O failure.
*/
void flushChangeSet() throws IOException;
/**
* Tries to transition from this mode to the readOnlyMode.
*/
void switchOnReadOnlyMode();
/**
* Tries to transition from this mode to the read/write mode.
*/
void switchOnReadWriteMode();
}
search-services/alfresco-search/src/main/java/org/alfresco/solr/content/ChangeSet.java
-460
View File
@@ -1,460 +0,0 @@
/*
* Copyright (C) 2005-2019 Alfresco Software Limited.
*
* This file is part of Alfresco
*
* Alfresco is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Alfresco is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
*/
package org.alfresco.solr.content;
import static java.util.Arrays.asList;
import static java.util.Arrays.stream;
import static java.util.Collections.emptySet;
import static java.util.Optional.ofNullable;
import org.alfresco.util.Pair;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.LongPoint;
import org.apache.lucene.document.NumericDocValuesField;
import org.apache.lucene.document.StoredField;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.search.FieldDoc;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.MatchAllDocsQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.SearcherManager;
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.SortField;
import org.apache.lucene.search.SortedNumericSortField;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.FSDirectory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.Closeable;
import java.io.File;
import java.io.IOException;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.function.BiFunction;
import java.util.function.BinaryOperator;
/**
* Stores and manages changes occurred in a content store.
*
* Note this entity is part of the content store only when it is set on writable mode (i.e. only when the hosting node
* is a standalone instance or it is a master node).
*
* That means if the hosting node has at least one standalone core or one master node, then the managed content store
* will be set in write mode and therefore a {@link ChangeSet} instance will track all changes applied to it.
*
* In that mode, each change is associated to a given (incremental) version; changes consist of adds/updates and deletes.
* Each time the {@link org.alfresco.solr.tracker.CommitTracker} executes a commit in the main index, all content store
* changes accumulated in the meantime in the current {@link ChangeSet} instance are flushed in an auxiliary Lucene index
* and therefore persisted.
*
* On the other side, when all cores on a given node are configured as slaves, the content store is in read-only mode,
* and there's no need to track any change (i.e. changes are coming from master through the replication procedure).
*
* As a side note: we can have a persistent or transient {@link ChangeSet} instance. The first one is used to manage, as
* the name suggests, in a persistent way the content store changes accumulated during the indexing operations (see the
* description above about its behaviour in master nodes).
*
* The second one is used instead for computing/reducing the "merged" list of changes that we need to communicate to slave nodes:
* as part of the replication mechanism, a slave communicates its last synched content store version; on the master side,
* we need to compute and communicate back all changes that the slave needs to apply since that version in order to be in
* synch with master.
*
* @author Andrea Gazzarini
* @since 1.5
*/
public class ChangeSet implements AutoCloseable
{
private final static Logger LOGGER = LoggerFactory.getLogger(ChangeSet.class);
private final static ChangeSet EMPTY_CHANGESET = new ChangeSet.Builder().empty().build();
/**
* Builds class for creating {@link ChangeSet} instances.
* The builder is here for creating three kind of {@link ChangeSet} instances:
*
* <ul>
* <li>Persistent: used for tracking and persisting content store changes.</li>
* <li>Transient: used for computing the merged list of changes a slave should apply for being in synch with master.</li>
* <li>Empty: an immutable "NullObject" used for denoting an empty {@link ChangeSet} instance.</li>
* </ul>
*
* @see <a href="https://en.wikipedia.org/wiki/Null_object_pattern">Null Object Design Pattern</a>
*/
public static class Builder
{
private String root;
private boolean immutable;
/**
* Adds a content store root folder to this builder.
* This automatically implies we are creating a persistent {@link ChangeSet} instance.
*
* @param root the absolute path of the content store root folder.
* @return this builder, for being used in a fluent mode.
*/
Builder withContentStoreRoot(String root)
{
if (root == null) throw new IllegalArgumentException("Unable to build the Changeset structures with a null content store root folder.");
if (!new File(root).canWrite()) throw new IllegalArgumentException("Unable to build the Changeset structures with a non-writeable content store root folder.");
this.root = root;
return this;
}
/**
* We are interested in building an empty, immutable {@link ChangeSet} instance.
*
* @return this builder, for being used in a fluent mode.
*/
Builder empty()
{
this.root = null;
this.immutable = true;
return this;
}
/**
* Builds the product of this builder (i.e. the {@link ChangeSet} instance).
*
* @return the product of this builder (i.e. the {@link ChangeSet} instance).
*/
ChangeSet build()
{
if (root == null)
{
// Creates a transient changeset (no persistence); mainly used for reducing the changes during replication.
return new ChangeSet(
immutable ? emptySet() : new HashSet<>(),
immutable ? emptySet() : new HashSet<>());
}
IndexWriter writer = null;
try
{
File indexDirectory = new File(root, CHANGESETS_ROOT_FOLDER_NAME);
writer = new IndexWriter(FSDirectory.open(indexDirectory.toPath()), new IndexWriterConfig());
writer.commit();
final SearcherManager searcher = new SearcherManager(writer, null);
LOGGER.info("ContentStore Changeset index has been correctly mounted on {}", indexDirectory.getAbsolutePath());
return new ChangeSet(
searcher,
writer,
immutable ? emptySet() : new HashSet<>(),
immutable ? emptySet() : new HashSet<>());
}
catch (Exception exception)
{
ofNullable(writer).ifPresent(ChangeSet::silentyClose);
throw new IllegalArgumentException("Unable to create a ContentStore ChangeSet data structure. See further details in the stacktrack below.", exception);
}
}
}
private final static String VERSION_FIELD_NAME = "version";
private final static String RVERSION_FIELD_NAME = "rversion";
private final static String ADDS_FIELD_NAME = "adds";
private final static String DELETES_FIELD_NAME = "deletes";
final static String CHANGESETS_ROOT_FOLDER_NAME = "changeSets";
Set<String> deletes;
Set<String> adds;
SearcherManager searcher;
private final IndexWriter writer;
Query selectEverything = new MatchAllDocsQuery();
/**
* Builds a new transient {@link ChangeSet} with the given Lucene facades.
*
* @param deletesContainer the container which will hold the deletes.
* @param addsContainer the container which will hold the adds/updates.
*/
private ChangeSet(
final Set<String> deletesContainer,
final Set<String> addsContainer)
{
this(null, null, deletesContainer, addsContainer);
}
/**
* Builds a new {@link ChangeSet} with the given Lucene facades.
*
* @param searcher the searcher reference (actually a {@link SearcherManager} instance instead of dealing with {@link IndexSearcher} directly.
* @param writer the {@link IndexWriter} instance used for persisting the content store changes.
* @param deletesContainer the container which will hold the deletes.
* @param addsContainer the container which will hold the adds/updates.
*/
private ChangeSet(
final SearcherManager searcher,
final IndexWriter writer,
final Set<String> deletesContainer,
final Set<String> addsContainer)
{
this.searcher = searcher;
this.writer = writer;
this.deletes = deletesContainer;
this.adds = addsContainer;
}
/**
* Records a delete change.
*
* @param path the relative path of the file which has been deleted.
*/
synchronized void delete(String path)
{
adds.remove(path);
deletes.add(path);
LOGGER.debug("ContentStore change recorded: item {} has been deleted.", path);
debugPendingChanges();
}
/**
* Records an add or update change.
*
* @param path the relative path of the file which has been updated or added.
*/
synchronized void addOrReplace(String path)
{
deletes.remove(path);
adds.add(path);
LOGGER.debug("ContentStore change recorded: item {} has been added/updated.", path);
debugPendingChanges();
}
/**
* Flushes all pending collected content store changes.
*
* @throws IOException in case of I/O failure.
*/
void flush() throws IOException
{
// No ops if this is a transient changeset
if (searcher == null || writer == null) {
return;
}
if (adds.isEmpty() && deletes.isEmpty())
{
LOGGER.debug("No changes in contentstore to flush");
return;
}
final long version = System.currentTimeMillis();
LOGGER.debug("About to add a new Changeset entry (version = {}, deletes = {}, adds = {})", version, deletes.size(), adds.size());
final Document document = new Document();
document.add(new NumericDocValuesField(VERSION_FIELD_NAME, version));
document.add(new LongPoint(RVERSION_FIELD_NAME, version));
Set<String> tmpDel;
Set<String> tmpAdd;
synchronized (this) {
tmpDel = deletes;
deletes = new HashSet<>();
tmpAdd = adds;
adds = new HashSet<>();
}
tmpDel.stream()
.map(item -> new StoredField(DELETES_FIELD_NAME, item))
.forEach(document::add);
tmpAdd.stream()
.map(item -> new StoredField(ADDS_FIELD_NAME, item))
.forEach(document::add);
writer.addDocument(document);
writer.commit();
searcher.maybeRefresh();
LOGGER.debug("New Changeset entry have been added (version = {}, deletes = {}, adds = {})", version, deletes.size(), adds.size());
}
/**
* Sanity check for making sure the requestor sent a valid and known content store version.
*
* @param version the content store version (sent by a requestor)
* @return true if the given version is unknown, that is, is not part of the content store version history.
*/
boolean isUnknownVersion(long version)
{
try
{
TopDocs hits = searcher().search(LongPoint.newExactQuery(RVERSION_FIELD_NAME, version), 1);
return hits.totalHits != 1;
}
catch(Exception exception)
{
LOGGER.error("Unable to check the requested version ({}) in the local versioning store. See further details in the stacktrace below.", version, exception);
return true;
}
}
@Override
public void close()
{
ofNullable(writer).ifPresent(ChangeSet::silentyClose);
ofNullable(searcher).ifPresent(ChangeSet::silentyClose);
}
/**
* Returns the last persisted content store version.
*
* @return the last persisted content store version, SolrContentStore#NO_VERSION_AVAILABLE in case the version isn't available.
*/
long getLastCommittedVersion()
{
try
{
TopDocs hits = searcher().search(
selectEverything,
1,
new Sort(new SortedNumericSortField(VERSION_FIELD_NAME, SortField.Type.LONG, true)),
false,
false);
return ofNullable(hits.scoreDocs)
.filter(docs -> docs.length > 0)
.map(docs -> ((FieldDoc) docs[0]).fields)
.filter(fields -> fields.length > 0)
.map(fields -> (Long)fields[0])
.orElse(SolrContentStore.NO_VERSION_AVAILABLE);
}
catch(Exception exception)
{
LOGGER.error("Unable to retrieve the last committed content store changeset version. " +
"As consequence of that a dummy value of " + SolrContentStore.NO_VERSION_AVAILABLE +
" will be returned. See further details in the stacktrack below.",
exception);
return SolrContentStore.NO_VERSION_AVAILABLE;
}
}
/**
* Returns the content store changes (adds / deletes) since the given version (exclusive).
*
* @param version the start offset version (exclusive).
* @return the content store changes (adds / deletes) since the given version (exclusive).
*/
public ChangeSet since(long version)
{
try
{
Query query = LongPoint.newRangeQuery(RVERSION_FIELD_NAME, Math.addExact(version, 1), Long.MAX_VALUE);
TopDocs hits = searcher().search(
query,
100,
new Sort(new SortedNumericSortField(VERSION_FIELD_NAME, SortField.Type.LONG)),
false,
false);
final BiFunction<ChangeSet, ? super Pair<List<String>,List<String>>, ChangeSet> accumulator =
(partial, nth) -> {
final List<String> nthDeletes = nth.getFirst();
final List<String> nthAdds = nth.getSecond();
nthDeletes.forEach(partial::delete);
nthAdds.forEach(partial::addOrReplace);
return partial;
};
final BinaryOperator<ChangeSet> combiner = (c1, c2) -> {
c1.deletes.forEach(c2::delete);
c1.adds.forEach(c2::addOrReplace);
return c1;
};
return stream(hits.scoreDocs)
.map(this::toDoc)
.map(doc ->
new Pair<>(
asList(doc.getValues(DELETES_FIELD_NAME)),
asList(doc.getValues(ADDS_FIELD_NAME))))
.reduce(new ChangeSet.Builder().build(), accumulator, combiner);
}
catch(Exception exception)
{
LOGGER.error("Unable to retrieve the changeset since version {}. " +
"As consequence of that an empty result will be returned. " +
"See further details in the stacktrack below.",
version,
exception);
return EMPTY_CHANGESET;
}
}
private Document toDoc(ScoreDoc hit)
{
try
{
return searcher().doc(hit.doc);
}
catch (IOException e)
{
throw new RuntimeException(e);
}
}
private void debugPendingChanges()
{
if (LOGGER.isDebugEnabled())
{
LOGGER.debug("ContentStore pending deletes: " + deletes);
LOGGER.debug("ContentStore pending adds/updates: " + adds);
}
}
/**
* Silently close (i.e. without any exception re-throwing) the incoming resource.
*
* @param resource the closeable resource.
*/
private static void silentyClose(Closeable resource)
{
try
{
resource.close();
}
catch (Exception exception)
{
LOGGER.error("Unable to properly close the resource instance {}. See further details in the stacktrace below.", resource, exception);
}
}
private IndexSearcher searcher() throws IOException
{
return searcher.acquire();
}
}
search-services/alfresco-search/src/main/java/org/alfresco/solr/content/SolrContentStore.java
-707
View File
@@ -1,707 +0,0 @@
/*
* Copyright (C) 2005-2019 Alfresco Software Limited.
*
* This file is part of Alfresco
*
* Alfresco is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Alfresco is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
*/
package org.alfresco.solr.content;
import org.alfresco.repo.content.ContentContext;
import org.alfresco.service.cmr.repository.ContentReader;
import org.alfresco.service.cmr.repository.ContentWriter;
import org.alfresco.solr.AlfrescoSolrDataModel;
import org.alfresco.solr.client.NodeMetaData;
import org.alfresco.solr.config.ConfigUtil;
import org.alfresco.solr.handler.AlfrescoReplicationHandler;
import org.apache.commons.io.FileUtils;
import org.apache.lucene.util.BytesRef;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.util.JavaBinCodec;
import org.apache.solr.core.SolrResourceLoader;
import org.apache.solr.handler.SnapShooter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.Closeable;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.function.Predicate;
import java.util.stream.Stream;
import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;
import static java.util.Collections.emptyList;
import static java.util.Collections.emptyMap;
import static java.util.Collections.singletonList;
import static java.util.Optional.of;
import static java.util.Optional.ofNullable;
import static java.util.stream.Collectors.toList;
import static org.alfresco.solr.content.SolrContentUrlBuilder.FILE_EXTENSION;
import static org.alfresco.solr.content.SolrContentUrlBuilder.logger;
/**
* A content store specific to SOLR's requirements: The URL is generated from a
* set of properties such as:
*
* <ul>
* <li>ACL ID</li>
* <li>DB ID</li>
* <li>Other metadata</li>
* </ul>
*
* The URL, if not known, can be reliably regenerated using the {@link SolrContentUrlBuilder}.
* <br/>
*
* Since version 1.5 this class acts as a logical singleton: there will be only one instance per node.
* That reflects exactly what we physically have in the filesystem (i.e. there's only one content store per node).
*
* That unique instance is created at startup in {@link org.alfresco.solr.AlfrescoCoreAdminHandler} and then passed to
* each registered core (see {@link org.alfresco.solr.lifecycle.SolrCoreLoadListener})
*
* The State pattern implemented by means of the {@link AccessMode} interface allows for a given {@link SolrContentStore} instance to act:
*
* <ul>
* <li>in READ/WRITE mode: when <b>at least one core</b> of the hosting node is a master or it is a standalone shard/instance.</li>
* <li>in READ ONLY mode: when <b>all cores</b> of the hosting node are slaves.</li>
* </ul>
*
* The Finite State Machine (FST) provides three possible states: Initial, Read Only, Read/Write.
* The allowed transitions are:
*
* <ul>
* <li>Initial -> ReadOnly: a slave core has been registered, the content store hasn't been yet initialised.</li>
* <li>
* ReadOnly -> Read/Write: this scenario is unusual because we should have on the same instance a mixed set of cores
* (some master/standalone, some slaves). It happens when a first slave core registers and causes the transition
* described in the first point (initial -> read only). Then a master or standalone core registers so we need to
* move from a read only to a complete readable/writable managed content store.
* </li>
* <li> Initial -> Read/Write: a master or standalone core has been registered, and the content store hasn't been yet initialised.</li>
* </ul>
*
* Note the following transitions are not allowed:
*
* <ul>
* <li>Coming back to Initial state: once it has been initialised the Content Store cannot return back to the "Initial" state.</li>
* <li>
* Read/Write -> ReadOnly: if at least one master or standalone core registers, the content store is permanently moved in Read/Write mode.
* So even a further slave node registers (not very usually as described above) the content store remains in RW mode.
* </li>
* </ul>
*
* @author Derek Hulley
* @author Michael Suzuki
* @author Andrea Gazzarini
* @since 1.5
* @see org.alfresco.solr.lifecycle.SolrCoreLoadListener
* @see <a href="https://it.wikipedia.org/wiki/State_pattern">State Pattern</a>
*/
public final class SolrContentStore implements Closeable, AccessMode
{
private final static Logger LOGGER = LoggerFactory.getLogger(SolrContentStore.class);
public static final long NO_VERSION_AVAILABLE = -1L;
public static final long NO_CONTENT_STORE_REPLICATION_REQUIRED = -2L;
static final String CONTENT_STORE = "contentstore";
static final String SOLR_CONTENT_DIR = "solr.content.dir";
public static final String INFO = "info";
public static final String FULL_REPLICATION = "full-replication";
public static final String DELETES = "deletes";
public static final String ADDS = "adds";
private final Predicate<File> onlyDatafiles = file -> file.isFile() && file.getName().endsWith(FILE_EXTENSION);
private final String root;
/**
* Used for denoting the very beginning state, when the {@link SolrContentStore} has been created
* but we don't know (yet) which is the role played by the cores that will register on the hosting Solr node.
*/
AccessMode notYetSet = new AccessMode()
{
@Override
public long getLastCommittedVersion()
{
throw new IllegalStateException("ContentStore hasn't been properly initialised.");
}
@Override
public void setLastCommittedVersion(long version)
{
throw new IllegalStateException("ContentStore hasn't been properly initialised.");
}
@Override
public Map<String, List<Map<String, Object>>> getChanges(long version)
{
throw new IllegalStateException("ContentStore hasn't been properly initialised.");
}
@Override
public void storeDocOnSolrContentStore(String tenant, long dbId, SolrInputDocument doc)
{
throw new IllegalStateException("ContentStore hasn't been properly initialised.");
}
@Override
public void storeDocOnSolrContentStore(NodeMetaData nodeMetaData, SolrInputDocument doc)
{
throw new IllegalStateException("ContentStore hasn't been properly initialised.");
}
@Override
public void removeDocFromContentStore(NodeMetaData nodeMetaData)
{
throw new IllegalStateException("ContentStore hasn't been properly initialised.");
}
@Override
public void flushChangeSet()
{
throw new IllegalStateException("ContentStore hasn't been properly initialised.");
}
@Override
public void switchOnReadOnlyMode()
{
logger.info("Switching the content store to ReadOnly mode.");
currentAccessMode = readOnly;
}
@Override
public void switchOnReadWriteMode()
{
logger.info("Switching the content store to Read/Write mode.");
readWrite.init();
currentAccessMode = readWrite;
}
@Override
public void close()
{
// Nothing to close here
}
};
final InitialisableAccessMode readOnly = new InitialisableAccessMode()
{
@Override
public void init()
{
// Nothing to be done here...
}
@Override
public void switchOnReadWriteMode()
{
logger.info("Switching from ReadOnly to Read/Write Content Store.");
readWrite.init();
currentAccessMode = readWrite;
logger.info("Switching from ReadOnly to Read/Write Content Store.");
}
@Override
public void switchOnReadOnlyMode()
{
logger.info("The content store is already in ReadOnly mode so this call won't have any effect.");
}
@Override
public long getLastCommittedVersion()
{
try
{
return Files.readAllLines(Paths.get(root, ".version"))
.stream()
.map(Long::parseLong)
.findFirst()
.orElse(NO_VERSION_AVAILABLE);
}
catch (Exception e)
{
return NO_VERSION_AVAILABLE;
}
}
@Override
public void setLastCommittedVersion(long version)
{
File tmpFile = new File(root, ".version-" + new SimpleDateFormat(SnapShooter.DATE_FMT, Locale.ROOT).format(new Date()));
try
{
FileWriter wr = new FileWriter(tmpFile);
wr.write(Long.toString(version));
wr.close();
// file.renameTo(..) does not work on windows. Use Files.move instead.
Files.move(tmpFile.toPath(), new File(root, ".version").toPath(), StandardCopyOption.ATOMIC_MOVE);
}
catch (IOException exception)
{
logger.error("Unable to persist the last committed content store version {}. See the stacktrace below for furtger details.", version, exception);
try
{
Files.delete(tmpFile.toPath());
}
catch (IOException e)
{
logger.error("Unable to delete tmp contentstore version file {}.", version);
}
}
}
@Override
public Map<String, List<Map<String, Object>>> getChanges(long version)
{
logger.warn("NoOp SolrContentStore changes call on slave side: this shouldn't happen because the ContentStore is in read-only mode when the hosting node is a slave.");
return emptyMap();
}
@Override
public void storeDocOnSolrContentStore(String tenant, long dbId, SolrInputDocument doc)
{
logger.warn("NoOp SolrContentStore write call on slave side: this shouldn't happen because the ContentStore is in read-only mode when the hosting node is a slave.");
}
@Override
public void storeDocOnSolrContentStore(NodeMetaData nodeMetaData, SolrInputDocument doc)
{
logger.warn("NoOp SolrContentStore write call on slave side: this shouldn't happen because the ContentStore is in read-only mode when the hosting node is a slave.");
}
@Override
public void removeDocFromContentStore(NodeMetaData nodeMetaData)
{
logger.warn("NoOp SolrContentStore write call on slave side: this shouldn't happen because the ContentStore is in read-only mode when the hosting node is a slave.");
}
@Override
public void flushChangeSet()
{
logger.warn("NoOp ChangeSet tracking call on slave side: this shouldn't happen because the ContentStore is in read-only mode when the hosting node is a slave.");
}
@Override
public void close()
{
// There's nothing to close on slave side
}
};
final InitialisableAccessMode readWrite = new InitialisableAccessMode()
{
private ChangeSet changeSet;
@Override
public void init()
{
changeSet = ofNullable(changeSet).orElseGet(() -> new ChangeSet.Builder().withContentStoreRoot(root).build());
}
@Override
public long getLastCommittedVersion()
{
return changeSet.getLastCommittedVersion();
}
@Override
public void setLastCommittedVersion(long version)
{
// Do nothing here, as we are on master-side
}
@Override
public Map<String, List<Map<String, Object>>> getChanges(long version)
{
// The slave doesn't have a version, we are listing the whole content store
if (version <= NO_VERSION_AVAILABLE || changeSet.isUnknownVersion(version))
{
String message = "A slave requested the content store synchronization " +
((version <= NO_VERSION_AVAILABLE)
? " without providing any local version (actually {})."
: " with an invalid/unknown local version number ({}).") +
"As consequence of that this master will list the whole content store because a full replication is needed.";
logger.info(message, version);
return Map.of(
INFO, singletonList(Map.of(FULL_REPLICATION, true)),
ADDS, fullContentStore(),
DELETES, emptyList());
}
ChangeSet changes = changeSet.since(version);
return Map.of(
INFO, singletonList(Map.of(FULL_REPLICATION, false)),
DELETES,
changes.deletes.stream()
.map(path -> Map.<String, Object>of("name", path))
.collect(toList()),
ADDS,
changes.adds.stream()
.map(relativePath -> root + relativePath)
.map(File::new)
.map(file -> new AlfrescoReplicationHandler.FileInfo(file, file.getAbsolutePath().replace(root, "")))
.map(AlfrescoReplicationHandler.FileInfo::getAsMap)
.collect(toList()));
}
@Override
public void storeDocOnSolrContentStore(String tenant, long dbId, SolrInputDocument doc)
{
ContentContext contentContext =
of(SolrContentUrlBuilder
.start()
.add(SolrContentUrlBuilder.KEY_TENANT, tenant)
.add(SolrContentUrlBuilder.KEY_DB_ID, String.valueOf(dbId)))
.map(SolrContentUrlBuilder::getContentContext)
.orElseThrow(() -> new IllegalArgumentException("Unable to build a Content Context from tenant " + tenant + " and DBID " + dbId));
this.delete(contentContext.getContentUrl());
ContentWriter writer = this.getWriter(contentContext);
LOGGER.debug("Writing {}/{} to {}", tenant, dbId, contentContext.getContentUrl());
try (OutputStream contentOutputStream = writer.getContentOutputStream();
GZIPOutputStream gzip = new GZIPOutputStream(contentOutputStream))
{
JavaBinCodec codec = new JavaBinCodec(resolver);
codec.marshal(doc, gzip);
File file = getFileFromUrl(contentContext.getContentUrl());
changeSet.addOrReplace(relativePath(file));
}
catch (Exception exception)
{
LOGGER.warn("Unable to write to Content Store using URL: {}", contentContext.getContentUrl(), exception);
}
}
@Override
public void storeDocOnSolrContentStore(NodeMetaData nodeMetaData, SolrInputDocument doc) {
String fixedTenantDomain = AlfrescoSolrDataModel.getTenantId(nodeMetaData.getTenantDomain());
storeDocOnSolrContentStore(fixedTenantDomain, nodeMetaData.getId(), doc);
}
@Override
public void removeDocFromContentStore(NodeMetaData nodeMetaData)
{
String fixedTenantDomain = AlfrescoSolrDataModel.getTenantId(nodeMetaData.getTenantDomain());
String contentUrl = SolrContentUrlBuilder
.start()
.add(SolrContentUrlBuilder.KEY_TENANT, fixedTenantDomain)
.add(SolrContentUrlBuilder.KEY_DB_ID, String.valueOf(nodeMetaData.getId()))
.getContentContext()
.getContentUrl();
delete(contentUrl);
}
@Override
public void flushChangeSet() throws IOException
{
changeSet.flush();
}
@Override
public void switchOnReadWriteMode()
{
logger.debug("The content store is already in ReadWrite mode; as consequence of that, the incoming \"SET-TO-RW-MODE\" call won't have any effect.");
}
@Override
public void switchOnReadOnlyMode()
{
logger.debug("A writable content store cannot switch in ReadOnly mode. This could happen in an edge case where" +
" on the same Solr node we have masters and slaves nodes");
}
@Override
public void close()
{
changeSet.close();
}
private List<Map<String, Object>> fullContentStore()
{
try
{
return Files.walk(Paths.get(root))
.map(Path::toFile)
.filter(onlyDatafiles)
.map(file -> new AlfrescoReplicationHandler.FileInfo(file, file.getAbsolutePath().replace(root, "")))
.map(AlfrescoReplicationHandler.FileInfo::getAsMap)
.collect(toList());
}
catch (Exception e)
{
LOGGER.error("An exception occurred while retrieving the whole ContentStore filelist. " +
"As consequence of that an empty list will be returned (i.e. no ContentStore synch will happen).");
return emptyList();
}
}
private void delete(String contentUrl)
{
File file = getFileFromUrl(contentUrl);
if (file.delete()) changeSet.delete(relativePath(file));
}
private ContentWriter getWriter(ContentContext context)
{
String url = context.getContentUrl();
File file = getFileFromUrl(url);
return new SolrFileContentWriter(file, url);
}
};
AccessMode currentAccessMode = notYetSet;
private final JavaBinCodec.ObjectResolver resolver = (o, codec) -> {
if(o instanceof BytesRef)
{
BytesRef br = (BytesRef)o;
codec.writeByteArray(br.bytes,br.offset,br.length);
return null;
}
return o;
};
/**
* Builds a new {@link SolrContentStore} instance with the given SOLR HOME.
*
* @param solrHome the Solr HOME.
*/
public SolrContentStore(String solrHome)
{
if (solrHome == null || solrHome.isEmpty())
{
throw new RuntimeException("Path to SOLR_HOME is required");
}
File solrHomeFile = new File(SolrResourceLoader.normalizeDir(solrHome));
if (!solrHomeFile.exists())
{
//Its very unlikely that solrHome would not exist so we will log an error
//but continue because solr.content.dir may be specified, so it keeps working
LOGGER.error(solrHomeFile.getAbsolutePath() + " does not exist.");
}
String path = solrHomeFile.getParent() + "/" + CONTENT_STORE;
LOGGER.warn(path + " will be used as a default path if " + SOLR_CONTENT_DIR + " property is not defined");
File rootFile = new File(ConfigUtil.locateProperty(SOLR_CONTENT_DIR, path));
try
{
FileUtils.forceMkdir(rootFile);
}
catch (Exception e) {
throw new RuntimeException("Failed to create directory for content store: " + rootFile, e);
}
this.root = rootFile.getAbsolutePath();
}
/**
* Returns the content store changes since the given (requestor) version.
*
* @param version the requestor version.
* @return the content store changes since the given (requestor) version.
*/
@Override
public Map<String, List<Map<String, Object>>> getChanges(long version)
{
return currentAccessMode.getChanges(version);
}
/**
* Retrieve document from SolrContentStore.
*
* @param tenant identifier
* @param dbId identifier
* @return {@link SolrInputDocument} searched document
*/
public SolrInputDocument retrieveDocFromSolrContentStore(String tenant, long dbId)
{
String contentUrl =
SolrContentUrlBuilder.start()
.add(SolrContentUrlBuilder.KEY_TENANT, tenant)
.add(SolrContentUrlBuilder.KEY_DB_ID, String.valueOf(dbId))
.get();
ContentReader reader = this.getReader(contentUrl);
if (!reader.exists())
{
return null;
}
try (InputStream contentInputStream = reader.getContentInputStream();
InputStream gzip = new GZIPInputStream(contentInputStream))
{
return (SolrInputDocument) new JavaBinCodec(resolver).unmarshal(gzip);
}
catch (Exception exception)
{
// Don't fail for this
LOGGER.warn("Failed to get doc from store using URL: " + contentUrl, exception);
return null;
}
}
@Override
public long getLastCommittedVersion()
{
return currentAccessMode.getLastCommittedVersion();
}
@Override
public void setLastCommittedVersion(long version)
{
currentAccessMode.setLastCommittedVersion(version);
}
/**
* Returns the absolute path of the content store root folder.
*
* @return the absolute path of the content store root folder.
*/
public String getRootLocation()
{
return root;
}
public boolean exists(String contentUrl)
{
File file = getFileFromUrl(contentUrl);
return file.exists();
}
@Override
public void storeDocOnSolrContentStore(String tenant, long dbId, SolrInputDocument doc)
{
currentAccessMode.storeDocOnSolrContentStore(tenant, dbId, doc);
}
/**
* Store {@link SolrInputDocument} in to Alfresco solr content store.
*
* @param nodeMetaData the incoming node metadata.
* @param doc the document itself.
*/
@Override
public void storeDocOnSolrContentStore(NodeMetaData nodeMetaData, SolrInputDocument doc)
{
currentAccessMode.storeDocOnSolrContentStore(nodeMetaData, doc);
}
/**
* Removes {@link SolrInputDocument} from Alfresco solr content store.
*
* @param nodeMetaData the incoming node metadata.
*/
@Override
public void removeDocFromContentStore(NodeMetaData nodeMetaData)
{
currentAccessMode.removeDocFromContentStore(nodeMetaData);
}
@Override
public void flushChangeSet() throws IOException
{
currentAccessMode.flushChangeSet();
}
@Override
public void close() throws IOException
{
currentAccessMode.close();
}
@Override
public void switchOnReadWriteMode()
{
currentAccessMode = readWrite;
}
@Override
public void switchOnReadOnlyMode()
{
currentAccessMode = readOnly;
}
/**
* Assuming the input file belongs to the content store, it returns the corresponding relative path.
*
* @param file the content store file.
* @return the relative file path.
*/
private String relativePath(File file)
{
return file.getAbsolutePath().replace(root, "");
}
/**
* Convert a content URL into a File, whether it exists or not
*/
private File getFileFromUrl(String contentUrl)
{
return new File(contentUrl.replace(SolrContentUrlBuilder.SOLR_PROTOCOL_PREFIX, root + "/"));
}
private ContentReader getReader(String contentUrl)
{
File file = getFileFromUrl(contentUrl);
return new SolrFileContentReader(file, contentUrl);
}
/**
* Enables/disables the content store access mode.
* The term "toggles" is just for indicating that this method will be called several times (one for each registered
* core). The underlying FSM makes sure this call will be idempotent so in case of read/write content store the
* data structure used for maintaining the content store versioning will be initialised only once, even if this
* method is called repeatedly.
*
* @param enableReadOnlyMode a flag indicating if the requesting core requires a readOnly (true) or readWrite (false) content store.
*/
public synchronized void toggleReadOnlyMode(boolean enableReadOnlyMode)
{
if (enableReadOnlyMode)
{
currentAccessMode.switchOnReadOnlyMode();
}
else
{
currentAccessMode.switchOnReadWriteMode();
}
}
}
search-services/alfresco-search/src/main/java/org/alfresco/solr/content/SolrContentUrlBuilder.java
-213
View File
@@ -1,213 +0,0 @@
/*
* Copyright (C) 2005-2019 Alfresco Software Limited.
*
* This file is part of Alfresco
*
* Alfresco is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Alfresco is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
*/
package org.alfresco.solr.content;
import java.nio.charset.StandardCharsets;
import java.util.Map;
import java.util.TreeMap;
import java.util.zip.CRC32;
import org.alfresco.repo.content.ContentContext;
import org.alfresco.repo.content.ContentStore;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Utility class that wraps up the creation of SOLR content URLs given arbitrary,
* string-based metadata.
* <p/>
* The URL is constructed from a 19 digit number (zero-padded long), which is built from
* the ACL ID, the DB ID or a CRC32 of the provided metadata, and a numerical version starting with "000".<br/>
* For example, the DB ID "4775808" will generate
* <tt>"someprefix://<tenant>/db/4775/808.gz"</tt><br/>
* The
*
* @author Derek Hulley
* @since 5.0
*/
public class SolrContentUrlBuilder
{
private static final String SOLR_PROTOCOL = "solr";
static final String SOLR_PROTOCOL_PREFIX = SOLR_PROTOCOL + ContentStore.PROTOCOL_DELIMITER;
static final String FILE_EXTENSION = ".gz";
static final String KEY_TENANT = "tenant";
static final String KEY_DB_ID = "dbId";
static final String KEY_ACL_ID = "aclId";
protected final static Logger logger = LoggerFactory.getLogger(SolrContentUrlBuilder.class);
/** Metadata ordered by key */
private final TreeMap<String, String> metadata;
/**
* Protected constructor used by {@link SolrContentUrlBuilder#start()}
*/
private SolrContentUrlBuilder()
{
this.metadata = new TreeMap<>();
}
/**
* Factory method to start building the SOLR content URL.
*
* @return an instance of the builder
*/
public static SolrContentUrlBuilder start()
{
return new SolrContentUrlBuilder();
}
/**
* Add some metadata to the URL generator. The order in which metadata is added is irrelevant.
* Note that there are specific keys that are commonly used and, if provided, may not be null or empty.
*
* <ul>
* <li><b>{@link #KEY_TENANT}:</b> The name of the tenant or 'default' if missing.</li>
* <li><b>{@link #KEY_DB_ID}:</b> The database ID.</li>
* <li><b>{@link #KEY_ACL_ID}:</b> The ACL ID.</li>
* </ul>
*
* @param key an arbitrary metadata key (never <tt>null</tt>>
* @param value some metadata value (<tt>null</tt> is supported)
* @return this builder for more building
*
* @throws IllegalArgumentException if the key is null
* @throws IllegalStateException if the key has been used already
*/
public SolrContentUrlBuilder add(String key, String value)
{
if (key == null)
{
throw new IllegalArgumentException("The metadata 'key' may not be null.");
}
String previous = metadata.put(key, value);
if (previous != null)
{
throw new IllegalStateException("The metadata key, '" + key + "', has already been used.");
}
// Check well-known keys
if (key.equals(KEY_TENANT) || key.equals(KEY_DB_ID) || key.equals(KEY_ACL_ID))
{
if (value == null || value.length() == 0)
{
throw new IllegalArgumentException("Invalid value for key '" + key + "': " + value);
}
}
// Store metadata
metadata.put(key, value);
// Done
if (logger.isDebugEnabled())
{
logger.debug("Appending SOLR metadata: " + key + " - " + value);
}
return this;
}
/**
* Get the final content URL using the {@link #add(String, String) supplied metadata}.
*
* @return the SOLR content URL
* @throws IllegalStateException if no metadata has been added
*/
public synchronized String get()
{
if (metadata.size() == 0)
{
throw new IllegalStateException("No metadata added. Usage add.");
}
StringBuilder sb = new StringBuilder(72);
// Is there a 'tenant'?
String tenant = metadata.get(KEY_TENANT);
if (tenant == null) // We checked it for length before
{
tenant = "default";
}
sb.append(SOLR_PROTOCOL_PREFIX).append(tenant).append("/");
// Build a numeric value using the CRC and special IDs, if available
StringBuilder numSb = new StringBuilder(52);
if (metadata.containsKey(KEY_DB_ID))
{
sb.append("db/");
// We have a unique DB ID, which can be used by itself
numSb.append(metadata.get(KEY_DB_ID));
}
else if (metadata.containsKey(KEY_ACL_ID))
{
sb.append("acl/");
// We have a unique ACL ID, which can be used completely
numSb.append(metadata.get(KEY_ACL_ID));
}
else
{
sb.append("misc/");
// Calculate the CRC
CRC32 crc = new CRC32();
for (Map.Entry<String, String> entry : metadata.entrySet())
{
// This is ordered, so just add each entry as "key = value".
// DO NOT USE entry.toString() because the format is not a contract
// and we have to have the same string for the same metadata
String entryStr = entry.getKey() + "=" + entry.getValue() + "; ";
crc.update(entryStr.getBytes(StandardCharsets.UTF_8));
}
numSb.append(crc.getValue());
}
String numStr = numSb.toString();
// We use 3 characters at a time from the CRC, which gives up to 999 entries per path element of the URL
int pathCharCount = 0;
for (int i = 0; i < numStr.length(); i++)
{
// If we have 4 chars in a path part (and we have more chars) then we add a separator
if (pathCharCount == 4)
{
sb.append("/");
pathCharCount = 0;
}
// Append the char
sb.append(numStr.charAt(i));
pathCharCount++;
}
// We always have a numeric value ending, never '/'. That's it. Just give it an extension.
sb.append(FILE_EXTENSION);
String url = sb.toString();
// Done
if (logger.isDebugEnabled())
{
logger.debug("Converted SOLR metadata to URL: " + url + " -- " + metadata.toString());
}
return url;
}
/**
* Helper method to retrieve a {@link ContentContext} constructed using the final {@link #get()} url.
*/
ContentContext getContentContext()
{
String url = get();
return new ContentContext(null, url);
}
}
search-services/alfresco-search/src/main/java/org/alfresco/solr/content/SolrFileContentReader.java
-265
View File
@@ -1,265 +0,0 @@
/*
* Copyright (C) 2005-2019 Alfresco Software Limited.
*
* This file is part of Alfresco
*
* Alfresco is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Alfresco is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
*/
package org.alfresco.solr.content;
import java.io.BufferedInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.channels.FileChannel;
import java.nio.channels.ReadableByteChannel;
import java.util.Locale;
import org.alfresco.service.cmr.repository.ContentData;
import org.alfresco.service.cmr.repository.ContentIOException;
import org.alfresco.service.cmr.repository.ContentReader;
import org.alfresco.service.cmr.repository.ContentStreamListener;
import org.apache.commons.io.FileUtils;
import org.springframework.util.FileCopyUtils;
/**
* Bare-bones implementation of the reader for SOLR purposes
*
* @author Derek Hulley
* @since 5.0
*/
class SolrFileContentReader implements ContentReader
{
private final File file;
private final String contentUrl;
/**
* @param file the file to write to
* @param contentUrl the content URL for information purposes
*/
SolrFileContentReader(File file, String contentUrl)
{
this.file = file;
this.contentUrl = contentUrl;
}
@Override
public String toString()
{
return "SolrFileContentReader [file=" + file + "]";
}
@Override
public long getSize()
{
if (file.exists())
{
return file.length();
}
else
{
return 0L;
}
}
@Override
public final ContentReader getReader() throws ContentIOException
{
throw new UnsupportedOperationException();
}
@Override
public synchronized final boolean isClosed()
{
throw new UnsupportedOperationException();
}
public synchronized boolean isChannelOpen()
{
throw new UnsupportedOperationException();
}
@Override
public FileChannel getFileChannel() throws ContentIOException
{
throw new UnsupportedOperationException();
}
@Override
public boolean exists()
{
return file.exists();
}
@Override
public ReadableByteChannel getReadableChannel() throws ContentIOException
{
throw new UnsupportedOperationException("Auto-created method not implemented.");
}
@Override
public synchronized InputStream getContentInputStream() throws ContentIOException
{
if (!file.exists())
{
throw new IllegalStateException("The file does not exist: " + file);
}
try
{
// done
return new BufferedInputStream(new FileInputStream(file));
}
catch (Throwable e)
{
throw new ContentIOException("Failed to open stream onto file: " + file, e);
}
}
@Override
public synchronized void getContent(OutputStream os) throws ContentIOException
{
if (!file.exists())
{
throw new IllegalStateException("The file does not exist: " + file);
}
try
{
FileUtils.copyFile(file, os);
}
catch (Throwable e)
{
throw new ContentIOException("Failed to copy stream onto file: " + file, e);
}
}
@Override
public synchronized void getContent(File targetFile) throws ContentIOException
{
if (!this.file.exists())
{
throw new IllegalStateException("The file does not exist: " + this.file);
}
else if (targetFile.exists())
{
throw new IllegalStateException("The target file already exists: " + targetFile);
}
try
{
FileUtils.copyFile(this.file, targetFile, false);
}
catch (Throwable e)
{
throw new ContentIOException("Failed to copy stream onto file: " + targetFile, e);
}
}
@Override
public String getContentString(int length) throws ContentIOException
{
String str = getContentString();
if (str.length() > length)
{
return str.substring(0, length - 1);
}
else
{
return str;
}
}
@Override
public final String getContentString() throws ContentIOException
{
try
{
// read from the stream into a byte[]
InputStream is = getContentInputStream();
ByteArrayOutputStream os = new ByteArrayOutputStream();
FileCopyUtils.copy(is, os); // both streams are closed
byte[] bytes = os.toByteArray();
String encoding = "UTF-8";
return new String(bytes, encoding);
}
catch (IOException e)
{
throw new ContentIOException("Failed to copy content to string: \n" +
" accessor: " + this,
e);
}
}
@Override
public long getLastModified()
{
return file.lastModified();
}
@Override
public void addListener(ContentStreamListener listener)
{
throw new UnsupportedOperationException("Auto-created method not implemented.");
}
@Override
public ContentData getContentData()
{
throw new UnsupportedOperationException("Auto-created method not implemented.");
}
@Override
public String getContentUrl()
{
return contentUrl;
}
@Override
public String getMimetype()
{
throw new UnsupportedOperationException("Auto-created method not implemented.");
}
@Override
public void setMimetype(String mimetype)
{
throw new UnsupportedOperationException("Auto-created method not implemented.");
}
@Override
public String getEncoding()
{
throw new UnsupportedOperationException("Auto-created method not implemented.");
}
@Override
public void setEncoding(String encoding)
{
throw new UnsupportedOperationException("Auto-created method not implemented.");
}
@Override
public Locale getLocale()
{
throw new UnsupportedOperationException("Auto-created method not implemented.");
}
@Override
public void setLocale(Locale locale)
{
throw new UnsupportedOperationException("Auto-created method not implemented.");
}
}
search-services/alfresco-search/src/main/java/org/alfresco/solr/content/SolrFileContentWriter.java
-277
View File
@@ -1,277 +0,0 @@
/*
* Copyright (C) 2005-2019 Alfresco Software Limited.
*
* This file is part of Alfresco
*
* Alfresco is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Alfresco is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
*/
package org.alfresco.solr.content;
import java.io.BufferedOutputStream;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.channels.FileChannel;
import java.nio.channels.WritableByteChannel;
import java.util.Locale;
import org.alfresco.service.cmr.repository.ContentData;
import org.alfresco.service.cmr.repository.ContentIOException;
import org.alfresco.service.cmr.repository.ContentReader;
import org.alfresco.service.cmr.repository.ContentStreamListener;
import org.alfresco.service.cmr.repository.ContentWriter;
import org.apache.commons.io.FileUtils;
/**
* Bare-bones implementation of the writer for SOLR purposes
*
* @author Derek Hulley
* @since 5.0
*/
class SolrFileContentWriter implements ContentWriter
{
private final File file;
private final String contentUrl;
private boolean written;
/**
* @param file the file to write to
* @param contentUrl the content URL for information purposes
*/
SolrFileContentWriter(File file, String contentUrl)
{
this.file = file;
this.contentUrl = contentUrl;
}
@Override
public String toString()
{
return "SolrFileContentWriter [file=" + file + "]";
}
@Override
public long getSize()
{
if (file.exists())
{
return file.length();
}
else
{
return 0L;
}
}
@Override
public final ContentReader getReader() throws ContentIOException
{
throw new UnsupportedOperationException();
}
@Override
public synchronized final boolean isClosed()
{
throw new UnsupportedOperationException();
}
public synchronized boolean isChannelOpen()
{
throw new UnsupportedOperationException();
}
@Override
public synchronized final WritableByteChannel getWritableChannel() throws ContentIOException
{
throw new UnsupportedOperationException();
}
@Override
public FileChannel getFileChannel(boolean truncate) throws ContentIOException
{
throw new UnsupportedOperationException();
}
@Override
public synchronized OutputStream getContentOutputStream() throws ContentIOException
{
if (written)
{
throw new IllegalStateException("The writer has already been used: " + file);
}
else if (file.exists())
{
throw new IllegalStateException("The file already exists: " + file);
}
try
{
OutputStream is = new BufferedOutputStream(FileUtils.openOutputStream(file));
written = true;
// done
return is;
}
catch (Throwable e)
{
throw new ContentIOException("Failed to open stream onto file: " + file, e);
}
}
@Override
public void putContent(ContentReader reader) throws ContentIOException
{
throw new UnsupportedOperationException();
}
@Override
public synchronized void putContent(InputStream is) throws ContentIOException
{
if (written)
{
throw new IllegalStateException("The writer has already been used: " + file);
}
else if (file.exists())
{
throw new IllegalStateException("The file already exists: " + file);
}
try
{
FileUtils.copyInputStreamToFile(is, file);
written = true;
}
catch (Throwable e)
{
throw new ContentIOException("Failed to copy stream onto file: " + file, e);
}
}
@Override
public synchronized void putContent(File sourceFile) throws ContentIOException
{
if (written)
{
throw new IllegalStateException("The writer has already been used: " + this.file);
}
else if (this.file.exists())
{
throw new IllegalStateException("The file already exists: " + this.file);
}
else if (!sourceFile.exists())
{
throw new IllegalStateException("The source file does not exist: " + sourceFile);
}
try
{
FileUtils.copyFile(sourceFile, this.file, false);
written = true;
}
catch (Throwable e)
{
throw new ContentIOException("Failed to copy file onto file: " + sourceFile, e);
}
}
@Override
public synchronized void putContent(String content) throws ContentIOException
{
try
{
// attempt to use the correct encoding
String encoding = "UTF-8";
byte[] bytes = content.getBytes(encoding);
// get the stream
ByteArrayInputStream is = new ByteArrayInputStream(bytes);
putContent(is);
// done
}
catch (IOException e)
{
throw new ContentIOException("Failed to copy content from string: \n" +
" writer: " + this +
" content length: " + content.length(),
e);
}
}
@Override
public void guessEncoding()
{
throw new UnsupportedOperationException();
}
@Override
public void guessMimetype(String filename)
{
throw new UnsupportedOperationException();
}
@Override
public void addListener(ContentStreamListener listener)
{
throw new UnsupportedOperationException("Auto-created method not implemented.");
}
@Override
public ContentData getContentData()
{
throw new UnsupportedOperationException("Auto-created method not implemented.");
}
@Override
public String getContentUrl()
{
return contentUrl;
}
@Override
public String getMimetype()
{
throw new UnsupportedOperationException("Auto-created method not implemented.");
}
@Override
public void setMimetype(String mimetype)
{
throw new UnsupportedOperationException("Auto-created method not implemented.");
}
@Override
public String getEncoding()
{
throw new UnsupportedOperationException("Auto-created method not implemented.");
}
@Override
public void setEncoding(String encoding)
{
throw new UnsupportedOperationException("Auto-created method not implemented.");
}
@Override
public Locale getLocale()
{
throw new UnsupportedOperationException("Auto-created method not implemented.");
}
@Override
public void setLocale(Locale locale)
{
throw new UnsupportedOperationException("Auto-created method not implemented.");
}
}
search-services/alfresco-search/src/main/java/org/alfresco/solr/content/package-info.java
-24
View File
@@ -1,24 +0,0 @@
/*
* Copyright (C) 2005-2019 Alfresco Software Limited.
*
* This file is part of Alfresco
*
* Alfresco is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Alfresco is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
*/
/**
* The package contains all components that manage the Solr ContentStore.
* Note that in SearchServices 2.0 the whole package will be deprecated/removed because the content store will be
* replaced by the built-in Solr storage capability (i.e. stored fields).
*/
package org.alfresco.solr.content;
search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/AlfrescoIndexFetcher.java
-2762
View File
File diff suppressed because it is too large Load Diff
search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/AlfrescoReplicationHandler.java
-2377
View File
File diff suppressed because it is too large Load Diff
search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/OldBackupDirectory.java
-102
View File
@@ -1,102 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*
* Modification copyright (C) 2005-2019 Alfresco Software Limited.
*
* This file is part of Alfresco
*
* Alfresco is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Alfresco is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with Alfresco. If not, see <http://www.gnu.org/licenses/>
*
*/
package org.alfresco.solr.handler;
import org.apache.solr.handler.SnapShooter;
import java.net.URI;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.Locale;
import java.util.Objects;
import java.util.Optional;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
class OldBackupDirectory implements Comparable<OldBackupDirectory>
{
private static final Pattern dirNamePattern = Pattern.compile("^snapshot[.](.*)$");
private URI basePath;
private String dirName;
private Optional<Date> timestamp = Optional.empty();
OldBackupDirectory(URI basePath, String dirName)
{
this.dirName = Objects.requireNonNull(dirName);
this.basePath = Objects.requireNonNull(basePath);
Matcher m = dirNamePattern.matcher(dirName);
if (m.find())
{
try
{
this.timestamp = Optional.of(new SimpleDateFormat(SnapShooter.DATE_FMT, Locale.ROOT).parse(m.group(1)));
}
catch (ParseException e)
{
this.timestamp = Optional.empty();
}
}
}
public URI getPath()
{
return this.basePath.resolve(dirName);
}
String getDirName()
{
return dirName;
}
public Optional<Date> getTimestamp()
{
return timestamp;
}
@Override
public int compareTo(OldBackupDirectory that)
{
if (this.timestamp.isPresent() && that.timestamp.isPresent())
{
return that.timestamp.get().compareTo(this.timestamp.get());
}
// Use absolute value of path in case the time-stamp is missing on either side.
return that.getPath().compareTo(this.getPath());
}
}
search-services/alfresco-search/src/main/java/org/alfresco/solr/handler/SnapShooter.java
-377
View File
@@ -1,377 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*
* Modification copyright (C) 2005-2019 Alfresco Software Limited.
*
* This file is part of Alfresco
*
* Alfresco is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Alfresco is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with Alfresco. If not, see <http://www.gnu.org/licenses/>
*
*/
package org.alfresco.solr.handler;
import org.apache.lucene.index.IndexCommit;
import org.apache.lucene.store.Directory;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.core.DirectoryFactory.DirContext;
import org.apache.solr.core.IndexDeletionPolicyWrapper;
import org.apache.solr.core.SolrCore;
import org.apache.solr.core.backup.repository.BackupRepository;
import org.apache.solr.core.backup.repository.BackupRepository.PathType;
import org.apache.solr.core.backup.repository.LocalFileSystemRepository;
import org.apache.solr.core.snapshots.SolrSnapshotMetaDataManager;
import org.apache.solr.search.SolrIndexSearcher;
import org.apache.solr.util.RefCounted;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.net.URI;
import java.nio.file.Paths;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
import java.util.List;
import java.util.Locale;
import java.util.Objects;
import java.util.Optional;
import java.util.function.Consumer;
/**
* <p> Provides functionality equivalent to the snapshooter script </p>
* This is no longer used in standard replication.
*
* @since solr 1.4
*/
public class SnapShooter
{
private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
private SolrCore solrCore;
private String snapshotName = null;
private String directoryName = null;
private URI baseSnapDirPath = null;
private URI snapshotDirPath = null;
private BackupRepository backupRepo = null;
private String commitName; // can be null
@Deprecated
public SnapShooter(SolrCore core, String location, String snapshotName)
{
String snapDirStr;
// Note - This logic is only applicable to the usecase where a shared file-system is exposed via
// local file-system interface (primarily for backwards compatibility). For other use-cases, users
// will be required to specify "location" where the backup should be stored.
if (location == null)
{
snapDirStr = core.getDataDir();
}
else
{
snapDirStr = core.getCoreDescriptor().getInstanceDir().resolve(location).normalize().toString();
}
initialize(new LocalFileSystemRepository(), core, Paths.get(snapDirStr).toUri(), snapshotName, null);
}
SnapShooter(BackupRepository backupRepo, SolrCore core, URI location, String snapshotName, String commitName)
{
initialize(backupRepo, core, location, snapshotName, commitName);
}
private void initialize(BackupRepository backupRepo, SolrCore core, URI location, String snapshotName, String commitName)
{
this.solrCore = Objects.requireNonNull(core);
this.backupRepo = Objects.requireNonNull(backupRepo);
this.baseSnapDirPath = location;
this.snapshotName = snapshotName;
if (snapshotName != null)
{
directoryName = "snapshot." + snapshotName;
}
else
{
SimpleDateFormat fmt = new SimpleDateFormat(DATE_FMT, Locale.ROOT);
directoryName = "snapshot." + fmt.format(new Date());
}
this.snapshotDirPath = backupRepo.resolve(location, directoryName);
this.commitName = commitName;
}
public BackupRepository getBackupRepository()
{
return backupRepo;
}
/**
* Gets the parent directory of the snapshots. This is the {@code location}
* given in the constructor.
*/
public URI getLocation()
{
return this.baseSnapDirPath;
}
void validateDeleteSnapshot()
{
Objects.requireNonNull(this.snapshotName);
boolean dirFound = false;
String[] paths;
try
{
paths = backupRepo.listAll(baseSnapDirPath);
for (String path : paths)
{
if (path.equals(this.directoryName)
&& backupRepo.getPathType(baseSnapDirPath.resolve(path)) == PathType.DIRECTORY)
{
dirFound = true;
break;
}
}
if (!dirFound)
{
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
"Snapshot " + snapshotName + " cannot be found in directory: " + baseSnapDirPath);
}
}
catch (IOException e)
{
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
"Unable to find snapshot " + snapshotName + " in directory: " + baseSnapDirPath, e);
}
}
void deleteSnapAsync(final AlfrescoReplicationHandler alfrescoReplicationHandler)
{
new Thread(() -> deleteNamedSnapshot(alfrescoReplicationHandler)).start();
}
void validateCreateSnapshot() throws IOException
{
// Note - Removed the current behavior of creating the directory hierarchy.
// Do we really need to provide this support?
if (!backupRepo.exists(baseSnapDirPath))
{
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
" Directory does not exist: " + snapshotDirPath);
}
if (backupRepo.exists(snapshotDirPath))
{
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
"Snapshot directory already exists: " + snapshotDirPath);
}
}
public NamedList createSnapshot() throws Exception
{
RefCounted<SolrIndexSearcher> searcher = solrCore.getSearcher();
try
{
if (commitName != null)
{
SolrSnapshotMetaDataManager snapshotMgr = solrCore.getSnapshotMetaDataManager();
Optional<IndexCommit> commit = snapshotMgr.getIndexCommitByName(commitName);
if (commit.isPresent())
{
return createSnapshot(commit.get());
}
throw new SolrException(ErrorCode.SERVER_ERROR,
"Unable to find an index commit with name " + commitName + " for core " + solrCore.getName());
}
else
{
//TODO should we try solrCore.getDeletionPolicy().getLatestCommit() first?
IndexDeletionPolicyWrapper deletionPolicy = solrCore.getDeletionPolicy();
IndexCommit indexCommit = searcher.get().getIndexReader().getIndexCommit();
deletionPolicy.saveCommitPoint(indexCommit.getGeneration());
try
{
return createSnapshot(indexCommit);
}
finally
{
deletionPolicy.releaseCommitPoint(indexCommit.getGeneration());
}
}
}
finally
{
searcher.decref();
}
}
void createSnapAsync(final IndexCommit indexCommit, final int numberToKeep, Consumer<NamedList> result)
{
solrCore.getDeletionPolicy().saveCommitPoint(indexCommit.getGeneration());
new Thread(() -> {
try
{
result.accept(createSnapshot(indexCommit));
}
catch (Exception e)
{
LOG.error("Exception while creating snapshot", e);
NamedList snapShootDetails = new NamedList<>();
snapShootDetails.add("snapShootException", e.getMessage());
result.accept(snapShootDetails);
}
finally
{
solrCore.getDeletionPolicy().releaseCommitPoint(indexCommit.getGeneration());
}
if (snapshotName == null)
{
try
{
deleteOldBackups(numberToKeep);
}
catch (IOException e)
{
LOG.warn("Unable to delete old snapshots ", e);
}
}
}).start();
}
// note: remember to reserve the indexCommit first so it won't get deleted concurrently
private NamedList createSnapshot(final IndexCommit indexCommit) throws Exception
{
LOG.info("Creating backup snapshot " + (snapshotName == null ? "<not named>" : snapshotName) + " at "
+ baseSnapDirPath);
boolean success = false;
try
{
NamedList<Object> details = new NamedList<>();
details.add("startTime", new Date().toString());//bad; should be Instant.now().toString()
Collection<String> files = indexCommit.getFileNames();
Directory dir = solrCore.getDirectoryFactory()
.get(solrCore.getIndexDir(), DirContext.DEFAULT, solrCore.getSolrConfig().indexConfig.lockType);
try
{
for (String fileName : files)
{
backupRepo.copyFileFrom(dir, fileName, snapshotDirPath);
}
}
finally
{
solrCore.getDirectoryFactory().release(dir);
}
details.add("fileCount", files.size());
details.add("status", "success");
details.add("snapshotCompletedAt", new Date().toString());//bad; should be Instant.now().toString()
details.add("snapshotName", snapshotName);
LOG.info("Done creating backup snapshot: " + (snapshotName == null ? "<not named>" : snapshotName) + " at "
+ baseSnapDirPath);
success = true;
return details;
}
finally
{
if (!success)
{
try
{
backupRepo.deleteDirectory(snapshotDirPath);
}
catch (Exception excDuringDelete)
{
LOG.warn("Failed to delete " + snapshotDirPath + " after snapshot creation failed due to: "
+ excDuringDelete);
}
}
}
}
private void deleteOldBackups(int numberToKeep) throws IOException
{
String[] paths = backupRepo.listAll(baseSnapDirPath);
List<OldBackupDirectory> dirs = new ArrayList<>();
for (String f : paths)
{
if (backupRepo.getPathType(baseSnapDirPath.resolve(f)) == PathType.DIRECTORY)
{
OldBackupDirectory obd = new OldBackupDirectory(baseSnapDirPath, f);
if (obd.getTimestamp().isPresent())
{
dirs.add(obd);
}
}
}
if (numberToKeep > dirs.size() - 1)
{
return;
}
Collections.sort(dirs);
int i = 1;
for (OldBackupDirectory dir : dirs)
{
if (i++ > numberToKeep)
{
backupRepo.deleteDirectory(dir.getPath());
}
}
}
protected void deleteNamedSnapshot(AlfrescoReplicationHandler alfrescoReplicationHandler)
{
LOG.info("Deleting snapshot: " + snapshotName);
NamedList<Object> details = new NamedList<>();
try
{
URI path = baseSnapDirPath.resolve("snapshot." + snapshotName);
backupRepo.deleteDirectory(path);
details.add("status", "success");
details.add("snapshotDeletedAt", new Date().toString());
}
catch (IOException e)
{
details.add("status", "Unable to delete snapshot: " + snapshotName);
LOG.warn("Unable to delete snapshot: " + snapshotName, e);
}
alfrescoReplicationHandler.snapShootDetails = details;
}
private static final String DATE_FMT = "yyyyMMddHHmmssSSS";
}
search-services/alfresco-search/src/main/java/org/alfresco/solr/lifecycle/SolrCoreLoadListener.java
+28 -20
View File
@@ -21,6 +21,15 @@ package org.alfresco.solr.lifecycle;
import static java.util.Arrays.asList;
import static java.util.Optional.ofNullable;
import static org.alfresco.solr.SolrInformationServer.CASCADE_TRACKER_ENABLED;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Properties;
import java.util.function.Function;
import java.util.function.Predicate;
import org.alfresco.opencmis.dictionary.CMISStrictDictionaryService;
import org.alfresco.solr.AlfrescoCoreAdminHandler;
import org.alfresco.solr.AlfrescoSolrDataModel;
@@ -28,7 +37,6 @@ import org.alfresco.solr.SolrInformationServer;
import org.alfresco.solr.SolrKeyResourceLoader;
import org.alfresco.solr.client.SOLRAPIClient;
import org.alfresco.solr.client.SOLRAPIClientFactory;
import org.alfresco.solr.content.SolrContentStore;
import org.alfresco.solr.tracker.AclTracker;
import org.alfresco.solr.tracker.CascadeTracker;
import org.alfresco.solr.tracker.CommitTracker;
@@ -54,13 +62,6 @@ import org.apache.solr.search.SolrIndexSearcher;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Properties;
import java.util.function.Function;
import java.util.function.Predicate;
/**
* Listeners for *FIRST SEARCHER* events in order to prepare and register the SolrContentStore and the Tracking Subsystem.
*
@@ -113,8 +114,7 @@ public class SolrCoreLoadListener extends AbstractSolrEventListener
AlfrescoSolrDataModel.getInstance().getDictionaryService(CMISStrictDictionaryService.DEFAULT),
AlfrescoSolrDataModel.getInstance().getNamespaceDAO());
SolrContentStore contentStore = admin.getSolrContentStore();
SolrInformationServer informationServer = new SolrInformationServer(admin, core, repositoryClient, contentStore);
SolrInformationServer informationServer = new SolrInformationServer(admin, core, repositoryClient);
coreProperties.putAll(informationServer.getProps());
admin.getInformationServers().put(core.getName(), informationServer);
@@ -162,7 +162,6 @@ public class SolrCoreLoadListener extends AbstractSolrEventListener
boolean trackersHaveBeenEnabled = Boolean.parseBoolean(coreProperties.getProperty("enable.alfresco.tracking", "true"));
boolean owningCoreIsSlave = isSlaveModeEnabledFor(core);
contentStore.toggleReadOnlyMode(owningCoreIsSlave);
if (trackerRegistry.hasTrackersForCore(core.getName()))
{
@@ -251,25 +250,33 @@ public class SolrCoreLoadListener extends AbstractSolrEventListener
MetadataTracker metadataTracker =
registerAndSchedule(
new MetadataTracker(true, props, repositoryClient, core.getName(), srv),
new MetadataTracker(true, props, repositoryClient, core.getName(), srv, true),
core,
props,
trackerRegistry,
scheduler);
CascadeTracker cascadeTracker =
registerAndSchedule(
new CascadeTracker(props, repositoryClient, core.getName(), srv),
core,
props,
trackerRegistry,
scheduler);
List<Tracker> trackers = new ArrayList<>();
String cascadeTrackerEnabledProp = ofNullable((String) props.get(CASCADE_TRACKER_ENABLED)).orElse("true");
if (Boolean.valueOf(cascadeTrackerEnabledProp))
{
CascadeTracker cascadeTracker =
registerAndSchedule(
new CascadeTracker(props, repositoryClient, core.getName(), srv),
core,
props,
trackerRegistry,
scheduler);
trackers.add(cascadeTracker);
}
//The CommitTracker will acquire these locks in order
//The ContentTracker will likely have the longest runs so put it first to ensure the MetadataTracker is not paused while
//waiting for the ContentTracker to release it's lock.
//The aclTracker will likely have the shortest runs so put it last.
return asList(cascadeTracker, contentTracker, metadataTracker, aclTracker);
trackers.addAll(asList(contentTracker, metadataTracker, aclTracker));
return trackers;
}
/**
@@ -411,6 +418,7 @@ public class SolrCoreLoadListener extends AbstractSolrEventListener
* @param core the hosting {@link SolrCore} instance.
* @return true if the content store must be set in read only mode, false otherwise.
*/
@SuppressWarnings("rawtypes")
boolean isSlaveModeEnabledFor(SolrCore core)
{
Predicate<PluginInfo> onlyReplicationHandler =
search-services/alfresco-search/src/main/java/org/alfresco/solr/query/MimetypeGroupingQParserPlugin.java
+6 -8
View File
@@ -35,12 +35,16 @@ import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.search.QParser;
import org.apache.solr.search.QParserPlugin;
import org.apache.solr.search.SyntaxError;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* @author Andy
*/
public class MimetypeGroupingQParserPlugin extends QParserPlugin
{
protected static final Logger LOGGER = LoggerFactory.getLogger(MimetypeGroupingQParserPlugin.class);
private static HashMap<String, String> mappings = new HashMap<>();
private static HashMap<String, ArrayList<String>> reverseMappings = new HashMap<>();
@@ -86,15 +90,9 @@ public class MimetypeGroupingQParserPlugin extends QParserPlugin
}
}
}
catch (FileNotFoundException e1)
catch (Exception exception)
{
// TODO Auto-generated catch block
e1.printStackTrace();
}
catch (IOException e1)
{
// TODO Auto-generated catch block
e1.printStackTrace();
LOGGER.error("Exception during the MimetypeGroupingQParserPlugin. See the stacktrace below for further details.", exception);
}
}
search-services/alfresco-search/src/main/java/org/alfresco/solr/query/Solr4QueryParser.java
+66 -90
View File
@@ -18,28 +18,7 @@
*/
package org.alfresco.solr.query;
import java.io.IOException;
import java.io.StringReader;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.Date;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Locale;
import java.util.Set;
import java.util.TimeZone;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import edu.umd.cs.findbugs.annotations.SuppressWarnings;
import org.alfresco.model.ContentModel;
import org.alfresco.repo.dictionary.IndexTokenisationMode;
import org.alfresco.repo.search.MLAnalysisMode;
@@ -64,15 +43,12 @@ import org.alfresco.service.cmr.search.SearchParameters;
import org.alfresco.service.namespace.NamespacePrefixResolver;
import org.alfresco.service.namespace.QName;
import org.alfresco.solr.AlfrescoAnalyzerWrapper;
import org.alfresco.solr.AlfrescoCoreAdminHandler;
import org.alfresco.solr.AlfrescoSolrDataModel;
import org.alfresco.solr.AlfrescoSolrDataModel.ContentFieldType;
import org.alfresco.solr.AlfrescoSolrDataModel.FieldInstance;
import org.alfresco.solr.AlfrescoSolrDataModel.FieldUse;
import org.alfresco.solr.AlfrescoSolrDataModel.IndexedField;
import org.alfresco.solr.SolrInformationServer;
import org.alfresco.solr.component.FingerPrintComponent;
import org.alfresco.solr.content.SolrContentStore;
import org.alfresco.solr.utils.ThrowingFunction;
import org.alfresco.util.CachingDateFormat;
import org.alfresco.util.Pair;
@@ -103,6 +79,7 @@ import org.apache.lucene.search.BooleanClause.Occur;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.BooleanQuery.Builder;
import org.apache.lucene.search.ConstantScoreQuery;
import org.apache.lucene.search.LegacyNumericRangeQuery;
import org.apache.lucene.search.MatchAllDocsQuery;
import org.apache.lucene.search.MultiTermQuery;
import org.apache.lucene.search.Query;
@@ -125,16 +102,14 @@ import org.apache.solr.client.solrj.impl.HttpClientUtil;
import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.apache.solr.client.solrj.request.QueryRequest;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.SolrInputField;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.core.CoreContainer;
import org.apache.solr.core.SolrCore;
import org.apache.solr.handler.component.HttpShardHandlerFactory;
import org.apache.solr.handler.component.ShardHandlerFactory;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.response.DocsStreamer;
import org.apache.solr.schema.IndexSchema;
import org.apache.solr.schema.SchemaField;
import org.apache.solr.search.SolrIndexSearcher;
@@ -144,7 +119,28 @@ import org.jaxen.saxpath.base.XPathReader;
import org.json.JSONObject;
import org.springframework.extensions.surf.util.I18NUtil;
import edu.umd.cs.findbugs.annotations.SuppressWarnings;
import java.io.IOException;
import java.io.StringReader;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.Date;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Locale;
import java.util.Set;
import java.util.TimeZone;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.stream.Collectors;
import java.util.stream.Stream;
/**
* @author Andy
@@ -152,13 +148,6 @@ import edu.umd.cs.findbugs.annotations.SuppressWarnings;
*/
public class Solr4QueryParser extends QueryParser implements QueryConstants
{
/**
* IndexSchema
* @param matchVersion
* @param f
* @param a
*/
public Solr4QueryParser(SolrQueryRequest req, Version matchVersion, String f, Analyzer a,
FTSQueryParser.RerankPhase rerankPhase)
{
@@ -167,7 +156,6 @@ public class Solr4QueryParser extends QueryParser implements QueryConstants
setAnalyzeRangeTerms(true);
this.rerankPhase = rerankPhase;
this.schema = req.getSchema();
this.solrContentStore = getContentStore(req);
this.solrParams = req.getParams();
SolrCore core = req.getCore();
if(core != null) {
@@ -178,26 +166,10 @@ public class Solr4QueryParser extends QueryParser implements QueryConstants
}
private RerankPhase rerankPhase;
private SolrContentStore solrContentStore;
private SolrParams solrParams;
private ShardHandlerFactory shardHandlerFactory;
private SolrQueryRequest request;
/**
* Extracts the contentStore from SolrQueryRequest.
* @param req
* @return
*/
private SolrContentStore getContentStore(SolrQueryRequest req)
{
if(req.getSearcher() != null)
{
CoreContainer coreContainer = req.getSearcher().getCore().getCoreContainer();
AlfrescoCoreAdminHandler coreAdminHandler = (AlfrescoCoreAdminHandler) coreContainer.getMultiCoreHandler();
SolrInformationServer srv = (SolrInformationServer) coreAdminHandler.getInformationServers().get(req.getSearcher().getCore().getName());
return srv.getSolrContentStore();
}
return null;
}
IndexSchema schema;
@@ -651,9 +623,6 @@ public class Solr4QueryParser extends QueryParser implements QueryConstants
namespacePrefixResolver, dictionaryService, field) != null)
{
return createDataTypeDefinitionQuery(field, queryText, analysisMode, luceneFunction);
} else if (field.equals(FIELD_FTSSTATUS))
{
return createTermQuery(field, queryText);
} else if (field.equals(FIELD_TXID))
{
return createTxIdQuery(queryText);
@@ -710,33 +679,12 @@ public class Solr4QueryParser extends QueryParser implements QueryConstants
json = new JSONObject();
}
//Is the fingerprint in the local SolrContentStore
if(values == null)
{
long dbid = fetchDBID(nodeId);
if(dbid == -1 && isNumber(nodeId))
{
dbid = Long.parseLong(nodeId);
}
if(dbid > -1)
{
SolrInputDocument solrDoc = solrContentStore.retrieveDocFromSolrContentStore(AlfrescoSolrDataModel.getTenantId(TenantService.DEFAULT_DOMAIN), dbid);
if (solrDoc != null)
{
SolrInputField mh = solrDoc.getField("MINHASH");
if (mh != null)
{
values = mh.getValues();
}
}
}
if (values == null) {
values = fetchMinHash(nodeId);
}
String shards = this.solrParams.get("shards");
if(values == null && shards != null)
{
if(values == null && shards != null) {
//we are in distributed mode
//Fetch the fingerPrint from the shards.
//The UUID and DBID will both work for method call.
@@ -744,8 +692,7 @@ public class Solr4QueryParser extends QueryParser implements QueryConstants
}
//If we're in distributed mode then add the fingerprint to the json
if(values != null && shards != null && fingerPrint == null)
{
if(values != null && shards != null && fingerPrint == null) {
ModifiableSolrParams newParams = new ModifiableSolrParams();
newParams.add(solrParams);
solrParams = newParams;
@@ -795,21 +742,50 @@ public class Solr4QueryParser extends QueryParser implements QueryConstants
}
private long fetchDBID(String UUID) throws IOException {
/**
* fetch MINHASH values from lucene index
* @param id argument of search. It is taken as DBID if is numberic, LID otherwise
* @return List of minhashes
* @throws IOException
*/
private Collection fetchMinHash(String id) throws IOException {
SolrIndexSearcher searcher = request.getSearcher();
String query = UUID.startsWith("workspace") ? UUID : "workspace://SpacesStore/"+UUID;
TermQuery q = new TermQuery(new Term(FIELD_LID, query));
Query q;
if(isNumber(id))
{
long dbid = Long.parseLong(id);
q = LegacyNumericRangeQuery.newLongRange("DBID", dbid, dbid + 1, true, false);
}
else
{
String query = id.startsWith("workspace") ? id : "workspace://SpacesStore/"+id;
q = new TermQuery(new Term(FIELD_LID, query));
}
TopDocs docs = searcher.search(q, 1);
Set<String> fields = new HashSet();
fields.add(FIELD_DBID);
fields.add(FIELD_FINGERPRINT);
if(docs.totalHits == 1) {
ScoreDoc scoreDoc = docs.scoreDocs[0];
Document doc = searcher.doc(scoreDoc.doc, fields);
IndexableField dbidField = doc.getField(FIELD_DBID);
return dbidField.numericValue().longValue();
List<Object> values = new ArrayList<>();
IndexableField[] minHashes = doc.getFields("MINHASH");
for (IndexableField minHash : minHashes)
{
SchemaField sf = schema.getFieldOrNull(minHash.name());
Object value = DocsStreamer.getValue(sf, minHash);
values.add(value);
}
return values;
}
return -1;
return null;
}
private Collection fetchFingerPrint(String shards, String nodeId) {
search-services/alfresco-search/src/main/java/org/alfresco/solr/schema/highlight/CompositeReader.java
+78
View File
@@ -0,0 +1,78 @@
/*
* Copyright (C) 2005-2020 Alfresco Software Limited.
*
* This file is part of Alfresco
*
* Alfresco is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Alfresco is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
*/
package org.alfresco.solr.schema.highlight;
import static java.util.Arrays.asList;
import org.alfresco.solr.utils.Utils;
import java.io.Closeable;
import java.io.IOException;
import java.io.Reader;
import java.util.Iterator;
import java.util.List;
/**
* A {@link Reader} able to wrap multiple readers.
* This class acts as a Facade if the wrapped readers, meaning with that its consumer will access to the underlying
* content in a transparent way: the composite reader takes care about switching on the next wrapped reader when the
* current one is exhausted.
*
* @author Andrea Gazzarini
*/
class CompositeReader extends Reader
{
private final Iterator<Reader> iterator;
private final List<Closeable> exhausted;
private Reader current;
public CompositeReader(final Reader ... readers)
{
if (readers == null || readers.length == 0)
{
throw new IllegalArgumentException("At least one reader instance is needed.");
}
this.exhausted = asList(readers);
this.iterator = asList(readers).iterator();
current = iterator.next();
}
@Override
public int read(char[] cbuf, int off, int len) throws IOException
{
int read = current.read(cbuf, off, len);
if (read == -1 && iterator.hasNext())
{
current = iterator.next();
read = current.read(cbuf, off, len);
}
else if (read < len && iterator.hasNext())
{
current = iterator.next();
read += current.read(cbuf, read, len - read);
}
return read;
}
@Override
public void close()
{
exhausted.forEach(Utils::silentyClose);
}
}
search-services/alfresco-search/src/main/java/org/alfresco/solr/schema/highlight/LanguagePrefixedTextAnalyzer.java
+66
View File
@@ -0,0 +1,66 @@
/*
* Copyright (C) 2005-2020 Alfresco Software Limited.
*
* This file is part of Alfresco
*
* Alfresco is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Alfresco is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
*/
package org.alfresco.solr.schema.highlight;
import org.alfresco.solr.AlfrescoAnalyzerWrapper;
import org.apache.lucene.analysis.Analyzer;
import org.apache.solr.schema.IndexSchema;
import org.apache.solr.schema.TextField;
/**
* A custom {@link Analyzer} type, aware about the Solr {@link IndexSchema }, which delegates the processing to a custom
* {@link org.apache.lucene.analysis.TokenStream}.
* Although the core part of the logic needed for that dynamic assignment is in {@link LanguagePrefixedTokenStream}
* the actors group is also composed by:
*
* <ul>
* <li>a {@link TextField} subclass (this class)</li>
* <li>a custom {@link Analyzer}</li>
* <li>a custom token stream {@link LanguagePrefixedTokenStream}</li>
* </ul>
*
* Both of them have no specific logic: they exist only because the components involved in the analysis chain don't have
* access to the {@link IndexSchema} instance (e.g. a {@link org.apache.lucene.analysis.Tokenizer} is a schema concept,
* while {@link IndexSchema} belongs to Solr classes).
*
* @see LanguagePrefixedTokenStream
* @see LanguagePrefixedTextField
* @author Andrea Gazzarini
*/
public class LanguagePrefixedTextAnalyzer extends Analyzer
{
protected IndexSchema indexSchema;
public final AlfrescoAnalyzerWrapper.Mode mode;
public LanguagePrefixedTextAnalyzer(IndexSchema indexSchema, AlfrescoAnalyzerWrapper.Mode mode)
{
super(Analyzer.PER_FIELD_REUSE_STRATEGY);
this.mode = mode;
this.indexSchema = indexSchema;
}
@Override
public TokenStreamComponents createComponents(String fieldName)
{
final LanguagePrefixedTokenStream decorator =
new LanguagePrefixedTokenStream(indexSchema, fieldName, mode);
return new TokenStreamComponents(decorator, decorator);
}
}
search-services/alfresco-search/src/main/java/org/alfresco/solr/schema/highlight/LanguagePrefixedTextField.java
+56
View File
@@ -0,0 +1,56 @@
/*
* Copyright (C) 2005-2020 Alfresco Software Limited.
*
* This file is part of Alfresco
*
* Alfresco is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Alfresco is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
*/
package org.alfresco.solr.schema.highlight;
import org.alfresco.solr.AlfrescoAnalyzerWrapper;
import org.apache.solr.schema.IndexSchema;
import org.apache.solr.schema.TextField;
import java.util.Map;
/**
* A custom {@link TextField} type which is able to set at runtime the analyzer to be used.
* Although the core part of the logic needed for that dynamic assignment is in {@link LanguagePrefixedTokenStream}
* the actors group is also composed by:
*
* <ul>
* <li>a {@link TextField} subclass (this class)</li>
* <li>a custom {@link org.apache.lucene.analysis.Analyzer}</li>
* <li>a custom token stream {@link LanguagePrefixedTokenStream}</li>
* </ul>
*
* Both of them have no specific logic: they exist only because the components involved in the analysis chain don't have
* access to the {@link IndexSchema} instance (e.g. a {@link org.apache.lucene.analysis.Tokenizer} is a schema concept,
* while {@link IndexSchema} belongs to Solr classes).
* On top of that, the purpose of this class is to associate to this field type the index and query time analyzer.
*
* @see LanguagePrefixedTokenStream
* @see LanguagePrefixedTextAnalyzer
* @author Andrea Gazzarini
*/
public class LanguagePrefixedTextField extends TextField
{
@Override
protected final void init(IndexSchema schema, Map<String,String> args)
{
super.init(schema, args);
setIndexAnalyzer(new LanguagePrefixedTextAnalyzer(schema, AlfrescoAnalyzerWrapper.Mode.INDEX));
setQueryAnalyzer(new LanguagePrefixedTextAnalyzer(schema, AlfrescoAnalyzerWrapper.Mode.QUERY));
}
}
search-services/alfresco-search/src/main/java/org/alfresco/solr/schema/highlight/LanguagePrefixedTokenStream.java
+368
View File
@@ -0,0 +1,368 @@
/*
* Copyright (C) 2005-2020 Alfresco Software Limited.
*
* This file is part of Alfresco
*
* Alfresco is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Alfresco is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
*/
package org.alfresco.solr.schema.highlight;
import static java.util.Optional.of;
import static java.util.Optional.ofNullable;
import org.alfresco.solr.AlfrescoAnalyzerWrapper;
import org.alfresco.util.Pair;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
import org.apache.solr.schema.FieldType;
import org.apache.solr.schema.IndexSchema;
import java.io.CharArrayReader;
import java.io.IOException;
import java.io.Reader;
import java.util.Optional;
/**
* A {@link TokenStream} decorator which determines dynamically the field type and the analyzer used for
* executing the analysis of an input text.
* Although this class extends {@link Tokenizer}, actually it is not a tokenizer: this because in order to individuate
* the analyzer dynamically, a component must access to a {@link IndexSchema} instance, and usually this is not possible
* in the components involved in the analysis chain (e.g. tokenizer, token filters, char filters).
*
* The field type and the analyzer that will control the text analysis are computed in the following way:
*
* <ul>
* <li>
* pre-process the input reader given to this chain in order to detect the locale language code at the very
* beginning.
* The locale language prefix includes
* <ul>
* <li>a beginning sentinel token #0;</li>
* <li>a language code (two or three chars)</li>
* <li>a closing sentinel token #0;</li>
* </ul>
* </li>
* <li>
* if any language code has been found, it is used for determine a field type name composed by the
* prefix "highlighted_text_" and the detected language code (e.g. highlighted_text_ + en = highlighted_text_en).
* </li>
* <li>
* If the field type above doesn't exist in the schema, the the same procedure is repeated using the prefix
* "text_" (e.g. text_ + en = text_en)
* </li>
* <li>
* If the field type above doesn't exist in the schema, then the "text___" general text field type is used.
* </li>
* <li>
* The input text is analyzed using the (query or index) analyzer associated to the field type determined above.
* </li>
* </ul>
*
* @author Andrea Gazzarini
*/
public final class LanguagePrefixedTokenStream extends Tokenizer
{
static final String FALLBACK_TEXT_FIELD_TYPE_NAME = "text___";
static final String LOCALISED_FIELD_TYPE_NAME_PREFIX = "text_";
static final String LOCALISED_HIGHLIGHTING_FIELD_TYPE_NAME_PREFIX = "highlighted_text_";
private final static char LANGUAGE_SENTINEL_TOKEN = '\u0000';
private final static char [] EMPTY_CHARSTREAM = {};
protected String fieldName;
protected IndexSchema indexSchema;
protected AlfrescoAnalyzerWrapper.Mode mode;
protected Analyzer analyzer;
private TokenStream stream;
private int localeMarkerLength;
private CharTermAttribute decoratorTerm;
private PositionIncrementAttribute decoratorPositionIncrement;
private OffsetAttribute decoratorOffset;
private TypeAttribute decoratorType;
private CharTermAttribute decoratedTerm;
private PositionIncrementAttribute decoratedPositionInc;
private OffsetAttribute decoratedOffset;
private TypeAttribute decoratedType;
LanguagePrefixedTokenStream(IndexSchema indexSchema, String fieldName, AlfrescoAnalyzerWrapper.Mode mode)
{
this.indexSchema = indexSchema;
this.fieldName = fieldName;
this.mode = mode;
decoratorTerm = addAttribute(CharTermAttribute.class);
decoratorPositionIncrement = addAttribute(PositionIncrementAttribute.class);
decoratorOffset = addAttribute(OffsetAttribute.class);
decoratorType = addAttribute(TypeAttribute.class);
}
@Override
public void reset() throws IOException
{
super.reset();
clearAttributes();
final Pair<Optional<String>, Reader> info = languageAndReaderFrom(input);
this.localeMarkerLength = localeMarkerLength(info.getFirst());
String language = info.getFirst().orElse("__");
this.analyzer = analyzer(language);
this.stream = analyzer.tokenStream(fieldName, info.getSecond());
this.stream.reset();
createOrRefreshAttributesOfDecoratedStream();
}
@Override
public boolean incrementToken() throws IOException
{
boolean result = stream.incrementToken();
if (result)
{
copyAttributesFromDecorateeToDecorator();
}
return result;
}
@Override
public void end() throws IOException
{
super.end();
stream.end();
}
@Override
public void close() throws IOException
{
super.close();
stream.close();
}
/**
* Starting from the input reader, this method detects the (eventual) locale language prefix and then
* creates a new {@link Reader} instance which, behind the scenes, reuses the unread part of the input reader.
* This avoids to read the full stream in a new string or char array and the create a completely new reader
* on top of that.
*
* We can have the following scenarios:
*
* <ul>
* <li>the underlying stream doesn't have any locale language prefix: in this case the returned reader will
* consume the whole stream from the beginning</li>
* <li>the underlying stream starts with a locale language marker (\u0000 + locale language + \u0000): in this
* case the language is isolated (that's the reason why this method returns a pair) and the new reader will
* consume the stream from the first char after the language marker.</li>
* </ul>
*
* @param reader the input reader
* @return a pair consisting of a locale language (a String) and a new content reader.
*/
Pair<Optional<String>, Reader> languageAndReaderFrom(Reader reader) throws IOException
{
final char [] prefix = new char[5];
final int read = reader.read(prefix);
if (read < prefix.length)
{
return new Pair<>(Optional.empty(),
isValidLocaleMarker(prefix)
? new CharArrayReader(EMPTY_CHARSTREAM)
: new CharArrayReader(prefix, 0, Math.max(read, 0)));
}
else
{
if (isValidLocaleMarker(prefix))
{
if (isTwoCharsLanguageCode(prefix))
{
Reader alreadyConsumedCharsReader = new CharArrayReader(prefix, prefix.length - 1, 1);
return new Pair<>(
of(new String(prefix, 1, 2).toLowerCase()),
new CompositeReader(alreadyConsumedCharsReader, reader));
}
else
{
return new Pair<>(
of(new String(prefix, 1, 3).toLowerCase()),
reader);
}
}
else
{
Reader alreadyConsumedCharsReader = new CharArrayReader(prefix);
return new Pair<>(Optional.empty(), new CompositeReader(alreadyConsumedCharsReader, reader));
}
}
}
/**
* Returns true if the input marker contains a language code composed by 2 chars.
* A language code can be composed by two (e.g. en) or three characters (e.g. kyr).
* Since the buffer used for reading the marker from the input reader has 9 slots, we need to understand the size of
* the language code in the array. This because in case it is a 2 chars code, there will be a remaining character
* in the buffer which represents content to be indexed.
*
* The buffer is composed by
*
* <ul>
* <li>
* 3 chars (#0;)
* </li>
* <li>
* locale language code
* </li>
* <li>
* 3 chars (#0;)
* </li>
* </ul>
*
* The buffer where we read the information above has a size equal to 9. That means,
*
* <ul>
* <li>
* if we read a 3-chars language code the whole array is used (#0;kyr#0;)
* </li>
* <li>
* if we read a 2-chars language code the last position in the array is not part of the marker and
* we need to make it available for the analyzer consumption.
* </li>
* </ul>
*
* @param marker the char array where we previously read the locale marker.
* @return true if the locale language code in the marker is a 2-chars code, false otherwise (3-chars code).
*/
private boolean isTwoCharsLanguageCode(final char[] marker)
{
return marker.length == 5 && marker[3] == LANGUAGE_SENTINEL_TOKEN;
}
/**
* Checks if the given prefix chars corresponds to a valid locale marker.
* The locale key delimiter used in SearchServices (\u0000) becomes "#0;" in the indexable value,
* so in order to understand if a given prefix is a valid locale marker or not we expected a 8/9 char array where
*
* pos [0] and [length - 3] = '#'
* pos [1] and [length - 2] = '0'
* pos [2] and [length - 1] = ';'
* pos [3] and [4] and optionally pos [5] = two/three chars that should* correspond to a locale language.
*
* Example: "#0;en#0;", "#0;kyr#0;"
*
* *Note this method doesn't check if the two/three chars at pos 3 and 4 (opt 5) actually correspond to a valid locale language code.
*/
private boolean isValidLocaleMarker(char[] prefix)
{
int length = prefix.length;
return prefix[0] == LANGUAGE_SENTINEL_TOKEN
&& (prefix[length - 1] == LANGUAGE_SENTINEL_TOKEN ||
(prefix[length - 2] == LANGUAGE_SENTINEL_TOKEN));
}
/**
* Returns the {@link Analyzer} associated with the given language.
* The proper {@link Analyzer} is retrieved from the first field type not null in the following list:
*
* <ul>
* <li>highlighted_text_ + locale (e.g. highlighted_text_en)</li>
* <li>text_ + locale (e.g. text_en)</li>
* <li>text___ (text general field)</li>
* </ul>
*
* @param language the language code.
* @return the {@link Analyzer} associated with the given language.
*/
Analyzer analyzer(String language) {
FieldType localisedFieldType =
ofNullable(indexSchema.getFieldTypeByName(highlightingFieldTypeName(language)))
.orElseGet(() -> indexSchema.getFieldTypeByName(localisedFieldTypeName(language)));
FieldType targetFieldType =
ofNullable(localisedFieldType)
.orElseGet(() -> indexSchema.getFieldTypeByName(FALLBACK_TEXT_FIELD_TYPE_NAME));
switch (mode)
{
case QUERY:
return targetFieldType.getQueryAnalyzer();
case INDEX:
default:
return targetFieldType.getIndexAnalyzer();
}
}
/**
* After a consumption cycle of the current managed token stream (see {@link #incrementToken()} we need to copy
* the attributes of that stream on the top level attributes of this decorator so the caller can retrieve them
* properly.
*/
void copyAttributesFromDecorateeToDecorator()
{
this.decoratorTerm.copyBuffer(decoratedTerm.buffer(), 0, decoratedTerm.length());
this.decoratorOffset.setOffset(
decoratedOffset.startOffset() + localeMarkerLength,
decoratedOffset.endOffset() + localeMarkerLength);
this.decoratorType.setType(decoratedType.type());
this.decoratorPositionIncrement.setPositionIncrement(decoratedPositionInc.getPositionIncrement());
}
/**
* The current token stream has been consumed. This decorator will switch on the next stream for consumption.
* In order to do that, we need to create a valid set of attributes from the new token stream.
*/
void createOrRefreshAttributesOfDecoratedStream()
{
decoratedTerm = stream.addAttribute(CharTermAttribute.class);
decoratedPositionInc = stream.addAttribute(PositionIncrementAttribute.class);
decoratedOffset = stream.addAttribute(OffsetAttribute.class);
decoratedType = stream.addAttribute(TypeAttribute.class);
}
/**
* Returns the name of the localised field type specifically used for the highlighting.
* By convention, the field type name is obtained using a "highlighted_text_" prefix followed by the
* locale language code (e.g. text_en, text_fr).
*
* @return the name of the localised field type specifically used for the highlighting.
*/
String highlightingFieldTypeName(String language)
{
return LOCALISED_HIGHLIGHTING_FIELD_TYPE_NAME_PREFIX + language;
}
/**
* Returns the name of the localised field type we are interested in the current processing.
* By convention, the field type name is obtained using a "text_" prefix followed by the locale language code
* (e.g. text_en, text_fr).
*
* @return the localised field type name.
*/
String localisedFieldTypeName(String language)
{
return LOCALISED_FIELD_TYPE_NAME_PREFIX + language;
}
int localeMarkerLength(Optional<String> language)
{
return language.map(String::length).map(length -> length + 2).orElse(0);
}
}
search-services/alfresco-search/src/main/java/org/alfresco/solr/schema/highlight/package-info.java
+67
View File
@@ -0,0 +1,67 @@
/*
* Copyright (C) 2005-2020 Alfresco Software Limited.
*
* This file is part of Alfresco
*
* Alfresco is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Alfresco is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
*/
/**
* This package contains a custom field type (and dependent component) which is supposed to be used
* only on fields that
*
* <ul>
* <li>are marked as stored and not indexed (indexed = false, stored = true)</li>
* <li>are used in highlighting requests (the reason why this package belongs to a "highlight" namespace)</li>
* </ul>
*
* The underlying reason of this customisation is mainly related with the custom highlighter used in Alfresco Search
* Services: highlight fields needs have the following requirements:
*
* <ul>
* <li>they have to be stored</li>
* <li>
* they don't have to be indexed but they must have a TextField (or a subclass) as type, because they must
* provide an index time {@link org.apache.lucene.analysis.Analyzer} (yes, even if indexed is set to false)
* which will be used for analysing the stored content and extract the highlighting snippets.
* </li>
* </ul>
*
* The field type purpose is actually to define a custom analyzer which is able to detect the proper localised analyzer
* at runtime, depending on the locale marker prefix put on the stored content.
* For example,
*
* <ul>
* <li>
* a field "title" with the following content: "\u0000en\u0000this is an english title" will be highlighted
* using an english analyzer (specifically the index analyzer of the "highlighted_text_en" field type, or
* the index analyzer of the "text_en" field type in case the previous one is missing)
* </li>
* <li>
* a field "title" with the following content: "\u0000it\u0000Questo sarebbe un titolo" will be highlighted
* using an italian analyzer (specifically the index analyzer of the "highlighted_text_it" field type, or
* the index analyzer of the "text_it" field type in case the previous one is missing)
* </li>
* <li>
* a field "title" with the following content: "This is a title without any locale marker" will be highlighted
* using the analyzer of the general text field "text___".
* </li>
* <li>
* a field "title" with the following content: "\u0000unknown_locale\u0000This is a title without an unknown locale marker"
* will be highlighted using the analyzer of the general text field "text___".
* </li>
* </ul>
*
*
*/
package org.alfresco.solr.schema.highlight;
search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/AbstractTracker.java
+20 -15
View File
@@ -68,6 +68,10 @@ public abstract class AbstractTracker implements Tracker
protected boolean transformContent;
String shardTemplate;
protected volatile boolean rollback;
/**
* When rollback is set, original error is also gathered in order to provide detailed logging.
*/
protected Throwable rollbackCausedBy;
protected final Type type;
/*
@@ -176,13 +180,13 @@ public abstract class AbstractTracker implements Tracker
assert(assertTrackerStateRemainsNull());
}
LOGGER.info("... Running {} for core [{}]", this.getClass().getSimpleName(), coreName);
LOGGER.info("[CORE {}] Running {}", coreName, this.getClass().getSimpleName());
if(this.state == null)
{
this.state = getTrackerState();
LOGGER.debug("Global Tracker State set to: {}", this.state.toString());
LOGGER.debug("[CORE {}] Global Tracker State set to: {}", coreName, this.state.toString());
this.state.setRunning(true);
}
else
@@ -199,34 +203,29 @@ public abstract class AbstractTracker implements Tracker
}
catch(IndexTrackingShutdownException t)
{
setRollback(true);
LOGGER.info("Stopping index tracking for {} - {}", getClass().getSimpleName(), coreName);
setRollback(true, t);
LOGGER.info("[CORE {}] Stopping index tracking for {}", coreName, getClass().getSimpleName());
}
catch(Throwable t)
{
setRollback(true);
setRollback(true, t);
if (t instanceof SocketTimeoutException || t instanceof ConnectException)
{
LOGGER.warn("[CORE {}] Tracking communication timed out for {}", coreName, getClass().getSimpleName());
if (LOGGER.isDebugEnabled())
{
// DEBUG, so give the whole stack trace
LOGGER.warn("Tracking communication timed out for {} - {}", getClass().getSimpleName(), coreName, t);
}
else
{
// We don't need the stack trace. It timed out.
LOGGER.warn("Tracking communication timed out for for {} - {}", getClass().getSimpleName(), coreName);
LOGGER.debug("[CORE {}] Stack trace", coreName, t);
}
}
else
{
LOGGER.error("Tracking failed for for {} - {}", getClass().getSimpleName(), coreName, t);
LOGGER.error("[CORE {}] Tracking failed for {}", coreName, getClass().getSimpleName(), t);
}
}
}
catch (InterruptedException e)
{
LOGGER.error("Semaphore interrupted for for {} - {}", getClass().getSimpleName(), coreName, e);
LOGGER.error("[CORE {}] Semaphore interrupted for {}", coreName, getClass().getSimpleName(), e);
}
finally
{
@@ -244,10 +243,16 @@ public abstract class AbstractTracker implements Tracker
{
return this.rollback;
}
public Throwable getRollbackCausedBy()
{
return this.rollbackCausedBy;
}
public void setRollback(boolean rollback)
public void setRollback(boolean rollback, Throwable rollbackCausedBy)
{
this.rollback = rollback;
this.rollbackCausedBy = rollbackCausedBy;
}
private void continueState()
search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/AbstractWorkerRunnable.java
+5 -3
View File
@@ -40,6 +40,7 @@ abstract class AbstractWorkerRunnable implements Runnable
public void run()
{
boolean failed = true;
Exception failCausedBy = null;
try
{
doWork();
@@ -47,7 +48,8 @@ abstract class AbstractWorkerRunnable implements Runnable
}
catch (Exception e)
{
log.warn("Index tracking batch hit an unrecoverable error ", e);
log.warn("Index tracking batch hit an unrecoverable error ", e);
failCausedBy = e;
}
finally
{
@@ -55,12 +57,12 @@ abstract class AbstractWorkerRunnable implements Runnable
queueHandler.removeFromQueueAndProdHead(this);
if(failed)
{
onFail();
onFail(failCausedBy);
}
}
}
abstract protected void doWork() throws Exception;
abstract protected void onFail();
abstract protected void onFail(Throwable failCausedBy);
}
search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/AclTracker.java
+90 -35
View File
@@ -54,7 +54,7 @@ import org.slf4j.LoggerFactory;
public class AclTracker extends AbstractTracker
{
protected final static Logger log = LoggerFactory.getLogger(AclTracker.class);
protected final static Logger LOGGER = LoggerFactory.getLogger(AclTracker.class);
private static final int DEFAULT_CHANGE_SET_ACLS_BATCH_SIZE = 100;
private static final int DEFAULT_ACL_BATCH_SIZE = 10;
@@ -135,8 +135,15 @@ public class AclTracker extends AbstractTracker
indexAcl(readers, false);
}
this.infoSrv.indexAclTransaction(changeSet, false);
LOGGER.info("[CORE {}] - INDEX ACTION - AclChangeSetId {} has been indexed", coreName, aclChangeSetId);
requiresCommit = true;
}
else
{
LOGGER.info(
"[CORE {}] - INDEX ACTION - AclChangeSetId {} was not found in database, it has NOT been reindexed",
coreName, aclChangeSetId);
}
}
checkShutdown();
}
@@ -160,6 +167,7 @@ public class AclTracker extends AbstractTracker
//AclReaders r = readers.get(0);
//System.out.println("############## READERS ID:"+r.getId()+":"+r.getReaders());
indexAcl(readers, false);
LOGGER.info("[CORE {}] - INDEX ACTION - AclId {} has been indexed", coreName, aclId);
}
checkShutdown();
}
@@ -187,8 +195,15 @@ public class AclTracker extends AbstractTracker
}
this.infoSrv.indexAclTransaction(changeSet, true);
LOGGER.info("[CORE {}] - REINDEX ACTION - AclChangeSetId {} has been reindexed", coreName, aclChangeSetId);
requiresCommit = true;
}
else
{
LOGGER.info(
"[CORE {}] - REINDEX ACTION - AclChangeSetId {} was not found in database, it has NOT been reindexed",
coreName, aclChangeSetId);
}
}
checkShutdown();
}
@@ -212,6 +227,7 @@ public class AclTracker extends AbstractTracker
Acl acl = new Acl(0, aclId);
List<AclReaders> readers = client.getAclReaders(Collections.singletonList(acl));
indexAcl(readers, true);
LOGGER.info("[CORE {}] - REINDEX ACTION - aclId {} has been reindexed", coreName, aclId);
requiresCommit = true;
}
checkShutdown();
@@ -231,6 +247,7 @@ public class AclTracker extends AbstractTracker
if (aclChangeSetId != null)
{
this.infoSrv.deleteByAclChangeSetId(aclChangeSetId);
LOGGER.info("[CORE {}] - PURGE ACTION - Purged aclChangeSetId {}", coreName, aclChangeSetId);
}
checkShutdown();
}
@@ -245,6 +262,7 @@ public class AclTracker extends AbstractTracker
if (aclId != null)
{
this.infoSrv.deleteByAclId(aclId);
LOGGER.info("[CORE {}] - PURGE ACTION - Purged aclId {}", coreName, aclId);
}
checkShutdown();
}
@@ -316,7 +334,7 @@ public class AclTracker extends AbstractTracker
{
state.setCheckedLastAclTransactionTime(true);
state.setCheckedFirstAclTransactionTime(true);
log.info("No acl transactions found - no verification required");
LOGGER.info("[CORE {}] - No acl transactions found - no verification required", coreName);
firstChangeSets = client.getAclChangeSets(null, 0L, null, 2000L, 1);
if (!firstChangeSets.getAclChangeSets().isEmpty())
@@ -340,20 +358,20 @@ public class AclTracker extends AbstractTracker
if (setSize == 0)
{
log.error("First acl transaction was not found with the correct timestamp.");
log.error("SOLR has successfully connected to your repository however the SOLR indexes and repository database do not match.");
log.error("If this is a new or rebuilt database your SOLR indexes also need to be re-built to match the database.");
log.error("You can also check your SOLR connection details in solrcore.properties.");
LOGGER.error("[CORE {}] First acl transaction was not found with the correct timestamp.", coreName);
LOGGER.error("SOLR has successfully connected to your repository however the SOLR indexes and repository database do not match.");
LOGGER.error("If this is a new or rebuilt database your SOLR indexes also need to be re-built to match the database.");
LOGGER.error("You can also check your SOLR connection details in solrcore.properties.");
throw new AlfrescoRuntimeException("Initial acl transaction not found with correct timestamp");
}
else if (setSize == 1)
{
state.setCheckedFirstTransactionTime(true);
log.info("Verified first acl transaction and timestamp in index");
LOGGER.info("[CORE {}] Verified first acl transaction and timestamp in index", coreName);
}
else
{
log.warn("Duplicate initial acl transaction found with correct timestamp");
LOGGER.warn("[CORE {}] Duplicate initial acl transaction found with correct timestamp", coreName);
}
}
}
@@ -374,19 +392,19 @@ public class AclTracker extends AbstractTracker
AclChangeSet maxAclTxInIndex = this.infoSrv.getMaxAclChangeSetIdAndCommitTimeInIndex();
if (maxAclTxInIndex.getCommitTimeMs() > maxChangeSetCommitTimeInRepo)
{
log.error("Last acl transaction was found in index with timestamp later than that of repository.");
log.error("Max Acl Tx In Index: " + maxAclTxInIndex.getId() + ", In Repo: " + maxChangeSetIdInRepo);
log.error("Max Acl Tx Commit Time In Index: " + maxAclTxInIndex.getCommitTimeMs() + ", In Repo: "
LOGGER.error("[CORE {}] Last acl transaction was found in index with timestamp later than that of repository.", coreName);
LOGGER.error("Max Acl Tx In Index: " + maxAclTxInIndex.getId() + ", In Repo: " + maxChangeSetIdInRepo);
LOGGER.error("Max Acl Tx Commit Time In Index: " + maxAclTxInIndex.getCommitTimeMs() + ", In Repo: "
+ maxChangeSetCommitTimeInRepo);
log.error("SOLR has successfully connected to your repository however the SOLR indexes and repository database do not match.");
log.error("If this is a new or rebuilt database your SOLR indexes also need to be re-built to match the database.");
log.error("You can also check your SOLR connection details in solrcore.properties.");
LOGGER.error("SOLR has successfully connected to your repository however the SOLR indexes and repository database do not match.");
LOGGER.error("If this is a new or rebuilt database your SOLR indexes also need to be re-built to match the database.");
LOGGER.error("You can also check your SOLR connection details in solrcore.properties.");
throw new AlfrescoRuntimeException("Last acl transaction found in index with incorrect timestamp");
}
else
{
state.setCheckedLastAclTransactionTime(true);
log.info("Verified last acl transaction timestamp in index less than or equal to that of repository.");
LOGGER.info("[CORE {}] - Verified last acl transaction timestamp in index less than or equal to that of repository.", coreName);
}
}
}
@@ -416,7 +434,7 @@ public class AclTracker extends AbstractTracker
AclChangeSets aclChangeSets;
// step forward in time until we find something or hit the time bound
// max id unbounded
Long startTime = fromCommitTime == null ? Long.valueOf(0L) :fromCommitTime;
Long startTime = fromCommitTime == null ? Long.valueOf(0L) : fromCommitTime;
do
{
aclChangeSets = client.getAclChangeSets(startTime, null, startTime + actualTimeStep, null, maxResults);
@@ -645,30 +663,58 @@ public class AclTracker extends AbstractTracker
*/
this.state = getTrackerState();
Long fromCommitTime = getChangeSetFromCommitTime(changeSetsFound, state.getLastGoodChangeSetCommitTimeInIndex());
aclChangeSets = getSomeAclChangeSets(changeSetsFound, fromCommitTime, TIME_STEP_1_HR_IN_MS, 2000,
state.getTimeToStopIndexing());
setLastChangeSetIdAndCommitTimeInTrackerState(aclChangeSets, state);
log.info("Scanning Acl change sets ...");
if (aclChangeSets.getAclChangeSets().size() > 0) {
log.info(".... from " + aclChangeSets.getAclChangeSets().get(0));
log.info(".... to " + aclChangeSets.getAclChangeSets().get(aclChangeSets.getAclChangeSets().size() - 1));
} else {
log.info(".... none found after lastTxCommitTime " + fromCommitTime);
if (aclChangeSets.getAclChangeSets().size() > 0)
{
LOGGER.info("{}-[CORE {}] Found {} ACL change sets after lastTxCommitTime {}, ACL Change Sets from {} to {}",
Thread.currentThread().getId(),
coreName,
aclChangeSets.getAclChangeSets().size(),
fromCommitTime,
aclChangeSets.getAclChangeSets().get(0),
aclChangeSets.getAclChangeSets().get(aclChangeSets.getAclChangeSets().size() - 1));
}
else
{
LOGGER.info("{}-[CORE {}] No ACL change set found after lastTxCommitTime {}",
Thread.currentThread().getId(), coreName, fromCommitTime);
}
ArrayList<AclChangeSet> changeSetBatch = new ArrayList<AclChangeSet>();
for (AclChangeSet changeSet : aclChangeSets.getAclChangeSets()) {
for (int i = 0; i < aclChangeSets.getAclChangeSets().size(); i++)
{
AclChangeSet changeSet = aclChangeSets.getAclChangeSets().get(i);
boolean isInIndex = (changeSet.getCommitTimeMs() <= state.getLastIndexedChangeSetCommitTime() &&
infoSrv.aclChangeSetInIndex(changeSet.getId(), true));
if (isInIndex) {
if (isInIndex)
{
// Logging progress for large ACL Change Set tracking every 100 tracked ACLs
if (LOGGER.isTraceEnabled())
{
LOGGER.trace("{}-[CORE {}] Tracking {} of {} ACL Change Sets. Change Set Id was already indexed: {}",
Thread.currentThread().getId(), coreName, i + 1, aclChangeSets.getAclChangeSets().size(), changeSet.getId());
}
changeSetsFound.add(changeSet);
} else {
}
else
{
// Logging progress for ACL Change Set
if (LOGGER.isTraceEnabled())
{
LOGGER.trace("{}-[CORE {}] Tracking {} of {} ACL Change Sets. Current Change Set Id to be indexed: {}",
Thread.currentThread().getId(), coreName, i + 1, aclChangeSets.getAclChangeSets().size(), changeSet.getId());
}
// Make sure we do not go ahead of where we started - we will check the holes here
// correctly next time
if (changeSet.getCommitTimeMs() > state.getTimeToStopIndexing()) {
@@ -731,10 +777,12 @@ public class AclTracker extends AbstractTracker
{
getWriteLock().release();
}
}
while ((aclChangeSets.getAclChangeSets().size() > 0) && (upToDate == false));
LOGGER.info("{}-[CORE {}] Tracked {} ACLs", Thread.currentThread().getId(), coreName, totalAclCount);
log.info("total number of acls updated: " + totalAclCount);
}
private void setLastChangeSetIdAndCommitTimeInTrackerState(AclChangeSets aclChangeSets, TrackerState state)
@@ -797,12 +845,19 @@ public class AclTracker extends AbstractTracker
ArrayList<Acl> aclBatch = new ArrayList<Acl>();
List<Acl> acls = client.getAcls(nonEmptyChangeSets, null, Integer.MAX_VALUE);
if (LOGGER.isDebugEnabled())
{
LOGGER.debug("{}-[CORE {}] Found {} Acls from Acl Change Sets: {}", Thread.currentThread().getId(),
coreName, acls.size(), nonEmptyChangeSets);
}
for (Acl acl : acls)
{
if (log.isDebugEnabled())
if (LOGGER.isTraceEnabled())
{
log.debug(acl.toString());
LOGGER.trace("{}-[CORE {}] Adding ACL {} to scheduled indexing job", Thread.currentThread().getId(),
coreName, acl.toString());
}
aclBatch.add(acl);
if (aclBatch.size() > aclBatchSize)
@@ -845,9 +900,9 @@ public class AclTracker extends AbstractTracker
}
@Override
protected void onFail()
protected void onFail(Throwable failCausedBy)
{
setRollback(true);
setRollback(true, failCausedBy);
}
private List<Acl> filterAcls(List<Acl> acls)
search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/CascadeTracker.java
+2 -6
View File
@@ -114,9 +114,9 @@ public class CascadeTracker extends AbstractTracker implements Tracker
}
@Override
protected void onFail()
protected void onFail(Throwable failCausedBy)
{
setRollback(true);
setRollback(true, failCausedBy);
}
}
@@ -127,7 +127,6 @@ public class CascadeTracker extends AbstractTracker implements Tracker
private void processCascades() throws IOException
{
//System.out.println("######### processCascades()");
int num = 50;
List<Transaction> txBatch = null;
do {
@@ -148,7 +147,6 @@ public class CascadeTracker extends AbstractTracker implements Tracker
List<NodeMetaData> nodeMetaDatas = infoSrv.getCascadeNodes(txIds);
//System.out.println("########### Cascade node meta datas:"+nodeMetaDatas.size());
if(nodeMetaDatas.size() > 0) {
LinkedList<NodeMetaData> stack = new LinkedList<NodeMetaData>();
stack.addAll(nodeMetaDatas);
@@ -167,7 +165,6 @@ public class CascadeTracker extends AbstractTracker implements Tracker
}
//Update the transaction records.
updateTransactionsAfterAsynchronous(txBatch);
//System.out.println("######################: Finished Cascade Run #########");
}
catch (AuthenticationException e)
{
@@ -183,7 +180,6 @@ public class CascadeTracker extends AbstractTracker implements Tracker
}
finally
{
//System.out.println("###################: Releasing Cascade write lock");
getWriteLock().release();
}
search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/CommitTracker.java
+29 -14
View File
@@ -19,7 +19,11 @@
package org.alfresco.solr.tracker;
import static java.util.Optional.empty;
import static java.util.Optional.ofNullable;
import java.util.List;
import java.util.Optional;
import java.util.Properties;
import java.util.concurrent.atomic.AtomicInteger;
@@ -41,7 +45,8 @@ public class CommitTracker extends AbstractTracker
private MetadataTracker metadataTracker;
private AclTracker aclTracker;
private ContentTracker contentTracker;
private CascadeTracker cascadeTracker;
/** The cascade tracker. Note that this may be empty if cascade tracking is disabled. */
private Optional<CascadeTracker> cascadeTracker = empty();
private AtomicInteger rollbackCount = new AtomicInteger(0);
protected final static Logger log = LoggerFactory.getLogger(CommitTracker.class);
@@ -71,7 +76,7 @@ public class CommitTracker extends AbstractTracker
} else if(tracker instanceof ContentTracker) {
this.contentTracker = (ContentTracker)tracker;
} else if(tracker instanceof CascadeTracker) {
this.cascadeTracker = (CascadeTracker)tracker;
this.cascadeTracker = ofNullable((CascadeTracker) tracker);
}
}
@@ -151,11 +156,8 @@ public class CommitTracker extends AbstractTracker
maintenance();
}
infoSrv.flushContentStore();
//Do the commit opening the searcher if needed. This will commit all the work done by indexing trackers.
//This will return immediately and not wait for searchers to warm
//System.out.println("################### Commit:"+openSearcherNeeded);
boolean searcherOpened = infoSrv.commit(openSearcherNeeded);
lastCommit = currentTime;
@@ -170,7 +172,6 @@ public class CommitTracker extends AbstractTracker
//Release the lock on the aclTracker
aclTracker.getWriteLock().release();
//System.out.println("######## Commit Tracker Releasing Write Locks ########");
}
}
@@ -182,10 +183,24 @@ public class CommitTracker extends AbstractTracker
contentTracker.getWriteLock().acquire();
assert(contentTracker.getWriteLock().availablePermits() == 0);
cascadeTracker.getWriteLock().acquire();
assert(cascadeTracker.getWriteLock().availablePermits() == 0);
if (cascadeTracker.isPresent())
{
cascadeTracker.get().getWriteLock().acquire();
assert (cascadeTracker.get().getWriteLock().availablePermits() == 0);
}
infoSrv.rollback();
// Log reasons why the rollback is performed
if (aclTracker.getRollbackCausedBy() != null)
{
log.warn("Rollback performed due to ACL Tracker error", aclTracker.getRollbackCausedBy());
}
if (metadataTracker.getRollbackCausedBy() != null)
{
log.warn("Rollback performed due to Metadata Tracker error", metadataTracker.getRollbackCausedBy());
}
}
catch (Exception e)
{
@@ -194,24 +209,24 @@ public class CommitTracker extends AbstractTracker
finally
{
//Reset acl Tracker
aclTracker.setRollback(false);
aclTracker.setRollback(false, null);
aclTracker.invalidateState();
//Reset metadataTracker
metadataTracker.setRollback(false);
metadataTracker.setRollback(false, null);
metadataTracker.invalidateState();
//Reset contentTracker
contentTracker.setRollback(false);
contentTracker.setRollback(false, null);
contentTracker.invalidateState();
//Reset cascadeTracker
cascadeTracker.setRollback(false);
cascadeTracker.invalidateState();
cascadeTracker.ifPresent(c -> c.setRollback(false, null));
cascadeTracker.ifPresent(c -> invalidateState());
//Release the locks
contentTracker.getWriteLock().release();
cascadeTracker.getWriteLock().release();
cascadeTracker.ifPresent(c -> c.getWriteLock().release());
rollbackCount.incrementAndGet();
}
search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/ContentTracker.java
+35 -27
View File
@@ -25,8 +25,8 @@ import java.util.Properties;
import org.alfresco.solr.AlfrescoSolrDataModel.TenantAclIdDbId;
import org.alfresco.solr.InformationServer;
import org.alfresco.solr.client.SOLRAPIClient;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static org.alfresco.solr.utils.Utils.notNullOrEmpty;
/**
* This tracker queries for docs with unclean content, and then updates them.
@@ -36,8 +36,6 @@ import org.slf4j.LoggerFactory;
*/
public class ContentTracker extends AbstractTracker implements Tracker
{
protected final static Logger log = LoggerFactory.getLogger(ContentTracker.class);
private int contentReadBatchSize;
private int contentUpdateBatchSize;
@@ -59,37 +57,41 @@ public class ContentTracker extends AbstractTracker implements Tracker
@Override
protected void doTrack() throws Exception
{
//System.out.println("############## Content Tracker doTrack()");
try {
try
{
long startElapsed = System.nanoTime();
checkShutdown();
final int ROWS = contentReadBatchSize;
int start = 0;
long totalDocs = 0l;
long totalDocs = 0L;
checkShutdown();
while (true) {
while (true)
{
try
{
getWriteLock().acquire();
List<TenantAclIdDbId> docs = this.infoSrv.getDocsWithUncleanContent(start, ROWS);
//System.out.println("####################### Unclean content: "+docs.size()+" ##############################:"+totalDocs);
if (docs.size() == 0) {
List<TenantAclIdDbId> docs = notNullOrEmpty(infoSrv.getDocsWithUncleanContent(start, ROWS));
if (docs.isEmpty())
{
LOGGER.debug("No unclean document has been detected in the current ContentTracker cycle.");
break;
}
int docsUpdatedSinceLastCommit = 0;
for (TenantAclIdDbId doc : docs) {
for (TenantAclIdDbId doc : docs)
{
ContentIndexWorkerRunnable ciwr = new ContentIndexWorkerRunnable(super.threadHandler, doc, infoSrv);
super.threadHandler.scheduleTask(ciwr);
docsUpdatedSinceLastCommit++;
if (docsUpdatedSinceLastCommit >= contentUpdateBatchSize) {
if (docsUpdatedSinceLastCommit >= contentUpdateBatchSize)
{
super.waitForAsynchronous();
checkShutdown();
//this.infoSrv.commit();
long endElapsed = System.nanoTime();
trackerStats.addElapsedContentTime(docsUpdatedSinceLastCommit, endElapsed - startElapsed);
startElapsed = endElapsed;
@@ -97,7 +99,8 @@ public class ContentTracker extends AbstractTracker implements Tracker
}
}
if (docsUpdatedSinceLastCommit > 0) {
if (docsUpdatedSinceLastCommit > 0)
{
super.waitForAsynchronous();
checkShutdown();
//this.infoSrv.commit();
@@ -113,7 +116,7 @@ public class ContentTracker extends AbstractTracker implements Tracker
}
}
log.info("total number of docs with content updated: " + totalDocs);
LOGGER.info("Total number of docs with content updated: {}", totalDocs);
}
catch(Exception e)
{
@@ -121,15 +124,18 @@ public class ContentTracker extends AbstractTracker implements Tracker
}
}
public boolean hasMaintenance() {
public boolean hasMaintenance()
{
return false;
}
public void maintenance() {
return;
public void maintenance()
{
// Nothing to be done here
}
public void invalidateState() {
public void invalidateState()
{
super.invalidateState();
this.infoSrv.setCleanContentTxnFloor(-1);
}
@@ -137,12 +143,13 @@ public class ContentTracker extends AbstractTracker implements Tracker
class ContentIndexWorkerRunnable extends AbstractWorkerRunnable
{
InformationServer infoServer;
TenantAclIdDbId doc;
TenantAclIdDbId docRef;
ContentIndexWorkerRunnable(QueueHandler queueHandler, TenantAclIdDbId doc, InformationServer infoServer)
ContentIndexWorkerRunnable(QueueHandler queueHandler, TenantAclIdDbId docRef, InformationServer infoServer)
{
super(queueHandler);
this.doc = doc;
this.docRef = docRef;
this.infoServer = infoServer;
}
@@ -150,14 +157,15 @@ public class ContentTracker extends AbstractTracker implements Tracker
protected void doWork() throws Exception
{
checkShutdown();
//System.out.println("################ Update doc:"+doc.dbId);
this.infoServer.updateContentToIndexAndCache(doc.dbId, doc.tenant);
infoServer.updateContent(docRef);
}
@Override
protected void onFail()
protected void onFail(Throwable failCausedBy)
{
// Will redo if not persisted
// This will be redone in future tracking operations
log.warn("Content tracker failed due to {}", failCausedBy.getMessage(), failCausedBy);
}
}
}
search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/CoreStatePublisher.java
+9 -7
View File
@@ -40,6 +40,7 @@ import org.alfresco.service.cmr.dictionary.PropertyDefinition;
import org.alfresco.service.namespace.QName;
import org.alfresco.solr.AlfrescoCoreAdminHandler;
import org.alfresco.solr.AlfrescoSolrDataModel;
import org.alfresco.solr.BoundedDeque;
import org.alfresco.solr.InformationServer;
import org.alfresco.solr.NodeReport;
import org.alfresco.solr.TrackerState;
@@ -52,8 +53,8 @@ import java.util.Properties;
/**
* Superclass for all components which are able to inform Alfresco about the hosting node state.
* This has been introduced in SEARCH-1752 for splitting the dual responsibility of the {@link MetadataTracker}.
* As consequence of that, this class contains all the members needed for obtaining a valid
* This has been introduced in SEARCH-1752 for splitting the dual responsibility of the {@link org.alfresco.solr.tracker.MetadataTracker}.
* As consequence of that, this class contains only the members needed for obtaining a valid
* {@link org.alfresco.repo.index.shard.ShardState} that can be periodically communicated to Alfresco.
*
* @author Andrea Gazzarini
@@ -176,15 +177,15 @@ public abstract class CoreStatePublisher extends AbstractTracker
* The {@link ShardState} is primarily used in two places:
*
* <ul>
* <li>Transaction tracking: (see {@link MetadataTracker#trackTransactions()}): for pulling/tracking transactions from Alfresco</li>
* <li>Transaction tracking: (see {@link MetadataTracker#getSomeTransactions(BoundedDeque, Long, long, int, long}): for pulling/tracking transactions from Alfresco</li>
* <li>
* DynamicSharding: when the {@link MetadataTracker} is running on a slave instance it doesn't actually act
* as a tracker, it calls Alfresco to register the state of the node (the shard) without pulling any transactions.
* As consequence of that, Alfresco will be aware about the shard which will be included in subsequent queries.
* DynamicSharding: the {@link MetadataTracker} is not running on a slave instances; in those cases a special
* "tracker" ({@link SlaveCoreStatePublisher}) will be in charge to send the correspondin shard state to Alfresco.
* </li>
* </ul>
*
* @return the {@link ShardState} instance which stores the current state of the hosting shard.
* @see SlaveCoreStatePublisher
*/
ShardState getShardState()
{
@@ -198,10 +199,11 @@ public abstract class CoreStatePublisher extends AbstractTracker
HashMap<String, String> propertyBag = new HashMap<>();
propertyBag.put("coreName", coreName);
HashMap<String, String> extendedPropertyBag = new HashMap<>(propertyBag);
updateShardProperty();
shardProperty.ifPresent(p -> extendedPropertyBag.putAll(docRouter.getProperties(p)));
extendedPropertyBag.putAll(docRouter.getProperties(shardProperty));
return ShardStateBuilder.shardState()
.withMaster(isMaster)
search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/DBIDRangeRouter.java
+2 -2
View File
@@ -19,6 +19,7 @@
package org.alfresco.solr.tracker;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
@@ -108,9 +109,8 @@ public class DBIDRangeRouter implements DocRouter
}
@Override
public Map<String, String> getProperties(QName shardProperty)
public Map<String, String> getProperties(Optional<QName> shardProperty)
{
return Map.of(DocRouterFactory.SHARD_RANGE_KEY, startRange + "-" + expandableRange);
}
}
search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/DateMonthRouter.java
+10 -9
View File
@@ -26,10 +26,12 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.Calendar;
import java.util.Collections;
import java.util.Date;
import java.util.GregorianCalendar;
import java.util.Map;
import java.util.Optional;
import static java.util.Collections.emptyMap;
/**
* The date-based sharding assigns dates sequentially through shards based on the month.
@@ -121,14 +123,13 @@ public class DateMonthRouter implements DocRouter
}
@Override
public Map<String, String> getProperties(QName shardProperty)
public Map<String, String> getProperties(Optional<QName> shardProperty)
{
return (shardProperty == null ?
Collections.emptyMap() :
Map.of(DocRouterFactory.SHARD_KEY_KEY, shardProperty.getPrefixString(),
DocRouterFactory.SHARD_DATE_GROUPING_KEY, String.valueOf(grouping)));
return shardProperty
.map(QName::getPrefixString)
.map(prefix -> Map.of(
DocRouterFactory.SHARD_KEY_KEY, prefix,
DocRouterFactory.SHARD_DATE_GROUPING_KEY, String.valueOf(grouping)))
.orElse(emptyMap());
}
}
search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/DocRouter.java
+5 -4
View File
@@ -22,6 +22,7 @@ import org.alfresco.solr.client.Node;
import java.util.Collections;
import java.util.Map;
import java.util.Optional;
import org.alfresco.service.namespace.QName;
import org.alfresco.solr.client.Acl;
@@ -65,13 +66,13 @@ public interface DocRouter
Boolean routeNode(int shardCount, int shardInstance, Node node);
/**
* Get additional properties to "shardProperty" depending on the Shard Method
* @param shardProperty custom property used to configure the Router
* Get additional properties to "shardProperty" depending on the Shard Method.
*
* @param shardProperty custom property used to configure the Router. Note not all routers need that.
* @return pair of key, value
*/
default public Map<String, String> getProperties(QName shardProperty) {
default Map<String, String> getProperties(Optional<QName> shardProperty) {
return Collections.emptyMap();
}
}
search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/DocRouterWithFallback.java
+2 -2
View File
@@ -9,6 +9,7 @@ import java.util.Objects;
import static java.util.Optional.ofNullable;
import java.util.Map;
import java.util.Optional;
/**
* A composable {@link DocRouter} which consists of
@@ -47,9 +48,8 @@ public class DocRouterWithFallback implements DocRouter
}
@Override
public Map<String, String> getProperties(QName shardProperty)
public Map<String, String> getProperties(Optional<QName> shardProperty)
{
return primaryStrategy.getProperties(shardProperty);
}
}
search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/ExplicitShardIdWithDynamicPropertyRouter.java
+8 -6
View File
@@ -18,13 +18,15 @@
*/
package org.alfresco.solr.tracker;
import java.util.Collections;
import java.util.Map;
import java.util.Optional;
import org.alfresco.service.namespace.QName;
import org.alfresco.solr.client.Acl;
import org.alfresco.solr.client.Node;
import static java.util.Collections.emptyMap;
/**
* Routes a document only if the shardInstance matches the provided shardId.
* The access control information is duplicated in each shard.
@@ -78,11 +80,11 @@ public class ExplicitShardIdWithDynamicPropertyRouter extends ComposableDocRoute
}
@Override
public Map<String, String> getProperties(QName shardProperty)
public Map<String, String> getProperties(Optional<QName> shardProperty)
{
return (shardProperty == null ?
Collections.emptyMap() :
Map.of(DocRouterFactory.SHARD_KEY_KEY, shardProperty.getPrefixString()));
return shardProperty
.map(QName::getPrefixString)
.map(prefix -> Map.of(DocRouterFactory.SHARD_KEY_KEY, prefix))
.orElse(emptyMap());
}
}
search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java
+169 -71
View File
@@ -18,8 +18,11 @@
*/
package org.alfresco.solr.tracker;
import static org.alfresco.repo.index.shard.ShardMethodEnum.DB_ID_RANGE;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.List;
@@ -64,7 +67,9 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
private ConcurrentLinkedQueue<Long> nodesToIndex = new ConcurrentLinkedQueue<>();
private ConcurrentLinkedQueue<Long> nodesToPurge = new ConcurrentLinkedQueue<>();
private ConcurrentLinkedQueue<String> queriesToReindex = new ConcurrentLinkedQueue<>();
private final boolean isRunningInProduction = !Boolean.parseBoolean(System.getProperty("alfresco.test", "false"));
/**
* Check if nextTxCommitTimeService is available in the repository.
* This service is used to find the next available transaction commit time from a given time,
@@ -83,44 +88,79 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
* {@link org.alfresco.solr.client.SOLRAPIClient#GET_TX_INTERVAL_COMMIT_TIME}
*/
private boolean txIntervalCommitTimeServiceAvailable = false;
/** Whether the cascade tracking is enabled. */
private boolean cascadeTrackerEnabled = true;
/**
* Transaction Id range to get the first transaction in database.
* 0-2000 by default.
*/
private Pair<Long, Long> minTxnIdRange;
public MetadataTracker(final boolean isMaster, Properties p, SOLRAPIClient client, String coreName,
InformationServer informationServer)
InformationServer informationServer)
{
this(isMaster, p, client, coreName, informationServer, false);
}
/**
* MetadataTracker constructor
*
* @param isMaster is true if SOLR instance is master, false otherwise
* @param p includes SOLR core properties (from environment variables and properties file)
* @param client Alfresco Repository http client
* @param coreName Name of the SOLR Core (alfresco, archive)
* @param informationServer SOLR Information Server
* @param checkRepoServicesAvailability is true if Repo Services availability needs to be checked
*/
public MetadataTracker(final boolean isMaster, Properties p, SOLRAPIClient client, String coreName,
InformationServer informationServer, boolean checkRepoServicesAvailability)
{
super(isMaster, p, client, coreName, informationServer, Tracker.Type.METADATA);
transactionDocsBatchSize = Integer.parseInt(p.getProperty("alfresco.transactionDocsBatchSize", "100"));
nodeBatchSize = Integer.parseInt(p.getProperty("alfresco.nodeBatchSize", "10"));
threadHandler = new ThreadHandler(p, coreName, "MetadataTracker");
cascadeTrackerEnabled = informationServer.cascadeTrackingEnabled();
String[] minTxninitialRangeString = p.getProperty("solr.initial.transaction.range", "0-2000").split("-");
minTxnIdRange = new Pair<Long, Long>(Long.valueOf(minTxninitialRangeString[0]), Long.valueOf(minTxninitialRangeString[1]));
// Try invoking getNextTxCommitTime service
try
// In order to apply performance optimizations, checking the availability of Repo Web Scripts is required.
// As these services are available from ACS 6.2
if (checkRepoServicesAvailability && isRunningInProduction)
{
client.getNextTxCommitTime(coreName, 0l);
nextTxCommitTimeServiceAvailable = true;
}
catch (NoSuchMethodException e)
{
log.warn("nextTxCommitTimeService is not available. Upgrade your ACS Repository version in order to use this feature: {} ", e.getMessage());
}
catch (Exception e)
{
log.error("Checking nextTxCommitTimeService failed.", e);
}
// Try invoking txIntervalCommitTime service
try
{
client.getTxIntervalCommitTime(coreName, 0l, 0l);
txIntervalCommitTimeServiceAvailable = true;
}
catch (NoSuchMethodException e)
{
log.warn("txIntervalCommitTimeServiceAvailable is not available. If you are using DB_ID_RANGE shard method, "
+ "upgrade your ACS Repository version in order to use this feature: {} ", e.getMessage());
}
catch (Exception e)
{
log.error("Checking txIntervalCommitTimeServiceAvailable failed.", e);
// Try invoking getNextTxCommitTime service
try
{
client.getNextTxCommitTime(coreName, 0l);
nextTxCommitTimeServiceAvailable = true;
}
catch (NoSuchMethodException e)
{
log.warn("nextTxCommitTimeService is not available. Upgrade your ACS Repository version in order to use this feature: {} ", e.getMessage());
}
catch (Exception e)
{
log.error("Checking nextTxCommitTimeService failed.", e);
}
// Try invoking txIntervalCommitTime service
if (shardMethod.equals(DB_ID_RANGE))
{
try
{
client.getTxIntervalCommitTime(coreName, 0l, 0l);
txIntervalCommitTimeServiceAvailable = true;
}
catch (NoSuchMethodException e)
{
log.warn("txIntervalCommitTimeServiceAvailable is not available. Upgrade your ACS Repository version " +
"to use this feature with DB_ID_RANGE sharding: {} ", e.getMessage());
}
catch (Exception e)
{
log.error("Checking txIntervalCommitTimeServiceAvailable failed.", e);
}
}
}
}
@@ -218,7 +258,7 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
state.setCheckedFirstTransactionTime(true);
log.info("No transactions found - no verification required");
firstTransactions = client.getTransactions(null, 0L, null, Long.MAX_VALUE, 1);
firstTransactions = client.getTransactions(null, minTxnIdRange.getFirst(), null, minTxnIdRange.getSecond(), 1);
if (!firstTransactions.getTransactions().isEmpty())
{
Transaction firstTransaction = firstTransactions.getTransactions().get(0);
@@ -230,30 +270,58 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
if (!state.isCheckedFirstTransactionTime())
{
firstTransactions = client.getTransactions(null, 0L, null, Long.MAX_VALUE, 1);
if (!firstTransactions.getTransactions().isEmpty())
// On Shards configured with DB_ID_RANGE, the first indexed transaction can be
// different from the first transaction in the repository as some transactions
// are skipped if they are not related with the range of the Shard.
// Getting the minCommitTime for the Shard is enough in order to check
// that the first transaction is present.
long minCommitTime = 0l;
if (docRouter instanceof DBIDRangeRouter && txIntervalCommitTimeServiceAvailable)
{
Transaction firstTransaction = firstTransactions.getTransactions().get(0);
long firstTxId = firstTransaction.getId();
long firstTransactionCommitTime = firstTransaction.getCommitTimeMs();
int setSize = this.infoSrv.getTxDocsSize(""+firstTxId, ""+firstTransactionCommitTime);
if (setSize == 0)
try
{
log.error("First transaction was not found with the correct timestamp.");
log.error("SOLR has successfully connected to your repository however the SOLR indexes and repository database do not match.");
log.error("If this is a new or rebuilt database your SOLR indexes also need to be re-built to match the database.");
log.error("You can also check your SOLR connection details in solrcore.properties.");
throw new AlfrescoRuntimeException("Initial transaction not found with correct timestamp");
DBIDRangeRouter dbIdRangeRouter = (DBIDRangeRouter) docRouter;
Pair<Long, Long> commitTimes = client.getTxIntervalCommitTime(coreName,
dbIdRangeRouter.getStartRange(), dbIdRangeRouter.getEndRange());
minCommitTime = commitTimes.getFirst();
}
else if (setSize == 1)
catch (NoSuchMethodException e)
{
state.setCheckedFirstTransactionTime(true);
log.info("Verified first transaction and timestamp in index");
log.warn("txIntervalCommitTimeServiceAvailable is not available. If you are using DB_ID_RANGE shard method, "
+ "upgrade your ACS Repository version in order to use the skip transactions feature: {} ", e.getMessage());
}
else
}
// When a Shard with DB_ID_RANGE method is empty, minCommitTime is -1.
// No firstTransaction checking is required for this case.
if (minCommitTime != -1l) {
firstTransactions = client.getTransactions(minCommitTime, 0L, null, 2000l, 1);
if (!firstTransactions.getTransactions().isEmpty())
{
log.warn("Duplicate initial transaction found with correct timestamp");
Transaction firstTransaction = firstTransactions.getTransactions().get(0);
long firstTxId = firstTransaction.getId();
long firstTransactionCommitTime = firstTransaction.getCommitTimeMs();
int setSize = this.infoSrv.getTxDocsSize(""+firstTxId, ""+firstTransactionCommitTime);
if (setSize == 0)
{
log.error("First transaction was not found with the correct timestamp.");
log.error("SOLR has successfully connected to your repository however the SOLR indexes and repository database do not match.");
log.error("If this is a new or rebuilt database your SOLR indexes also need to be re-built to match the database.");
log.error("You can also check your SOLR connection details in solrcore.properties.");
throw new AlfrescoRuntimeException("Initial transaction not found with correct timestamp");
}
else if (setSize == 1)
{
state.setCheckedFirstTransactionTime(true);
log.info("Verified first transaction and timestamp in index");
}
else
{
log.warn("Duplicate initial transaction found with correct timestamp");
}
}
}
}
@@ -263,7 +331,7 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
{
if (firstTransactions == null)
{
firstTransactions = client.getTransactions(null, 0L, null, Long.MAX_VALUE, 1);
firstTransactions = client.getTransactions(null, minTxnIdRange.getFirst(), null, minTxnIdRange.getSecond(), 1);
}
setLastTxCommitTimeAndTxIdInTrackerState(firstTransactions, state);
@@ -334,10 +402,15 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
// Index the transaction doc after the node - if this is not found then a reindex will be done.
this.infoSrv.indexTransaction(info, false);
log.info("INDEX ACTION - Transaction {} has been indexed", transactionId);
requiresCommit = true;
trackerStats.addTxDocs(nodes.size());
}
else
{
log.info("INDEX ACTION - Transaction {} was not found in database, it has NOT been reindexed", transactionId);
}
}
if (docCount > batchCount)
@@ -377,6 +450,7 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
node.setTxnId(Long.MAX_VALUE);
this.infoSrv.indexNode(node, false);
log.info("INDEX ACTION - Node {} has been reindexed", node.getId());
requiresCommit = true;
}
checkShutdown();
@@ -427,6 +501,11 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
// Index the transaction doc after the node - if this is not found then a reindex will be done.
this.infoSrv.indexTransaction(info, true);
log.info("REINDEX ACTION - Transaction {} has been reindexed", transactionId);
}
else
{
log.info("REINDEX ACTION - Transaction {} was not found in database, it has NOT been reindexed", transactionId);
}
}
@@ -469,6 +548,7 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
node.setTxnId(Long.MAX_VALUE);
this.infoSrv.indexNode(node, true);
log.info("REINDEX ACTION - Node {} has been reindexed", node.getId());
requiresCommit = true;
}
checkShutdown();
@@ -490,6 +570,7 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
if (query != null)
{
this.infoSrv.reindexNodeByQuery(query);
log.info("REINDEX ACTION - Nodes from query {} have been reindexed", query);
requiresCommit = true;
}
checkShutdown();
@@ -514,6 +595,7 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
// make sure it is cleaned out so we do not miss deletes
this.infoSrv.deleteByTransactionId(transactionId);
requiresCommit = true;
log.info("PURGE ACTION - Purged transactionId {}", transactionId);
}
checkShutdown();
}
@@ -534,6 +616,7 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
{
// make sure it is cleaned out so we do not miss deletes
this.infoSrv.deleteByNodeId(nodeId);
log.info("PURGE ACTION - Purged nodeId {}", nodeId);
}
checkShutdown();
}
@@ -676,6 +759,7 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
// When using DB_ID_RANGE, fromCommitTime cannot be before the commit time of the first transaction
// for the DB_ID_RANGE to be indexed and commit time of the last transaction cannot be lower than fromCommitTime.
// When there isn't nodes in that range, -1 is returned as commit times
boolean shardOutOfRange = false;
if (docRouter instanceof DBIDRangeRouter && txIntervalCommitTimeServiceAvailable)
{
@@ -688,23 +772,24 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
// Node Range it's not still available in repository
if (shardMinCommitTime == -1)
{
log.debug("#### [DB_ID_RANGE] No nodes in range [" + dbIdRangeRouter.getStartRange() + "-"
+ dbIdRangeRouter.getEndRange() + "] "
+ "exist in the repository. Skipping metadata tracking.");
return;
log.debug(
"#### [DB_ID_RANGE] No nodes in range [{}-{}] "
+ "exist in the repository. Indexing only latest transaction.",
dbIdRangeRouter.getStartRange(), dbIdRangeRouter.getEndRange());
shardOutOfRange = true;
}
if (fromCommitTime > shardMaxCommitTime)
{
log.debug("#### [DB_ID_RANGE] Last commit time is greater that max commit time in in range ["
+ dbIdRangeRouter.getStartRange() + "-" + dbIdRangeRouter.getEndRange() + "]. "
+ "Skipping metadata tracking.");
return;
log.debug("#### [DB_ID_RANGE] Last commit time is greater that max commit time in in range [{}-{}]. "
+ "Indexing only latest transaction.",
dbIdRangeRouter.getStartRange(), dbIdRangeRouter.getEndRange());
shardOutOfRange = true;
}
// Initial commit time for Node Range is greater than calculated from commit time
if (fromCommitTime < shardMinCommitTime)
{
log.debug("#### [DB_ID_RANGE] SKIPPING TRANSACTIONS FROM " + fromCommitTime + " TO "
+ shardMinCommitTime);
log.debug("#### [DB_ID_RANGE] Skipping transactions from {} to {}",
fromCommitTime, shardMinCommitTime);
fromCommitTime = shardMinCommitTime;
}
}
@@ -712,6 +797,21 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
log.debug("#### Get txn from commit time: " + fromCommitTime);
transactions = getSomeTransactions(txnsFound, fromCommitTime, TIME_STEP_1_HR_IN_MS, 2000,
state.getTimeToStopIndexing());
// When transactions are out of Shard range, only the latest transaction needs to be indexed
// in order to preserve the state up-to-date of the MetadataTracker
if (shardOutOfRange)
{
Transaction latestTransaction = new Transaction();
latestTransaction.setCommitTimeMs(transactions.getMaxTxnCommitTime());
latestTransaction.setId(transactions.getMaxTxnId());
transactions = new Transactions(
Arrays.asList(latestTransaction),
transactions.getMaxTxnCommitTime(),
transactions.getMaxTxnId());
log.debug("#### [DB_ID_RANGE] Latest transaction to be indexed {}", latestTransaction);
}
setLastTxCommitTimeAndTxIdInTrackerState(transactions, state);
@@ -870,9 +970,9 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
private int indexBatchOfTransactions(List<Transaction> txBatch) throws AuthenticationException, IOException, JSONException
{
int nodeCount = 0;
ArrayList<Transaction> nonEmptyTxs = new ArrayList<>(txBatch.size());
List<Transaction> nonEmptyTxs = new ArrayList<>(txBatch.size());
GetNodesParameters gnp = new GetNodesParameters();
ArrayList<Long> txIds = new ArrayList<Long>();
List<Long> txIds = new ArrayList<>();
for (Transaction tx : txBatch)
{
if (tx.getUpdates() > 0 || tx.getDeletes() > 0)
@@ -886,7 +986,7 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
gnp.setStoreProtocol(storeRef.getProtocol());
gnp.setStoreIdentifier(storeRef.getIdentifier());
updateShardProperty();
shardProperty.ifPresent(p -> gnp.setShardProperty(p));
shardProperty.ifPresent(gnp::setShardProperty);
gnp.setCoreName(coreName);
List<Node> nodes = client.getNodes(gnp, Integer.MAX_VALUE);
@@ -936,29 +1036,27 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
List<Node> filteredNodes = filterNodes(nodes);
if(filteredNodes.size() > 0)
{
this.infoServer.indexNodes(filteredNodes, true, false);
this.infoServer.indexNodes(filteredNodes, true);
}
}
@Override
protected void onFail()
protected void onFail(Throwable failCausedBy)
{
setRollback(true);
setRollback(true, failCausedBy);
}
private List<Node> filterNodes(List<Node> nodes)
{
ArrayList<Node> filteredList = new ArrayList<Node>(nodes.size());
List<Node> filteredList = new ArrayList<>(nodes.size());
for(Node node : nodes)
{
if(docRouter.routeNode(shardCount, shardInstance, node))
{
filteredList.add(node);
}
else
else if (cascadeTrackerEnabled)
{
if(node.getStatus() == SolrApiNodeStatus.UPDATED)
{
Node doCascade = new Node();
@@ -1068,7 +1166,7 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
{
// DB TX Count
long firstTransactionCommitTime = 0;
Transactions firstTransactions = client.getTransactions(null, 0L, null, Long.MAX_VALUE, 1);
Transactions firstTransactions = client.getTransactions(null, 0L, null, 2000l, 1);
if(firstTransactions.getTransactions().size() > 0)
{
Transaction firstTransaction = firstTransactions.getTransactions().get(0);
search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/PropertyRouter.java
+10 -7
View File
@@ -26,11 +26,13 @@ import org.apache.solr.common.util.Hash;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.Collections;
import java.util.Map;
import java.util.Optional;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import static java.util.Collections.emptyMap;
/**
* Routes based on a text property field.
* In this method, the value of some property is hashed and this hash is used to assign the node to a random shard.
@@ -126,12 +128,13 @@ public class PropertyRouter implements DocRouter
}
@Override
public Map<String, String> getProperties(QName shardProperty)
public Map<String, String> getProperties(Optional<QName> shardProperty)
{
return (shardProperty == null ?
Collections.emptyMap() :
Map.of(DocRouterFactory.SHARD_KEY_KEY, shardProperty.getPrefixString(),
DocRouterFactory.SHARD_REGEX_KEY, propertyRegEx));
return shardProperty
.map(QName::getPrefixString)
.map(prefix -> Map.of(
DocRouterFactory.SHARD_KEY_KEY, prefix,
DocRouterFactory.SHARD_REGEX_KEY, propertyRegEx))
.orElse(emptyMap());
}
}
search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/SolrTrackerScheduler.java
+5 -1
View File
@@ -96,7 +96,11 @@ public class SolrTrackerScheduler
String jobName = this.getJobName(tracker, coreName);
JobDataMap jobDataMap = new JobDataMap();
jobDataMap.put(TrackerJob.JOBDATA_TRACKER_KEY, tracker);
JobDetail job = JobBuilder.newJob(TrackerJob.class).withIdentity(jobName, SOLR_JOB_GROUP).setJobData(jobDataMap).build();
JobDetail job =
JobBuilder.newJob(TrackerJob.class)
.withIdentity(jobName, SOLR_JOB_GROUP)
.withDescription(jobName)
.setJobData(jobDataMap).build();
Trigger trigger;
try
{
search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/Tracker.java
+3 -1
View File
@@ -41,9 +41,11 @@ public interface Tracker
boolean getRollback();
Throwable getRollbackCausedBy();
Properties getProps();
void setRollback(boolean rollback);
void setRollback(boolean rollback, Throwable rollbackCausedBy);
void invalidateState();
search-services/alfresco-search/src/main/java/org/alfresco/solr/transformer/AlfrescoFieldMapperTransformer.java
+133
View File
@@ -0,0 +1,133 @@
/*
* Copyright (C) 2005-2020 Alfresco Software Limited.
*
* This file is part of Alfresco
*
* Alfresco is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Alfresco is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
*/
package org.alfresco.solr.transformer;
import org.alfresco.solr.AlfrescoSolrDataModel;
import org.apache.lucene.index.IndexableField;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.response.DocsStreamer;
import org.apache.solr.response.ResultContext;
import org.apache.solr.response.transform.DocTransformer;
import org.apache.solr.schema.SchemaField;
import org.apache.solr.search.SolrReturnFields;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.Collection;
import java.util.stream.Collectors;
import static java.util.Optional.of;
/**
* @author Andy, Elia
*
*/
public class AlfrescoFieldMapperTransformer extends DocTransformer
{
protected final static Logger LOGGER = LoggerFactory.getLogger(AlfrescoFieldMapperTransformer.class);
private ResultContext context;
private SolrReturnFields solrReturnFields;
@SuppressWarnings("unchecked")
@Override
public void transform(SolrDocument doc, int docid, float score)
{
Collection<String> fieldNames = new ArrayList<>(doc.getFieldNames());
solrReturnFields = new SolrReturnFields(context.getRequest().getParams().get("originalFl"), context.getRequest());
for (String fieldName : fieldNames)
{
SchemaField schemaField = context.getSearcher().getSchema().getFieldOrNull(fieldName);
if(schemaField != null)
{
String alfrescoFieldName = AlfrescoSolrDataModel.getInstance().getAlfrescoPropertyFromSchemaField(fieldName);
if (isRequestedField(alfrescoFieldName) || alfrescoFieldName.equals("id"))
{
Object value = doc.getFieldValue(fieldName);
doc.removeFields(fieldName);
if (schemaField.multiValued())
{
Object collectionValue =
((Collection<Object>) value).stream()
.map(elem -> getFieldValue(schemaField, elem))
.collect(Collectors.toSet());
doc.setField(alfrescoFieldName, collectionValue);
}
else
{
doc.setField(transformToUnderscoreNotation(alfrescoFieldName), getFieldValue(schemaField, value));
}
}
else
{
doc.removeFields(alfrescoFieldName);
doc.removeFields(fieldName);
}
}
}
}
@Override
public String getName()
{
return "fmap";
}
@Override
public void setContext( ResultContext context )
{
this.context = context;
}
private boolean isRequestedField(String fieldName)
{
return solrReturnFields.wantsField(transformToUnderscoreNotation(fieldName));
}
private String transformToUnderscoreNotation(String value)
{
return value.replace(":", "_");
}
private String removeLocale(String value)
{
int start = value.lastIndexOf('\u0000');
if(start == -1)
{
return value;
}
else
{
return value.substring(start + 1);
}
}
private Object getFieldValue(SchemaField schemaField, Object value)
{
if (value instanceof IndexableField)
{
Object indexedValue = DocsStreamer.getValue(schemaField, (IndexableField) value);
return indexedValue instanceof String ? removeLocale((String) indexedValue) : indexedValue;
}
return value;
}
}
search-services/alfresco-search/src/main/java/org/alfresco/solr/transformer/CachedDocTransformerFactory.java → search-services/alfresco-search/src/main/java/org/alfresco/solr/transformer/AlfrescoFieldMapperTransformerFactory.java
+33 -42
View File
@@ -1,42 +1,33 @@
/*
* Copyright (C) 2005-2014 Alfresco Software Limited.
*
* This file is part of Alfresco
*
* Alfresco is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Alfresco is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
*/
package org.alfresco.solr.transformer;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.response.transform.DocTransformer;
import org.apache.solr.response.transform.TransformerFactory;
/**
* @author Andy
*
*/
public class CachedDocTransformerFactory extends TransformerFactory
{
/* (non-Javadoc)
* @see org.apache.solr.response.transform.TransformerFactory#create(java.lang.String, org.apache.solr.common.params.SolrParams, org.apache.solr.request.SolrQueryRequest)
*/
@Override
public DocTransformer create(String field, SolrParams params, SolrQueryRequest req)
{
return new CachedDocTransformer();
}
}
/*
* Copyright (C) 2005-2014 Alfresco Software Limited.
*
* This file is part of Alfresco
*
* Alfresco is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Alfresco is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
*/
package org.alfresco.solr.transformer;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.response.transform.DocTransformer;
import org.apache.solr.response.transform.TransformerFactory;
public class AlfrescoFieldMapperTransformerFactory extends TransformerFactory
{
@Override
public DocTransformer create(String field, SolrParams params, SolrQueryRequest req)
{
return new AlfrescoFieldMapperTransformer();
}
}
search-services/alfresco-search/src/main/java/org/alfresco/solr/transformer/CachedDocTransformer.java
-197
View File
@@ -1,197 +0,0 @@
/*
* Copyright (C) 2005-2014 Alfresco Software Limited.
*
* This file is part of Alfresco
*
* Alfresco is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Alfresco is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
*/
package org.alfresco.solr.transformer;
import static org.alfresco.repo.search.adaptor.lucene.QueryConstants.FIELD_SOLR4_ID;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import org.alfresco.solr.AlfrescoCoreAdminHandler;
import org.alfresco.solr.AlfrescoSolrDataModel;
import org.alfresco.solr.AlfrescoSolrDataModel.TenantAclIdDbId;
import org.alfresco.solr.SolrInformationServer;
import org.alfresco.solr.content.SolrContentStore;
import org.apache.lucene.index.IndexableField;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.core.CoreContainer;
import org.apache.solr.response.ResultContext;
import org.apache.solr.response.transform.DocTransformer;
import org.apache.solr.schema.SchemaField;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* @author Andy
*
*/
public class CachedDocTransformer extends DocTransformer
{
protected final static Logger log = LoggerFactory.getLogger(CachedDocTransformer.class);
private ResultContext context;
/* (non-Javadoc)
* @see org.apache.solr.response.transform.DocTransformer#getName()
*/
@Override
public String getName()
{
return "Alfresco cached document transformer";
}
public void setContext( ResultContext context )
{
this.context = context;
}
/* (non-Javadoc)
* @see org.apache.solr.response.transform.DocTransformer#transform(org.apache.solr.common.SolrDocument, int)
*/
@Override
public void transform(SolrDocument doc, int docid, float score) throws IOException
{
SolrInputDocument cachedDoc = null;
try
{
String id = getFieldValueString(doc, FIELD_SOLR4_ID);
TenantAclIdDbId tenantAndDbId = AlfrescoSolrDataModel.decodeNodeDocumentId(id);
CoreContainer coreContainer = context.getSearcher().getCore().getCoreContainer();
AlfrescoCoreAdminHandler coreAdminHandler = (AlfrescoCoreAdminHandler) coreContainer.getMultiCoreHandler();
SolrInformationServer srv = (SolrInformationServer) coreAdminHandler.getInformationServers().get(context.getSearcher().getCore().getName());
SolrContentStore solrContentStore = srv.getSolrContentStore();
cachedDoc = solrContentStore.retrieveDocFromSolrContentStore(tenantAndDbId.tenant, tenantAndDbId.dbId);
}
catch(StringIndexOutOfBoundsException e)
{
// ignore invalid forms ....
}
if(cachedDoc != null)
{
Collection<String> fieldNames = cachedDoc.getFieldNames();
for (String fieldName : fieldNames)
{
SchemaField schemaField = context.getSearcher().getSchema().getFieldOrNull(fieldName);
if(schemaField != null)
{
doc.removeFields(fieldName);
if(schemaField.multiValued())
{
int index = fieldName.lastIndexOf("@{");
if(index == -1)
{
doc.addField(fieldName, cachedDoc.getFieldValues(fieldName));
}
else
{
String alfrescoFieldName = AlfrescoSolrDataModel.getInstance().getAlfrescoPropertyFromSchemaField(fieldName);
Collection<Object> values = cachedDoc.getFieldValues(fieldName);
//Guard against null pointer in case data model field name does not match up with cachedDoc field name.
if(values != null) {
ArrayList<Object> newValues = new ArrayList<Object>(values.size());
for (Object value : values) {
if (value instanceof String) {
String stringValue = (String) value;
int start = stringValue.lastIndexOf('\u0000');
if (start == -1) {
newValues.add(stringValue);
} else {
newValues.add(stringValue.substring(start + 1));
}
} else {
newValues.add(value);
}
}
doc.removeFields(alfrescoFieldName);
doc.addField(alfrescoFieldName, newValues);
}
}
}
else
{
int index = fieldName.lastIndexOf("@{");
if(index == -1)
{
doc.addField(fieldName, cachedDoc.getFieldValue(fieldName));
}
else
{
String alfrescoFieldName = AlfrescoSolrDataModel.getInstance().getAlfrescoPropertyFromSchemaField(fieldName);
alfrescoFieldName = alfrescoFieldName.contains(":") ? alfrescoFieldName.replace(":", "_") : alfrescoFieldName;
Object value = cachedDoc.getFieldValue(fieldName);
if(value instanceof String)
{
String stringValue = (String) value;
int start = stringValue.lastIndexOf('\u0000');
if(start == -1)
{
doc.removeFields(alfrescoFieldName);
doc.addField(alfrescoFieldName, stringValue);
}
else
{
doc.removeFields(alfrescoFieldName);
doc.addField(alfrescoFieldName, stringValue.substring(start+1));
}
}
else
{
doc.removeFields(alfrescoFieldName);
doc.addField(alfrescoFieldName, value);
}
}
}
}
}
}
}
private String getFieldValueString(SolrDocument doc, String fieldName)
{
Object o = doc.getFieldValue(fieldName);
if(o != null)
{
if(o instanceof IndexableField)
{
IndexableField field = (IndexableField)o;
return field.stringValue();
}
else if(o instanceof String)
{
return (String)o;
}
else
{
return o.toString();
}
}
else
{
return null;
}
}
}
search-services/alfresco-search/src/main/java/org/alfresco/solr/content/InitialisableAccessMode.java → search-services/alfresco-search/src/main/java/org/alfresco/solr/utils/ThrowingConsumer.java
+24 -11
View File
@@ -16,18 +16,31 @@
* You should have received a copy of the GNU Lesser General Public License
* along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
*/
package org.alfresco.solr.content;
package org.alfresco.solr.utils;
import java.util.function.Consumer;
/**
* {@link AccessMode} specialisation for those modes that need some kind of initialisation.
*
* @author Andrea Gazzarini
* @since 1.5
* Class used to use a {@link java.util.function.Consumer} throwing a checked exception
*/
public interface InitialisableAccessMode extends AccessMode
@FunctionalInterface
public interface ThrowingConsumer<T, E extends Throwable>
{
/**
* Initialises this access mode instance.
*/
void init();
}
void accept(T t) throws E;
static <T, E extends Throwable> Consumer<T> execute(ThrowingConsumer<T, E> consumer)
{
return t -> {
try
{
consumer.accept(t);
}
catch (Throwable e)
{
throw new RuntimeException(e);
}
};
}
}
search-services/alfresco-search/src/main/java/org/alfresco/solr/utils/Utils.java
+77 -3
View File
@@ -18,14 +18,22 @@
*/
package org.alfresco.solr.utils;
import static java.util.Collections.emptyList;
import static java.util.Optional.ofNullable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.Closeable;
import java.io.IOException;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
/**
* SearchServices Booch utilities.
*
* @author Andrea Gazzarini
*/
public abstract class Utils
{
private static final Logger LOGGER = LoggerFactory.getLogger(Utils.class);
@@ -40,7 +48,32 @@ public abstract class Utils
*/
public static <T> Collection<T> notNullOrEmpty(Collection<T> values)
{
return values != null ? values : Collections.emptyList();
return values != null ? values : emptyList();
}
/**
* Returns the same input list if that is not null, otherwise a new empty list.
* Provides a safe way for iterating over a returned list (which could be null).
*
* @param values the input list.
* @param <T> the list elements type.
* @return the same input list if that is not null, otherwise a new empty list.
*/
public static <T> List<T> notNullOrEmpty(List<T> values)
{
return values != null ? values : emptyList();
}
/**
* Makes sure we are not dealing with a null array.
*
* @param values the input array.
* @param <T> the array type.
* @return the input array if it is not null, an empty array otherwise.
*/
public static <T> T[] notNullOrEmpty(final T[] values)
{
return values != null ? values : emptyList().toArray(values);
}
/**
@@ -97,4 +130,45 @@ public abstract class Utils
LOGGER.warn("Unable to properly close the resource instance {}. See the stacktrace below for further details.", resource, ignore);
}
}
}
/**
* Returns true if the input string is null or is empty.
* Note whitespaces are not considered, so if a string contains only whitespaces, it is considered empty.
*
* @param value the input string.
* @return true if the input string is null or is empty.
*/
public static boolean isNullOrEmpty(String value)
{
return ofNullable(value)
.map(String::trim)
.map(String::isEmpty)
.orElse(true);
}
/**
* Returns true if the first character of the input string is the locale marker character.
*
* @param value the input string.
* @return true true if the first character of the input string is the locale marker character.
*/
public static boolean startsWithLanguageMarker(String value)
{
return ofNullable(value)
.map(v -> v.charAt(0) == '\u0000')
.orElse(false);
}
public static Double doubleOrNull(String value)
{
try
{
return Double.parseDouble(value);
}
catch(Exception exception)
{
LOGGER.error("Input string >{}< cannot be converted in a valid double ", value);
return null;
}
}
}
search-services/alfresco-search/src/main/java/org/apache/solr/handler/component/AlfrescoSearchHandler.java
+15
View File
@@ -18,6 +18,12 @@
*/
package org.apache.solr.handler.component;
import static java.util.Optional.of;
import static java.util.Optional.ofNullable;
import static org.alfresco.solr.AlfrescoSolrDataModel.FieldUse.FACET;
import static org.alfresco.solr.AlfrescoSolrDataModel.FieldUse.FTS;
import static org.alfresco.solr.AlfrescoSolrDataModel.FieldUse.ID;
import static org.alfresco.solr.AlfrescoSolrDataModel.FieldUse.SORT;
import static org.apache.solr.common.params.CommonParams.PATH;
import java.io.BufferedReader;
@@ -28,9 +34,13 @@ import java.io.Reader;
import java.io.StringWriter;
import java.lang.invoke.MethodHandles;
import java.util.*;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.alfresco.error.AlfrescoRuntimeException;
import org.alfresco.solr.AlfrescoSolrDataModel;
import org.alfresco.solr.query.AbstractQParser;
import org.apache.cxf.transport.http.auth.HttpAuthHeader;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.ExitableDirectoryReader;
import org.apache.lucene.index.IndexableField;
@@ -58,6 +68,7 @@ import org.apache.solr.schema.SchemaField;
import org.apache.solr.search.DocIterator;
import org.apache.solr.search.DocList;
import org.apache.solr.search.SolrQueryTimeoutImpl;
import org.apache.solr.search.SolrReturnFields;
import org.apache.solr.search.facet.FacetModule;
import org.apache.solr.util.RTimerTree;
import org.apache.solr.util.SolrPluginUtils;
@@ -287,11 +298,15 @@ public class AlfrescoSearchHandler extends RequestHandlerBase implements
return shardHandler;
}
@Override
public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp)
throws Exception {
readJsonIntoContent(req);
List<SearchComponent> components = getComponents();
ResponseBuilder rb = new ResponseBuilder(req, rsp, components);
if (rb.requestInfo != null) {
search-services/alfresco-search/src/main/java/org/apache/solr/handler/component/AlfrescoSolrClusteringComponent.java
-400
View File
@@ -1,400 +0,0 @@
/*
* Copyright (C) 2005-2013 Alfresco Software Limited.
*
* This file is part of Alfresco
*
* Alfresco is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Alfresco is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
*/
package org.apache.solr.handler.component;
import static org.alfresco.repo.search.adaptor.lucene.QueryConstants.FIELD_SOLR4_ID;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.Map;
import org.alfresco.solr.AlfrescoCoreAdminHandler;
import org.alfresco.solr.AlfrescoSolrDataModel;
import org.alfresco.solr.AlfrescoSolrDataModel.TenantAclIdDbId;
import org.alfresco.solr.SolrInformationServer;
import org.alfresco.solr.content.SolrContentStore;
import org.apache.commons.lang.StringUtils;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.IndexableField;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.core.CoreContainer;
import org.apache.solr.core.SolrCore;
import org.apache.solr.core.SolrResourceLoader;
import org.apache.solr.handler.clustering.ClusteringEngine;
import org.apache.solr.handler.clustering.ClusteringParams;
import org.apache.solr.handler.clustering.DocumentClusteringEngine;
import org.apache.solr.handler.clustering.SearchClusteringEngine;
import org.apache.solr.handler.clustering.carrot2.CarrotClusteringEngine;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.search.DocIterator;
import org.apache.solr.search.DocList;
import org.apache.solr.search.DocListAndSet;
import org.apache.solr.util.plugin.SolrCoreAware;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.collect.Maps;
/**
* @author Andy
*
*/
public class AlfrescoSolrClusteringComponent extends SearchComponent implements
SolrCoreAware {
private static final Logger log = LoggerFactory.getLogger(MethodHandles
.lookup().lookupClass());
/**
* Base name for all component parameters. This name is also used to
* register this component with SearchHandler.
*/
public static final String COMPONENT_NAME = "clustering";
/**
* Declaration-order list of search clustering engines.
*/
private final LinkedHashMap<String, SearchClusteringEngine> searchClusteringEngines = Maps
.newLinkedHashMap();
/**
* Declaration order list of document clustering engines.
*/
private final LinkedHashMap<String, DocumentClusteringEngine> documentClusteringEngines = Maps
.newLinkedHashMap();
/**
* An unmodifiable view of {@link #searchClusteringEngines}.
*/
private final Map<String, SearchClusteringEngine> searchClusteringEnginesView = Collections
.unmodifiableMap(searchClusteringEngines);
/**
* Initialization parameters temporarily saved here, the component is
* initialized in {@link #inform(SolrCore)} because we need to know the
* core's {@link SolrResourceLoader}.
*
* @see #init(NamedList)
*/
private NamedList<Object> initParams;
@Override
@SuppressWarnings({ "rawtypes", "unchecked" })
public void init(NamedList args) {
this.initParams = args;
super.init(args);
}
@SuppressWarnings("unchecked")
@Override
public void inform(SolrCore core) {
if (initParams != null) {
log.info("Initializing Clustering Engines");
// Our target list of engines, split into search-results and
// document clustering.
SolrResourceLoader loader = core.getResourceLoader();
for (Map.Entry<String, Object> entry : initParams) {
if ("engine".equals(entry.getKey())) {
NamedList<Object> engineInitParams = (NamedList<Object>) entry
.getValue();
Boolean optional = engineInitParams
.getBooleanArg("optional");
optional = (optional == null ? Boolean.FALSE : optional);
String engineClassName = StringUtils.defaultIfBlank(
(String) engineInitParams.get("classname"),
CarrotClusteringEngine.class.getName());
// Instantiate the clustering engine and split to
// appropriate map.
final ClusteringEngine engine = loader.newInstance(
engineClassName, ClusteringEngine.class);
final String name = StringUtils.defaultIfBlank(
engine.init(engineInitParams, core), "");
if (!engine.isAvailable()) {
if (optional) {
log.info("Optional clustering engine not available: "
+ name);
} else {
throw new SolrException(ErrorCode.SERVER_ERROR,
"A required clustering engine failed to initialize, check the logs: "
+ name);
}
}
final ClusteringEngine previousEntry;
if (engine instanceof SearchClusteringEngine) {
previousEntry = searchClusteringEngines.put(name,
(SearchClusteringEngine) engine);
} else if (engine instanceof DocumentClusteringEngine) {
previousEntry = documentClusteringEngines.put(name,
(DocumentClusteringEngine) engine);
} else {
log.warn("Unknown type of a clustering engine for class: "
+ engineClassName);
continue;
}
if (previousEntry != null) {
log.warn("Duplicate clustering engine component named '"
+ name + "'.");
}
}
}
// Set up the default engine key for both types of engines.
setupDefaultEngine("search results clustering",
searchClusteringEngines);
setupDefaultEngine("document clustering", documentClusteringEngines);
log.info("Finished Initializing Clustering Engines");
}
}
@Override
public void prepare(ResponseBuilder rb) throws IOException {
SolrParams params = rb.req.getParams();
if (!params.getBool(COMPONENT_NAME, false)) {
return;
}
}
@Override
public void process(ResponseBuilder rb) throws IOException {
SolrParams params = rb.req.getParams();
if (!params.getBool(COMPONENT_NAME, false)) {
return;
}
final String name = getClusteringEngineName(rb);
boolean useResults = params.getBool(
ClusteringParams.USE_SEARCH_RESULTS, false);
if (useResults == true) {
SearchClusteringEngine engine = searchClusteringEngines.get(name);
if (engine != null) {
checkAvailable(name, engine);
DocListAndSet results = rb.getResults();
Map<SolrDocument, Integer> docIds = Maps
.newHashMapWithExpectedSize(results.docList.size());
SolrDocumentList solrDocList = docListToSolrDocumentList(
results.docList, rb.req, docIds);
Object clusters = engine.cluster(rb.getQuery(), solrDocList,
docIds, rb.req);
rb.rsp.add("clusters", clusters);
} else {
log.warn("No engine named: " + name);
}
}
boolean useCollection = params.getBool(ClusteringParams.USE_COLLECTION,
false);
if (useCollection == true) {
DocumentClusteringEngine engine = documentClusteringEngines
.get(name);
if (engine != null) {
checkAvailable(name, engine);
boolean useDocSet = params.getBool(
ClusteringParams.USE_DOC_SET, false);
NamedList<?> nl = null;
// TODO: This likely needs to be made into a background task
// that runs in an executor
if (useDocSet == true) {
nl = engine.cluster(rb.getResults().docSet, params);
} else {
nl = engine.cluster(params);
}
rb.rsp.add("clusters", nl);
} else {
log.warn("No engine named: " + name);
}
}
}
private void checkAvailable(String name, ClusteringEngine engine) {
if (!engine.isAvailable()) {
throw new SolrException(ErrorCode.SERVER_ERROR,
"Clustering engine declared, but not available, check the logs: "
+ name);
}
}
private String getClusteringEngineName(ResponseBuilder rb) {
return rb.req.getParams().get(ClusteringParams.ENGINE_NAME,
ClusteringEngine.DEFAULT_ENGINE_NAME);
}
public SolrDocumentList docListToSolrDocumentList(DocList docs,
SolrQueryRequest req, Map<SolrDocument, Integer> ids)
throws IOException {
SolrDocumentList list = new SolrDocumentList();
list.setNumFound(docs.matches());
list.setMaxScore(docs.maxScore());
list.setStart(docs.offset());
DocIterator dit = docs.iterator();
while (dit.hasNext()) {
int docid = dit.nextDoc();
Document luceneDoc = req.getSearcher().doc(docid);
SolrInputDocument input = getSolrInputDocument(luceneDoc, req);
SolrDocument doc = new SolrDocument();
for (String fieldName : input.getFieldNames()) {
doc.addField(fieldName, input.getFieldValue(fieldName));
}
doc.addField("score", dit.score());
list.add(doc);
if (ids != null) {
ids.put(doc, Integer.valueOf(docid));
}
}
return list;
}
private SolrInputDocument getSolrInputDocument(Document doc,
SolrQueryRequest req) throws IOException {
try {
String id = getFieldValueString(doc, FIELD_SOLR4_ID);
TenantAclIdDbId tenantAndDbId = AlfrescoSolrDataModel
.decodeNodeDocumentId(id);
CoreContainer coreContainer = req.getSearcher().getCore().getCoreContainer();
AlfrescoCoreAdminHandler coreAdminHandler = (AlfrescoCoreAdminHandler) coreContainer.getMultiCoreHandler();
SolrInformationServer srv = (SolrInformationServer) coreAdminHandler.getInformationServers().get(req.getSearcher().getCore().getName());
SolrContentStore solrContentStore = srv.getSolrContentStore();
SolrInputDocument sid = solrContentStore.retrieveDocFromSolrContentStore(
tenantAndDbId.tenant, tenantAndDbId.dbId);
return sid;
} catch (StringIndexOutOfBoundsException e) {
throw new IOException(e);
}
}
private String getFieldValueString(Document doc, String fieldName) {
IndexableField field = (IndexableField) doc.getField(fieldName);
String value = null;
if (field != null) {
value = field.stringValue();
}
return value;
}
@Override
public void finishStage(ResponseBuilder rb) {
SolrParams params = rb.req.getParams();
if (!params.getBool(COMPONENT_NAME, false)
|| !params.getBool(ClusteringParams.USE_SEARCH_RESULTS, false)) {
return;
}
if (rb.stage == ResponseBuilder.STAGE_GET_FIELDS) {
String name = getClusteringEngineName(rb);
SearchClusteringEngine engine = searchClusteringEngines.get(name);
if (engine != null) {
checkAvailable(name, engine);
SolrDocumentList solrDocList = (SolrDocumentList) rb.rsp
.getValues().get("response");
// TODO: Currently, docIds is set to null in distributed
// environment.
// This causes CarrotParams.PRODUCE_SUMMARY doesn't work.
// To work CarrotParams.PRODUCE_SUMMARY under distributed mode,
// we can choose either one of:
// (a) In each shard, ClusteringComponent produces summary and
// finishStage()
// merges these summaries.
// (b) Adding doHighlighting(SolrDocumentList, ...) method to
// SolrHighlighter and
// making SolrHighlighter uses "external text" rather than
// stored values to produce snippets.
Map<SolrDocument, Integer> docIds = null;
Object clusters = engine.cluster(rb.getQuery(), solrDocList,
docIds, rb.req);
rb.rsp.add("clusters", clusters);
} else {
log.warn("No engine named: " + name);
}
}
}
/**
* @return Expose for tests.
*/
Map<String, SearchClusteringEngine> getSearchClusteringEnginesView() {
return searchClusteringEnginesView;
}
@Override
public String getDescription() {
return "A Clustering component";
}
/**
* Setup the default clustering engine.
*
* @see "https://issues.apache.org/jira/browse/SOLR-5219"
*/
private static <T extends ClusteringEngine> void setupDefaultEngine(
String type, LinkedHashMap<String, T> map) {
// If there's already a default algorithm, leave it as is.
String engineName = ClusteringEngine.DEFAULT_ENGINE_NAME;
T defaultEngine = map.get(engineName);
if (defaultEngine == null || !defaultEngine.isAvailable()) {
// If there's no default algorithm, and there are any algorithms
// available,
// the first definition becomes the default algorithm.
for (Map.Entry<String, T> e : map.entrySet()) {
if (e.getValue().isAvailable()) {
engineName = e.getKey();
defaultEngine = e.getValue();
map.put(ClusteringEngine.DEFAULT_ENGINE_NAME, defaultEngine);
break;
}
}
}
if (defaultEngine != null) {
log.info("Default engine for " + type + ": " + engineName + " ["
+ defaultEngine.getClass().getSimpleName() + "]");
} else {
log.warn("No default engine for " + type + ".");
}
}
}
search-services/alfresco-search/src/main/java/org/apache/solr/handler/component/AlfrescoSolrHighlighter.java
+352 -619
View File
File diff suppressed because it is too large Load Diff
search-services/alfresco-search/src/main/resources/solr/instance/conf/shared.properties
+11 -8
View File
@@ -16,18 +16,21 @@ alfresco.identifier.property.3={http://www.alfresco.org/model/content/1.0}author
alfresco.identifier.property.4={http://www.alfresco.org/model/content/1.0}lockOwner
# Suggestable Propeties
#alfresco.suggestable.property.0={http://www.alfresco.org/model/content/1.0}name
#alfresco.suggestable.property.1={http://www.alfresco.org/model/content/1.0}title
#alfresco.suggestable.property.2={http://www.alfresco.org/model/content/1.0}description
#alfresco.suggestable.property.3={http://www.alfresco.org/model/content/1.0}content
alfresco.suggestable.property.0={http://www.alfresco.org/model/content/1.0}name
alfresco.suggestable.property.1={http://www.alfresco.org/model/content/1.0}title
alfresco.suggestable.property.2={http://www.alfresco.org/model/content/1.0}description
alfresco.suggestable.property.3={http://www.alfresco.org/model/content/1.0}content
# Data types that support cross locale/word splitting/token patterns if tokenised
alfresco.cross.locale.property.0={http://www.alfresco.org/model/content/1.0}name
alfresco.cross.locale.property.1={http://www.alfresco.org/model/content/1.0}lockOwner
# Data types that support cross locale/word splitting/token patterns if tokenised
# alfresco.cross.locale.datatype.0={http://www.alfresco.org/model/dictionary/1.0}text
# alfresco.cross.locale.datatype.1={http://www.alfresco.org/model/dictionary/1.0}content
# alfresco.cross.locale.datatype.2={http://www.alfresco.org/model/dictionary/1.0}mltext
alfresco.cross.locale.datatype.0={http://www.alfresco.org/model/dictionary/1.0}text
alfresco.cross.locale.datatype.1={http://www.alfresco.org/model/dictionary/1.0}content
alfresco.cross.locale.datatype.2={http://www.alfresco.org/model/dictionary/1.0}mltext
alfresco.model.tracker.cron=0/10 * * * * ? *
alfresco.model.tracker.cron=0/10 * * * * ? *
# Whether path queries are enabled.
alfresco.cascade.tracker.enabled=true
search-services/alfresco-search/src/main/resources/solr/instance/templates/noRerank/conf/README.txt
+2
View File
@@ -0,0 +1,2 @@
This configuration set is not aligned with the rerank one.
This because it will be removed as part of https://issues.alfresco.com/jira/browse/SEARCH-2064
search-services/alfresco-search/src/main/resources/solr/instance/templates/noRerank/conf/solrconfig.xml
+14 -8
View File
@@ -315,6 +315,10 @@
<!-- The default high-performance update handler -->
<updateHandler class="solr.DirectUpdateHandler2">
<updateLog>
<str name="dir">${solr.ulog.dir:}</str>
</updateLog>
<!-- Enables a transaction log, used for real-time get, durability, and
and solr cloud replica recovery. The log can grow as big as
uncommitted changes to the index, so use of a hard autoCommit
@@ -835,6 +839,7 @@
<arr name="components">
<str>setLocale</str>
<str>rewriteFacetParameters</str>
<str>rewriteFieldListComponent</str>
<str>consistencyComponent</str>
<str>query</str>
<str>facet</str>
@@ -867,6 +872,7 @@
<arr name="components">
<str>setLocale</str>
<str>rewriteFacetParameters</str>
<str>rewriteFieldListComponent</str>
<str>consistencyComponent</str>
<str>query</str>
<str>facet</str>
@@ -1372,8 +1378,7 @@
http://wiki.apache.org/solr/ClusteringComponent
http://carrot2.github.io/solr-integration-strategies/
-->
<searchComponent name="clustering"
class="org.apache.solr.handler.component.AlfrescoSolrClusteringComponent" >
<searchComponent name="clustering" class="solr.clustering.ClusteringComponent" >
<lst name="engine">
<str name="name">lingo</str>
@@ -1783,13 +1788,10 @@
EditorialMarkerFactory will do exactly that:
<transformer name="qecBooster" class="org.apache.solr.response.transform.EditorialMarkerFactory" />
-->
<transformer name="cached" class="org.alfresco.solr.transformer.CachedDocTransformerFactory" >
</transformer>
<transformer name="dv" class="org.alfresco.solr.transformer.DocValueDocTransformerFactory" >
</transformer>
<!-- CachedDocTransformer has been renamed, but we retained both codes (new and old) for retro-compatibility -->
<transformer name="cached" class="org.alfresco.solr.transformer.AlfrescoFieldMapperTransformerFactory" />
<transformer name="fmap" class="org.alfresco.solr.transformer.AlfrescoFieldMapperTransformerFactory" />
<!-- Legacy config for the admin interface -->
<admin>
@@ -1805,6 +1807,7 @@
<arr name="components">
<str>setLocale</str>
<str>rewriteFacetParameters</str>
<str>rewriteFieldListComponent</str>
<str>consistencyComponent</str>
<str>query</str>
<str>facet</str>
@@ -1851,6 +1854,7 @@
<arr name="components">
<str>setLocale</str>
<str>rewriteFacetParameters</str>
<str>rewriteFieldListComponent</str>
<str>consistencyComponent</str>
<str>query</str>
<str>facet</str>
@@ -1877,6 +1881,7 @@
<arr name="components">
<str>setLocale</str>
<str>rewriteFacetParameters</str>
<str>rewriteFieldListComponent</str>
<str>consistencyComponent</str>
<str>query</str>
<str>facet</str>
@@ -1915,6 +1920,7 @@
<searchComponent name="setLocale" class="org.alfresco.solr.component.SetLocaleComponent" />
<searchComponent name="clearLocale" class="org.alfresco.solr.component.ClearLocaleComponent" />
<searchComponent name="rewriteFacetParameters" class="org.alfresco.solr.component.RewriteFacetParametersComponent" />
<searchComponent name="rewriteFieldListComponent" class="org.alfresco.solr.component.RewriteFieldListComponent" />
<searchComponent name="rewriteFacetCounts" class="org.alfresco.solr.component.RewriteFacetCountsComponent" />
<searchComponent name="setProcessedDenies" class="org.alfresco.solr.component.SetProcessedDeniesComponent" />
<searchComponent name="consistencyComponent" class="org.alfresco.solr.component.ConsistencyComponent" />
search-services/alfresco-search/src/main/resources/solr/instance/templates/noRerank/conf/solrcore.properties
+8
View File
@@ -165,6 +165,7 @@ alfresco.metadata.ignore.datatype.1=app:configurations
alfresco.metadata.skipDescendantDocsForSpecificAspects=false
#alfresco.metadata.ignore.aspect.0=
# The number of matches from the index to include when rewriting wildcard search terms as an OR-ed list.
alfresco.topTermSpanRewriteLimit=1000
#
@@ -181,6 +182,13 @@ solr.suggester.minSecsBetweenBuilds=3600
#
solr.request.content.compress=false
#
# When checking repo and index consistency, first transaction is compared in both Repository and Index repositories.
# In order to get that initial transaction from database, 0-2000 range for txnId should be enough, but this parameter
# can be used when initial transaction Id is greater than 2000.
#
solr.initial.transaction.range=0-2000
#
# Limit the maximum text size of transformed content sent to the index - in bytes
search-services/alfresco-search/src/main/resources/solr/instance/templates/rerank/conf/generated_copy_fields.xml
+322
View File
@@ -0,0 +1,322 @@
<fields>
<dynamicField name="text@s_stored_____s@*" type="localePrefixedField" />
<copyField source="text@s_stored_____s@*" dest="suggest" />
<dynamicField name="text@s_stored_t____@*" type="localePrefixedField" />
<copyField source="text@s_stored_t____@*" dest="text@s__lt@*" />
<dynamicField name="text@s_stored____s_@*" type="localePrefixedField" />
<dynamicField name="text@s_stored___c__@*" type="localePrefixedField" />
<dynamicField name="text@s_stored__s___@*" type="localePrefixedField" />
<copyField source="text@s_stored__s___@*" dest="text@s__l_@*" />
<dynamicField name="text@s_stored_t___s@*" type="localePrefixedField" />
<copyField source="text@s_stored_t___s@*" dest="text@s__lt@*" />
<copyField source="text@s_stored_t___s@*" dest="suggest" />
<dynamicField name="text@s_stored____ss@*" type="localePrefixedField" />
<copyField source="text@s_stored____ss@*" dest="suggest" />
<dynamicField name="text@s_stored___c_s@*" type="localePrefixedField" />
<copyField source="text@s_stored___c_s@*" dest="suggest" />
<dynamicField name="text@s_stored__s__s@*" type="localePrefixedField" />
<copyField source="text@s_stored__s__s@*" dest="text@s__l_@*" />
<copyField source="text@s_stored__s__s@*" dest="suggest" />
<dynamicField name="text@s_stored_t__s_@*" type="localePrefixedField" />
<copyField source="text@s_stored_t__s_@*" dest="text@s__lt@*" />
<dynamicField name="text@s_stored_t_c__@*" type="localePrefixedField" />
<copyField source="text@s_stored_t_c__@*" dest="text@s__lt@*" />
<copyField source="text@s_stored_t_c__@*" dest="text@s___t@*" />
<dynamicField name="text@s_stored_ts___@*" type="localePrefixedField" />
<copyField source="text@s_stored_ts___@*" dest="text@s__lt@*" />
<copyField source="text@s_stored_ts___@*" dest="text@s__l_@*" />
<dynamicField name="text@s_stored___cs_@*" type="localePrefixedField" />
<dynamicField name="text@s_stored__s_s_@*" type="localePrefixedField" />
<copyField source="text@s_stored__s_s_@*" dest="text@s__l_@*" />
<copyField source="text@s_stored__s_s_@*" dest="text@s__sort@*" />
<dynamicField name="text@s_stored__sc__@*" type="localePrefixedField" />
<copyField source="text@s_stored__sc__@*" dest="text@s__l_@*" />
<copyField source="text@s_stored__sc__@*" dest="text@s____@*" />
<dynamicField name="text@s_stored_t__ss@*" type="localePrefixedField" />
<copyField source="text@s_stored_t__ss@*" dest="text@s__lt@*" />
<copyField source="text@s_stored_t__ss@*" dest="suggest" />
<dynamicField name="text@s_stored_t_c_s@*" type="localePrefixedField" />
<copyField source="text@s_stored_t_c_s@*" dest="text@s__lt@*" />
<copyField source="text@s_stored_t_c_s@*" dest="text@s___t@*" />
<copyField source="text@s_stored_t_c_s@*" dest="suggest" />
<dynamicField name="text@s_stored_ts__s@*" type="localePrefixedField" />
<copyField source="text@s_stored_ts__s@*" dest="text@s__lt@*" />
<copyField source="text@s_stored_ts__s@*" dest="text@s__l_@*" />
<copyField source="text@s_stored_ts__s@*" dest="suggest" />
<dynamicField name="text@s_stored___css@*" type="localePrefixedField" />
<copyField source="text@s_stored___css@*" dest="suggest" />
<dynamicField name="text@s_stored__s_ss@*" type="localePrefixedField" />
<copyField source="text@s_stored__s_ss@*" dest="text@s__l_@*" />
<copyField source="text@s_stored__s_ss@*" dest="text@s__sort@*" />
<copyField source="text@s_stored__s_ss@*" dest="suggest" />
<dynamicField name="text@s_stored__sc_s@*" type="localePrefixedField" />
<copyField source="text@s_stored__sc_s@*" dest="text@s__l_@*" />
<copyField source="text@s_stored__sc_s@*" dest="text@s____@*" />
<copyField source="text@s_stored__sc_s@*" dest="suggest" />
<dynamicField name="text@s_stored_t_cs_@*" type="localePrefixedField" />
<copyField source="text@s_stored_t_cs_@*" dest="text@s__lt@*" />
<copyField source="text@s_stored_t_cs_@*" dest="text@s___t@*" />
<dynamicField name="text@s_stored_ts_s_@*" type="localePrefixedField" />
<copyField source="text@s_stored_ts_s_@*" dest="text@s__lt@*" />
<copyField source="text@s_stored_ts_s_@*" dest="text@s__l_@*" />
<copyField source="text@s_stored_ts_s_@*" dest="text@s__sort@*" />
<dynamicField name="text@s_stored_tsc__@*" type="localePrefixedField" />
<copyField source="text@s_stored_tsc__@*" dest="text@s__lt@*" />
<copyField source="text@s_stored_tsc__@*" dest="text@s___t@*" />
<copyField source="text@s_stored_tsc__@*" dest="text@s__l_@*" />
<copyField source="text@s_stored_tsc__@*" dest="text@s____@*" />
<dynamicField name="text@s_stored__scs_@*" type="localePrefixedField" />
<copyField source="text@s_stored__scs_@*" dest="text@s__l_@*" />
<copyField source="text@s_stored__scs_@*" dest="text@s__sort@*" />
<copyField source="text@s_stored__scs_@*" dest="text@s____@*" />
<dynamicField name="text@s_stored_t_css@*" type="localePrefixedField" />
<copyField source="text@s_stored_t_css@*" dest="text@s__lt@*" />
<copyField source="text@s_stored_t_css@*" dest="text@s___t@*" />
<copyField source="text@s_stored_t_css@*" dest="suggest" />
<dynamicField name="text@s_stored_ts_ss@*" type="localePrefixedField" />
<copyField source="text@s_stored_ts_ss@*" dest="text@s__lt@*" />
<copyField source="text@s_stored_ts_ss@*" dest="text@s__l_@*" />
<copyField source="text@s_stored_ts_ss@*" dest="text@s__sort@*" />
<copyField source="text@s_stored_ts_ss@*" dest="suggest" />
<dynamicField name="text@s_stored_tsc_s@*" type="localePrefixedField" />
<copyField source="text@s_stored_tsc_s@*" dest="text@s__lt@*" />
<copyField source="text@s_stored_tsc_s@*" dest="text@s___t@*" />
<copyField source="text@s_stored_tsc_s@*" dest="text@s__l_@*" />
<copyField source="text@s_stored_tsc_s@*" dest="text@s____@*" />
<copyField source="text@s_stored_tsc_s@*" dest="suggest" />
<dynamicField name="text@s_stored__scss@*" type="localePrefixedField" />
<copyField source="text@s_stored__scss@*" dest="text@s__l_@*" />
<copyField source="text@s_stored__scss@*" dest="text@s__sort@*" />
<copyField source="text@s_stored__scss@*" dest="text@s____@*" />
<copyField source="text@s_stored__scss@*" dest="suggest" />
<dynamicField name="text@s_stored_tscs_@*" type="localePrefixedField" />
<copyField source="text@s_stored_tscs_@*" dest="text@s__lt@*" />
<copyField source="text@s_stored_tscs_@*" dest="text@s___t@*" />
<copyField source="text@s_stored_tscs_@*" dest="text@s__l_@*" />
<copyField source="text@s_stored_tscs_@*" dest="text@s__sort@*" />
<copyField source="text@s_stored_tscs_@*" dest="text@s____@*" />
<dynamicField name="text@s_stored_tscss@*" type="localePrefixedField" />
<copyField source="text@s_stored_tscss@*" dest="text@s__lt@*" />
<copyField source="text@s_stored_tscss@*" dest="text@s___t@*" />
<copyField source="text@s_stored_tscss@*" dest="text@s__l_@*" />
<copyField source="text@s_stored_tscss@*" dest="text@s__sort@*" />
<copyField source="text@s_stored_tscss@*" dest="text@s____@*" />
<copyField source="text@s_stored_tscss@*" dest="suggest" />
<dynamicField name="mltext@m_stored__s___@*" type="localePrefixedField" multiValued="true" />
<copyField source="mltext@m_stored__s___@*" dest="mltext@m__l_@*" />
<dynamicField name="mltext@m_stored___c__@*" type="localePrefixedField" multiValued="true" />
<dynamicField name="mltext@m_stored_____s@*" type="localePrefixedField" multiValued="true" />
<copyField source="mltext@m_stored_____s@*" dest="suggest" />
<dynamicField name="mltext@m_stored_t____@*" type="localePrefixedField" multiValued="true" />
<copyField source="mltext@m_stored_t____@*" dest="mltext@m__lt@*" />
<dynamicField name="mltext@m_stored__sc__@*" type="localePrefixedField" multiValued="true" />
<copyField source="mltext@m_stored__sc__@*" dest="mltext@m__l_@*" />
<copyField source="mltext@m_stored__sc__@*" dest="mltext@m____@*" />
<dynamicField name="mltext@m_stored__s__s@*" type="localePrefixedField" multiValued="true" />
<copyField source="mltext@m_stored__s__s@*" dest="mltext@m__l_@*" />
<copyField source="mltext@m_stored__s__s@*" dest="suggest" />
<dynamicField name="mltext@m_stored_ts___@*" type="localePrefixedField" multiValued="true" />
<copyField source="mltext@m_stored_ts___@*" dest="mltext@m__lt@*" />
<copyField source="mltext@m_stored_ts___@*" dest="mltext@m__l_@*" />
<dynamicField name="mltext@m_stored___c_s@*" type="localePrefixedField" multiValued="true" />
<copyField source="mltext@m_stored___c_s@*" dest="suggest" />
<dynamicField name="mltext@m_stored_t_c__@*" type="localePrefixedField" multiValued="true" />
<copyField source="mltext@m_stored_t_c__@*" dest="mltext@m__lt@*" />
<copyField source="mltext@m_stored_t_c__@*" dest="mltext@m___t@*" />
<dynamicField name="mltext@m_stored_t___s@*" type="localePrefixedField" multiValued="true" />
<copyField source="mltext@m_stored_t___s@*" dest="mltext@m__lt@*" />
<copyField source="mltext@m_stored_t___s@*" dest="suggest" />
<dynamicField name="mltext@m_stored__sc_s@*" type="localePrefixedField" multiValued="true" />
<copyField source="mltext@m_stored__sc_s@*" dest="mltext@m__l_@*" />
<copyField source="mltext@m_stored__sc_s@*" dest="mltext@m____@*" />
<copyField source="mltext@m_stored__sc_s@*" dest="suggest" />
<dynamicField name="mltext@m_stored_tsc__@*" type="localePrefixedField" multiValued="true" />
<copyField source="mltext@m_stored_tsc__@*" dest="mltext@m__lt@*" />
<copyField source="mltext@m_stored_tsc__@*" dest="mltext@m___t@*" />
<copyField source="mltext@m_stored_tsc__@*" dest="mltext@m__l_@*" />
<copyField source="mltext@m_stored_tsc__@*" dest="mltext@m____@*" />
<dynamicField name="mltext@m_stored_ts__s@*" type="localePrefixedField" multiValued="true" />
<copyField source="mltext@m_stored_ts__s@*" dest="mltext@m__lt@*" />
<copyField source="mltext@m_stored_ts__s@*" dest="mltext@m__l_@*" />
<copyField source="mltext@m_stored_ts__s@*" dest="suggest" />
<dynamicField name="mltext@m_stored_t_c_s@*" type="localePrefixedField" multiValued="true" />
<copyField source="mltext@m_stored_t_c_s@*" dest="mltext@m__lt@*" />
<copyField source="mltext@m_stored_t_c_s@*" dest="mltext@m___t@*" />
<copyField source="mltext@m_stored_t_c_s@*" dest="suggest" />
<dynamicField name="mltext@m_stored_tsc_s@*" type="localePrefixedField" multiValued="true" />
<copyField source="mltext@m_stored_tsc_s@*" dest="mltext@m__lt@*" />
<copyField source="mltext@m_stored_tsc_s@*" dest="mltext@m___t@*" />
<copyField source="mltext@m_stored_tsc_s@*" dest="mltext@m__l_@*" />
<copyField source="mltext@m_stored_tsc_s@*" dest="mltext@m____@*" />
<copyField source="mltext@m_stored_tsc_s@*" dest="suggest" />
<dynamicField name="content@s_stored__s___@*" type="localePrefixedField" />
<copyField source="content@s_stored__s___@*" dest="content@s__l_@*" />
<dynamicField name="content@s_stored___c__@*" type="localePrefixedField" />
<dynamicField name="content@s_stored_____s@*" type="localePrefixedField" />
<copyField source="content@s_stored_____s@*" dest="suggest" />
<dynamicField name="content@s_stored_t____@*" type="localePrefixedField" />
<copyField source="content@s_stored_t____@*" dest="content@s__lt@*" />
<dynamicField name="content@s_stored__sc__@*" type="localePrefixedField" />
<copyField source="content@s_stored__sc__@*" dest="content@s__l_@*" />
<copyField source="content@s_stored__sc__@*" dest="content@s____@*" />
<dynamicField name="content@s_stored__s__s@*" type="localePrefixedField" />
<copyField source="content@s_stored__s__s@*" dest="content@s__l_@*" />
<copyField source="content@s_stored__s__s@*" dest="suggest" />
<dynamicField name="content@s_stored_ts___@*" type="localePrefixedField" />
<copyField source="content@s_stored_ts___@*" dest="content@s__lt@*" />
<copyField source="content@s_stored_ts___@*" dest="content@s__l_@*" />
<dynamicField name="content@s_stored___c_s@*" type="localePrefixedField" />
<copyField source="content@s_stored___c_s@*" dest="suggest" />
<dynamicField name="content@s_stored_t_c__@*" type="localePrefixedField" />
<copyField source="content@s_stored_t_c__@*" dest="content@s__lt@*" />
<copyField source="content@s_stored_t_c__@*" dest="content@s___t@*" />
<dynamicField name="content@s_stored_t___s@*" type="localePrefixedField" />
<copyField source="content@s_stored_t___s@*" dest="content@s__lt@*" />
<copyField source="content@s_stored_t___s@*" dest="suggest" />
<dynamicField name="content@s_stored__sc_s@*" type="localePrefixedField" />
<copyField source="content@s_stored__sc_s@*" dest="content@s__l_@*" />
<copyField source="content@s_stored__sc_s@*" dest="content@s____@*" />
<copyField source="content@s_stored__sc_s@*" dest="suggest" />
<dynamicField name="content@s_stored_tsc__@*" type="localePrefixedField" />
<copyField source="content@s_stored_tsc__@*" dest="content@s__lt@*" />
<copyField source="content@s_stored_tsc__@*" dest="content@s___t@*" />
<copyField source="content@s_stored_tsc__@*" dest="content@s__l_@*" />
<copyField source="content@s_stored_tsc__@*" dest="content@s____@*" />
<dynamicField name="content@s_stored_ts__s@*" type="localePrefixedField" />
<copyField source="content@s_stored_ts__s@*" dest="content@s__lt@*" />
<copyField source="content@s_stored_ts__s@*" dest="content@s__l_@*" />
<copyField source="content@s_stored_ts__s@*" dest="suggest" />
<dynamicField name="content@s_stored_t_c_s@*" type="localePrefixedField" />
<copyField source="content@s_stored_t_c_s@*" dest="content@s__lt@*" />
<copyField source="content@s_stored_t_c_s@*" dest="content@s___t@*" />
<copyField source="content@s_stored_t_c_s@*" dest="suggest" />
<dynamicField name="content@s_stored_tsc_s@*" type="localePrefixedField" />
<copyField source="content@s_stored_tsc_s@*" dest="content@s__lt@*" />
<copyField source="content@s_stored_tsc_s@*" dest="content@s___t@*" />
<copyField source="content@s_stored_tsc_s@*" dest="content@s__l_@*" />
<copyField source="content@s_stored_tsc_s@*" dest="content@s____@*" />
<copyField source="content@s_stored_tsc_s@*" dest="suggest" />
<dynamicField name="text@m_stored__s___@*" type="localePrefixedField" multiValued="true" />
<copyField source="text@m_stored__s___@*" dest="text@m__l_@*" />
<dynamicField name="text@m_stored___c__@*" type="localePrefixedField" multiValued="true" />
<dynamicField name="text@m_stored_____s@*" type="localePrefixedField" multiValued="true" />
<copyField source="text@m_stored_____s@*" dest="suggest" />
<dynamicField name="text@m_stored_t____@*" type="localePrefixedField" multiValued="true" />
<copyField source="text@m_stored_t____@*" dest="text@m__lt@*" />
<dynamicField name="text@m_stored__sc__@*" type="localePrefixedField" multiValued="true" />
<copyField source="text@m_stored__sc__@*" dest="text@m__l_@*" />
<copyField source="text@m_stored__sc__@*" dest="text@m____@*" />
<dynamicField name="text@m_stored__s__s@*" type="localePrefixedField" multiValued="true" />
<copyField source="text@m_stored__s__s@*" dest="text@m__l_@*" />
<copyField source="text@m_stored__s__s@*" dest="suggest" />
<dynamicField name="text@m_stored_ts___@*" type="localePrefixedField" multiValued="true" />
<copyField source="text@m_stored_ts___@*" dest="text@m__lt@*" />
<copyField source="text@m_stored_ts___@*" dest="text@m__l_@*" />
<dynamicField name="text@m_stored___c_s@*" type="localePrefixedField" multiValued="true" />
<copyField source="text@m_stored___c_s@*" dest="suggest" />
<dynamicField name="text@m_stored_t_c__@*" type="localePrefixedField" multiValued="true" />
<copyField source="text@m_stored_t_c__@*" dest="text@m__lt@*" />
<copyField source="text@m_stored_t_c__@*" dest="text@m___t@*" />
<dynamicField name="text@m_stored_t___s@*" type="localePrefixedField" multiValued="true" />
<copyField source="text@m_stored_t___s@*" dest="text@m__lt@*" />
<copyField source="text@m_stored_t___s@*" dest="suggest" />
<dynamicField name="text@m_stored__sc_s@*" type="localePrefixedField" multiValued="true" />
<copyField source="text@m_stored__sc_s@*" dest="text@m__l_@*" />
<copyField source="text@m_stored__sc_s@*" dest="text@m____@*" />
<copyField source="text@m_stored__sc_s@*" dest="suggest" />
<dynamicField name="text@m_stored_tsc__@*" type="localePrefixedField" multiValued="true" />
<copyField source="text@m_stored_tsc__@*" dest="text@m__lt@*" />
<copyField source="text@m_stored_tsc__@*" dest="text@m___t@*" />
<copyField source="text@m_stored_tsc__@*" dest="text@m__l_@*" />
<copyField source="text@m_stored_tsc__@*" dest="text@m____@*" />
<dynamicField name="text@m_stored_ts__s@*" type="localePrefixedField" multiValued="true" />
<copyField source="text@m_stored_ts__s@*" dest="text@m__lt@*" />
<copyField source="text@m_stored_ts__s@*" dest="text@m__l_@*" />
<copyField source="text@m_stored_ts__s@*" dest="suggest" />
<dynamicField name="text@m_stored_t_c_s@*" type="localePrefixedField" multiValued="true" />
<copyField source="text@m_stored_t_c_s@*" dest="text@m__lt@*" />
<copyField source="text@m_stored_t_c_s@*" dest="text@m___t@*" />
<copyField source="text@m_stored_t_c_s@*" dest="suggest" />
<dynamicField name="text@m_stored_tsc_s@*" type="localePrefixedField" multiValued="true" />
<copyField source="text@m_stored_tsc_s@*" dest="text@m__lt@*" />
<copyField source="text@m_stored_tsc_s@*" dest="text@m___t@*" />
<copyField source="text@m_stored_tsc_s@*" dest="text@m__l_@*" />
<copyField source="text@m_stored_tsc_s@*" dest="text@m____@*" />
<copyField source="text@m_stored_tsc_s@*" dest="suggest" />
</fields>

Some files were not shown because too many files have changed in this diff Show More