inteligr8/SearchServices
1
0
Fork 0
mirror of https://github.com/Alfresco/SearchServices.git synced 2026-09-16 18:12:56 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branches 'feature/SEARCH-2166_DateFunctionWhere' and 'master' of https://git.alfresco.com/search_discovery/insightengine into feature/SEARCH-2166_DateFunctionWhere

Browse Source 
 Conflicts:
	search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoSolrDataModel.java
This commit is contained in:
agazzarini
2020-06-08 11:05:05 +02:00
parent 2487dd2a1d 89a55e1523
commit f32945ae6c
49 changed files with 2867 additions and 758 deletions
Download Patch File Download Diff File Expand all files Collapse all files
e2e-test/generator-alfresco-docker-compose/README.md
+13 -9
View File
@@ -17,8 +17,10 @@ $ tree generators/app/templates/
│   ├── .env
│   ├── docker-compose-ce.yml
│   └── docker-compose-ee.yml
├── empty
│   └── empty
├── latest
│   ├── .env
│   ├── docker-compose-ce.yml
│   └── docker-compose-ee.yml├── empty
├── images
│   ├── alfresco
│   │   ├── Dockerfile
@@ -42,7 +44,7 @@ $ tree generators/app/templates/
│   │   └── sharding-share-config-custom.xml
│   └── zeppelin
│   └── Dockerfile
└── keystores
└── keystores :: 6.x / 7.x
├── alfresco
├── client
├── solr
@@ -50,8 +52,8 @@ $ tree generators/app/templates/
```
* `.env` includes default values for Docker Compose environment variables
* `docker-compose-ce.yml` is the base Docker Compose Template for Alfresco Community deployment (for ACS 6.2 and ACS 6.1)
* `docker-compose-ee.yml` is the base Docker Compose Template for Alfresco Enterprise deployment (for ACS 6.2 and ACS 6.1)
* `docker-compose-ce.yml` is the base Docker Compose Template for Alfresco Community deployment (for latest, ACS 6.2 and ACS 6.1)
* `docker-compose-ee.yml` is the base Docker Compose Template for Alfresco Enterprise deployment (for latest, ACS 6.2 and ACS 6.1)
* `alfresco` includes a Dockerfile template to start Alfresco Repository
* `model` includes a default content model (Sharding Explicit Routing or empty). This `empty` file is required for Dockerfile to work, so it should not be deleted.
* `share` includes a Dockerfile template to start Share Web Application
@@ -59,7 +61,7 @@ $ tree generators/app/templates/
* `search` includes a Dockerfile template to start Search Services and Insight Engine
* `config` includes configuration for HTTP Web Proxy (NGINX)
* `zeppelin` includes a Dockerfile template to start Zeppelin with SSL
* `keystores` includes every truststore and keystore required for SSL configuration
* `keystores` includes every truststore and keystore required for SSL configuration. Included different stores for ACS 6.x and ACS 7.x, as there were changes between this versions
## Installation
@@ -80,15 +82,17 @@ $ yo alfresco-docker-compose
## ACS Version
Currently supported ACS Versions are `6.2` and `6.1`
Currently supported ACS Versions are `latest`, `6.2` and `6.1`
This is the first choice to be selected when the generator is executed.
```
? Which ACS version do you want to use?
6.1
❯ 6.2
6.2
❯ latest
```
## AGS Version
Currently supported AGS Version is `latest` (based in ACS 6.1).
@@ -197,7 +201,7 @@ $ yo alfresco-docker-compose --acsVersion=6.2 --alfrescoVersion=community --http
**Parameter names reference**
`--acsVersion`: default 6.2, but could be set to 6.1
`--acsVersion`: default latest, but could be set to 6.2 or 6.1
`--ags:`: only available when acsVersion=6.1
`--alfrescoVersion`: community or enterprise
`--httpMode`: http or https
e2e-test/generator-alfresco-docker-compose/generators/app/index.js
+11 -12
View File
@@ -21,8 +21,8 @@ module.exports = class extends Generator {
type: 'list',
name: 'acsVersion',
message: 'Which ACS version do you want to use?',
choices: [ "6.1", "6.2" ],
default: '6.2'
choices: [ '6.1', '6.2', 'latest' ],
default: 'latest'
},
{
whenFunction: response => response.acsVersion == '6.1',
@@ -143,10 +143,8 @@ module.exports = class extends Generator {
// Generate boilerplate from "templates" folder
writing() {
var dockerComposeTemplateDirectory = '6.2';
if (this.props.acsVersion.startsWith('6.1')) {
dockerComposeTemplateDirectory = '6.1';
}
// Set base template directory: 6.1, 6.2, latest
var dockerComposeTemplateDirectory = this.props.acsVersion;
// Docker Compose environment variables values
this.fs.copyTpl(
@@ -181,10 +179,10 @@ module.exports = class extends Generator {
(this.props.alfrescoVersion == 'community' ?
(this.props.ags ?
'alfresco/alfresco-governance-share-community' :
'alfresco/alfresco-share') :
'quay.io/alfresco/alfresco-share') :
(this.props.ags ?
'quay.io/alfresco/alfresco-governance-share-enterprise':
'alfresco/alfresco-share'
'quay.io/alfresco/alfresco-share'
)
);
@@ -297,21 +295,22 @@ module.exports = class extends Generator {
// Add resources for SSL configuration
if (this.props.httpMode == 'https') {
var subfolder = this.props.acsVersion == 'latest' ? '7.x' : '6.x'
this.fs.copy(
this.templatePath('keystores/alfresco'),
this.templatePath('keystores/' + subfolder + '/alfresco'),
this.destinationPath('keystores/alfresco')
)
this.fs.copy(
this.templatePath('keystores/solr'),
this.templatePath('keystores/' + subfolder + '/solr'),
this.destinationPath('keystores/solr')
)
this.fs.copy(
this.templatePath('keystores/client'),
this.templatePath('keystores/' + subfolder + '/client'),
this.destinationPath('keystores/client')
)
if (this.props.zeppelin == true) {
this.fs.copy(
this.templatePath('keystores/zeppelin'),
this.templatePath('keystores/' + subfolder + '/zeppelin'),
this.destinationPath('keystores/zeppelin')
)
}
e2e-test/generator-alfresco-docker-compose/generators/app/templates/6.2/.env
+8 -10
View File
@@ -1,15 +1,13 @@
ALFRESCO_TAG=latest
SHARE_TAG=6.2.0
ALFRESCO_TAG=6.2.1
ALFRESCO_CE_TAG=6.2.0-ga
SHARE_TAG=6.2.1
POSTGRES_TAG=11.4
TRANSFORM_ROUTER_TAG=1.1.0
PDF_RENDERER_TAG=2.1.0
IMAGE_MAGICK_TAG=2.1.0
LIBREOFFICE_TAG=2.1.0
TIKA_TAG=2.1.0
TRANSFORM_MISC_TAG=2.1.0
SHARED_FILE_STORE_TAG=0.5.3
TRANSFORM_ROUTER_TAG=1.2.0
TRANSFORM_CORE_AIO_TAG=2.2.1
TRANSFORMER_TAG=2.1.0
SHARED_FILE_STORE_TAG=0.7.0
ACTIVE_MQ_TAG=5.15.8
DIGITAL_WORKSPACE_TAG=1.3.0
DIGITAL_WORKSPACE_TAG=1.5.0
ACS_NGINX_TAG=3.0.1
ACS_COMMUNITY_NGINX_TAG=1.0.0
SEARCH_TAG=latest
e2e-test/generator-alfresco-docker-compose/generators/app/templates/6.2/docker-compose-ce.yml
+6 -6
View File
@@ -7,7 +7,7 @@ services:
build:
context: ./alfresco
args:
ALFRESCO_TAG: ${ALFRESCO_TAG}
ALFRESCO_TAG: ${ALFRESCO_CE_TAG}
SOLR_COMMS: <%=secureComms%> <% if (httpMode == 'https') { %>
TRUSTSTORE_TYPE: JCEKS
TRUSTSTORE_PASS: kT9X6oe68t
@@ -141,7 +141,7 @@ services:
<% } %>
alfresco-pdf-renderer:
image: alfresco/alfresco-pdf-renderer:${PDF_RENDERER_TAG}
image: alfresco/alfresco-pdf-renderer:${TRANSFORMER_TAG}
mem_limit: 1g
environment:
JAVA_OPTS: " -Xms256m -Xmx512m"
@@ -149,7 +149,7 @@ services:
- 8090:8090
imagemagick:
image: alfresco/alfresco-imagemagick:${IMAGE_MAGICK_TAG}
image: alfresco/alfresco-imagemagick:${TRANSFORMER_TAG}
mem_limit: 1g
environment:
JAVA_OPTS: " -Xms256m -Xmx512m"
@@ -157,7 +157,7 @@ services:
- 8091:8090
libreoffice:
image: alfresco/alfresco-libreoffice:${LIBREOFFICE_TAG}
image: alfresco/alfresco-libreoffice:${TRANSFORMER_TAG}
mem_limit: 1g
environment:
JAVA_OPTS: " -Xms256m -Xmx512m"
@@ -165,7 +165,7 @@ services:
- 8092:8090
tika:
image: alfresco/alfresco-tika:${TIKA_TAG}
image: alfresco/alfresco-tika:${TRANSFORMER_TAG}
mem_limit: 1g
environment:
JAVA_OPTS: " -Xms256m -Xmx512m"
@@ -173,7 +173,7 @@ services:
- 8093:8090
transform-misc:
image: alfresco/alfresco-transform-misc:${TRANSFORM_MISC_TAG}
image: alfresco/alfresco-transform-misc:${TRANSFORMER_TAG}
mem_limit: 1g
environment:
JAVA_OPTS: " -Xms256m -Xmx512m"
e2e-test/generator-alfresco-docker-compose/generators/app/templates/6.2/docker-compose-ee.yml
+13 -72
View File
@@ -34,20 +34,12 @@ services:
-Dtransform.service.enabled=true
-Dtransform.service.url=http://transform-router:8095
-Dsfs.url=http://shared-file-store:8099/
-Dlocal.transform.service.enabled=true
-DlocalTransform.pdfrenderer.url=http://alfresco-pdf-renderer:8090/
-DlocalTransform.imagemagick.url=http://imagemagick:8090/
-DlocalTransform.libreoffice.url=http://libreoffice:8090/
-DlocalTransform.tika.url=http://tika:8090/
-DlocalTransform.misc.url=http://transform-misc:8090/
-Dlegacy.transform.service.enabled=true
-Dalfresco-pdf-renderer.url=http://alfresco-pdf-renderer:8090/
-Djodconverter.url=http://libreoffice:8090/
-Dimg.url=http://imagemagick:8090/
-Dtika.url=http://tika:8090/
-Dtransform.misc.url=http://transform-misc:8090/
-DlocalTransform.core-aio.url=http://transform-core-aio:8090/
-Dalfresco-pdf-renderer.url=http://transform-core-aio:8090/
-Djodconverter.url=http://transform-core-aio:8090/
-Dimg.url=http://transform-core-aio:8090/
-Dtika.url=http://transform-core-aio:8090/
-Dtransform.misc.url=http://transform-core-aio:8090/
-Dcsrf.filter.enabled=false
-Dalfresco.restApi.basicAuthScheme=true
@@ -207,75 +199,24 @@ services:
environment:
JAVA_OPTS: " -Xms256m -Xmx512m"
ACTIVEMQ_URL: "nio://activemq:61616"
IMAGEMAGICK_URL: "http://imagemagick:8090"
PDF_RENDERER_URL : "http://alfresco-pdf-renderer:8090"
LIBREOFFICE_URL : "http://libreoffice:8090"
TIKA_URL : "http://tika:8090"
CORE_AIO_URL : "http://transform-core-aio:8090"
FILE_STORE_URL: "http://shared-file-store:8099/alfresco/api/-default-/private/sfs/versions/1/file"
ports:
- 8095:8095
links:
- activemq
alfresco-pdf-renderer:
image: alfresco/alfresco-pdf-renderer:${PDF_RENDERER_TAG}
mem_limit: 1g
transform-core-aio:
image: alfresco/alfresco-transform-core-aio:${TRANSFORM_CORE_AIO_TAG}
mem_limit: 1536m
environment:
JAVA_OPTS: " -Xms256m -Xmx512m"
JAVA_OPTS: " -Xms256m -Xmx1536m"
ACTIVEMQ_URL: "nio://activemq:61616"
FILE_STORE_URL: "http://shared-file-store:8099/alfresco/api/-default-/private/sfs/versions/1/file"
ports:
- 8090:8090
links:
- activemq
imagemagick:
image: alfresco/alfresco-imagemagick:${IMAGE_MAGICK_TAG}
mem_limit: 1g
environment:
JAVA_OPTS: " -Xms256m -Xmx512m"
ACTIVEMQ_URL: "nio://activemq:61616"
FILE_STORE_URL: "http://shared-file-store:8099/alfresco/api/-default-/private/sfs/versions/1/file"
ports:
- 8091:8090
links:
- activemq
libreoffice:
image: alfresco/alfresco-libreoffice:${LIBREOFFICE_TAG}
mem_limit: 1g
environment:
JAVA_OPTS: " -Xms256m -Xmx512m"
ACTIVEMQ_URL: "nio://activemq:61616"
FILE_STORE_URL: "http://shared-file-store:8099/alfresco/api/-default-/private/sfs/versions/1/file"
ports:
- 8092:8090
links:
- activemq
tika:
image: alfresco/alfresco-tika:${TIKA_TAG}
mem_limit: 1g
environment:
JAVA_OPTS: " -Xms256m -Xmx512m"
ACTIVEMQ_URL: "nio://activemq:61616"
FILE_STORE_URL: "http://shared-file-store:8099/alfresco/api/-default-/private/sfs/versions/1/file"
ports:
- 8093:8090
links:
- activemq
transform-misc:
image: alfresco/alfresco-transform-misc:${TRANSFORM_MISC_TAG}
mem_limit: 1g
environment:
JAVA_OPTS: " -Xms256m -Xmx512m"
ACTIVEMQ_URL: "nio://activemq:61616"
FILE_STORE_URL: "http://shared-file-store:8099/alfresco/api/-default-/private/sfs/versions/1/file"
ports:
- 8094:8090
links:
- activemq
- activemq
shared-file-store:
image: alfresco/alfresco-shared-file-store:${SHARED_FILE_STORE_TAG}
@@ -320,7 +261,7 @@ services:
activemq:
image: alfresco/alfresco-activemq:${ACTIVE_MQ_TAG}
mem_limit: 512m
mem_limit: 1g
ports:
- 8161:8161 # Web Console
- 5672:5672 # AMQP
e2e-test/generator-alfresco-docker-compose/generators/app/templates/images/search/Dockerfile
+21 -8
View File
@@ -32,16 +32,29 @@ ENV TRUSTSTORE_TYPE $TRUSTSTORE_TYPE
ARG KEYSTORE_TYPE
ENV KEYSTORE_TYPE $KEYSTORE_TYPE
# Passwords using Env Vars
ARG PASSWORDS_AS_ENV_VARS
ENV PASSWORDS_AS_ENV_VARS $PASSWORDS_AS_ENV_VARS
# Set SSL properties
RUN if [ "$ALFRESCO_COMMS" == "https" ] ; then \
sed -i '/^bash.*/i \
sed -i "'"s/alfresco.encryption.ssl.keystore.location=.*/alfresco.encryption.ssl.keystore.location=\\\/opt\\\/<%=searchPath%>\\\/keystore\\\/ssl.repo.client.keystore/g"'" ${DIST_DIR}/solrhome/templates/rerank/conf/solrcore.properties && \
sed -i "'"s/alfresco.encryption.ssl.keystore.passwordFileLocation=.*/alfresco.encryption.ssl.keystore.passwordFileLocation=\\\/opt\\\/<%=searchPath%>\\\/keystore\\\/ssl-keystore-passwords.properties/g"'" ${DIST_DIR}/solrhome/templates/rerank/conf/solrcore.properties && \
sed -i "'"s/alfresco.encryption.ssl.keystore.type=.*/alfresco.encryption.ssl.keystore.type=${KEYSTORE_TYPE}/g"'" ${DIST_DIR}/solrhome/templates/rerank/conf/solrcore.properties && \
sed -i "'"s/alfresco.encryption.ssl.truststore.location=.*/alfresco.encryption.ssl.truststore.location=\\\/opt\\\/<%=searchPath%>\\\/keystore\\\/ssl.repo.client.truststore/g"'" ${DIST_DIR}/solrhome/templates/rerank/conf/solrcore.properties && \
sed -i "'"s/alfresco.encryption.ssl.truststore.passwordFileLocation=.*/alfresco.encryption.ssl.truststore.passwordFileLocation=\\\/opt\\\/<%=searchPath%>\\\/keystore\\\/ssl-truststore-passwords.properties/g"'" ${DIST_DIR}/solrhome/templates/rerank/conf/solrcore.properties && \
sed -i "'"s/alfresco.encryption.ssl.truststore.type=.*/alfresco.encryption.ssl.truststore.type=${TRUSTSTORE_TYPE}/g"'" ${DIST_DIR}/solrhome/templates/rerank/conf/solrcore.properties' \
${DIST_DIR}/solr/bin/search_config_setup.sh; \
if [ "$PASSWORDS_AS_ENV_VARS" == "true" ]; then \
sed -i '/^bash.*/i \
sed -i "'"s/alfresco.encryption.ssl.keystore.location=.*/alfresco.encryption.ssl.keystore.location=\\\/opt\\\/<%=searchPath%>\\\/keystore\\\/ssl-repo-client.keystore/g"'" ${DIST_DIR}/solrhome/templates/rerank/conf/solrcore.properties && \
sed -i "'"s/alfresco.encryption.ssl.keystore.type=.*/alfresco.encryption.ssl.keystore.type=${KEYSTORE_TYPE}/g"'" ${DIST_DIR}/solrhome/templates/rerank/conf/solrcore.properties && \
sed -i "'"s/alfresco.encryption.ssl.truststore.location=.*/alfresco.encryption.ssl.truststore.location=\\\/opt\\\/<%=searchPath%>\\\/keystore\\\/ssl-repo-client.truststore/g"'" ${DIST_DIR}/solrhome/templates/rerank/conf/solrcore.properties && \
sed -i "'"s/alfresco.encryption.ssl.truststore.type=.*/alfresco.encryption.ssl.truststore.type=${TRUSTSTORE_TYPE}/g"'" ${DIST_DIR}/solrhome/templates/rerank/conf/solrcore.properties' \
${DIST_DIR}/solr/bin/search_config_setup.sh; \
else \
sed -i '/^bash.*/i \
sed -i "'"s/alfresco.encryption.ssl.keystore.location=.*/alfresco.encryption.ssl.keystore.location=\\\/opt\\\/<%=searchPath%>\\\/keystore\\\/ssl.repo.client.keystore/g"'" ${DIST_DIR}/solrhome/templates/rerank/conf/solrcore.properties && \
sed -i "'"s/alfresco.encryption.ssl.keystore.passwordFileLocation=.*/alfresco.encryption.ssl.keystore.passwordFileLocation=\\\/opt\\\/<%=searchPath%>\\\/keystore\\\/ssl-keystore-passwords.properties/g"'" ${DIST_DIR}/solrhome/templates/rerank/conf/solrcore.properties && \
sed -i "'"s/alfresco.encryption.ssl.keystore.type=.*/alfresco.encryption.ssl.keystore.type=${KEYSTORE_TYPE}/g"'" ${DIST_DIR}/solrhome/templates/rerank/conf/solrcore.properties && \
sed -i "'"s/alfresco.encryption.ssl.truststore.location=.*/alfresco.encryption.ssl.truststore.location=\\\/opt\\\/<%=searchPath%>\\\/keystore\\\/ssl.repo.client.truststore/g"'" ${DIST_DIR}/solrhome/templates/rerank/conf/solrcore.properties && \
sed -i "'"s/alfresco.encryption.ssl.truststore.passwordFileLocation=.*/alfresco.encryption.ssl.truststore.passwordFileLocation=\\\/opt\\\/<%=searchPath%>\\\/keystore\\\/ssl-truststore-passwords.properties/g"'" ${DIST_DIR}/solrhome/templates/rerank/conf/solrcore.properties && \
sed -i "'"s/alfresco.encryption.ssl.truststore.type=.*/alfresco.encryption.ssl.truststore.type=${TRUSTSTORE_TYPE}/g"'" ${DIST_DIR}/solrhome/templates/rerank/conf/solrcore.properties' \
${DIST_DIR}/solr/bin/search_config_setup.sh; \
fi \
fi
# REPLICATION
e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/alfresco/keystore → e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/6.x/alfresco/keystore
View File
e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/alfresco/keystore-passwords.properties → e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/6.x/alfresco/keystore-passwords.properties
View File
e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/alfresco/ssl-keystore-passwords.properties → e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/6.x/alfresco/ssl-keystore-passwords.properties
View File
e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/alfresco/ssl-truststore-passwords.properties → e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/6.x/alfresco/ssl-truststore-passwords.properties
View File
e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/alfresco/ssl.keystore → e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/6.x/alfresco/ssl.keystore
View File
e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/alfresco/ssl.truststore → e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/6.x/alfresco/ssl.truststore
View File
e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/client/browser.p12 → e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/6.x/client/browser.p12
View File
e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/solr/ssl-keystore-passwords.properties → e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/6.x/solr/ssl-keystore-passwords.properties
View File
e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/solr/ssl-truststore-passwords.properties → e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/6.x/solr/ssl-truststore-passwords.properties
View File
e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/solr/ssl.repo.client.keystore → e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/6.x/solr/ssl.repo.client.keystore
View File
e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/solr/ssl.repo.client.truststore → e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/6.x/solr/ssl.repo.client.truststore
View File
e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/zeppelin/ssl.repo.client.keystore → e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/6.x/zeppelin/ssl.repo.client.keystore
View File
e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/zeppelin/ssl.repo.client.truststore → e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/6.x/zeppelin/ssl.repo.client.truststore
View File
e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/7.x/alfresco/keystore
BIN
View File
Binary file not shown.
e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/7.x/alfresco/ssl.keystore
BIN
View File
Binary file not shown.
e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/7.x/alfresco/ssl.truststore
BIN
View File
Binary file not shown.
e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/7.x/client/browser.p12
BIN
View File
Binary file not shown.
e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/7.x/solr/ssl-repo-client.keystore
BIN
View File
Binary file not shown.
e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/7.x/solr/ssl-repo-client.truststore
BIN
View File
Binary file not shown.
e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/7.x/zeppelin/ssl.repo.client.keystore
BIN
View File
Binary file not shown.
e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/7.x/zeppelin/ssl.repo.client.truststore
BIN
View File
Binary file not shown.
e2e-test/generator-alfresco-docker-compose/generators/app/templates/latest/.env
Executable
+14
View File
@@ -0,0 +1,14 @@
ALFRESCO_TAG=latest
ALFRESCO_CE_TAG=latest
SHARE_TAG=latest
POSTGRES_TAG=11.4
TRANSFORM_ROUTER_TAG=1.2.0
TRANSFORM_CORE_AIO_TAG=2.2.1
SHARED_FILE_STORE_TAG=0.7.0
ACTIVE_MQ_TAG=5.15.8
DIGITAL_WORKSPACE_TAG=1.5.0
ACS_NGINX_TAG=3.0.1
ACS_COMMUNITY_NGINX_TAG=1.0.0
SEARCH_TAG=latest
ZEPPELIN_TAG=latest
ACA_TAG=master-latest
e2e-test/generator-alfresco-docker-compose/generators/app/templates/latest/docker-compose-ce.yml
Executable
+206
View File
@@ -0,0 +1,206 @@
# Using version 2 as 3 does not support resource constraint options (cpu_*, mem_* limits) for non swarm mode in Compose
version: "2"
services:
alfresco:
build:
context: ./alfresco
args:
ALFRESCO_TAG: ${ALFRESCO_CE_TAG}
SOLR_COMMS: <%=secureComms%> <% if (httpMode == 'https') { %>
TRUSTSTORE_TYPE: JCEKS
TRUSTSTORE_PASS: kT9X6oe68t
KEYSTORE_TYPE: JCEKS
KEYSTORE_PASS: kT9X6oe68t <% } %>
COMPRESS_CONTENT: "<%=gzip%>"
mem_limit: 1800m
environment:
JAVA_OPTS : "
-Ddb.driver=org.postgresql.Driver
-Ddb.username=alfresco
-Ddb.password=alfresco
-Ddb.url=jdbc:postgresql://postgres:5432/alfresco
-Dsolr.host=<%=searchSolrHost%>
-Dsolr.port.ssl=8983
-Dsolr.secureComms=<%=secureComms%>
-Dsolr.baseUrl=/solr
-Dindex.subsystem.name=solr6
-Dshare.host=localhost
-Dalfresco.port=8080
-Daos.baseUrlOverwrite=http://localhost:8080/alfresco/aos
-Dmessaging.broker.url=\"failover:(nio://activemq:61616)?timeout=3000&jms.useCompression=true\"
-Ddeployment.method=DOCKER_COMPOSE
-DlocalTransform.core-aio.url=http://transform-core-aio:8090/
-Dalfresco-pdf-renderer.url=http://transform-core-aio:8090/
-Djodconverter.url=http://transform-core-aio:8090/
-Dimg.url=http://transform-core-aio:8090/
-Dtika.url=http://transform-core-aio:8090/
-Dtransform.misc.url=http://transform-core-aio:8090/
-Dcsrf.filter.enabled=false
-Dalfresco.restApi.basicAuthScheme=true
-Xms1500m -Xmx1500m
" <% if (httpMode == 'https') { %>
ports:
- 8443:8443
volumes:
- ./keystores/alfresco:/usr/local/tomcat/alf_data/keystore <% } %>
solr6:
build:
context: ./search
args:
SEARCH_TAG: ${SEARCH_TAG}
SOLR_HOSTNAME: solr6
ALFRESCO_HOSTNAME: alfresco
ALFRESCO_COMMS: <%=secureComms%> <% if (httpMode == 'https') { %>
TRUSTSTORE_TYPE: JCEKS
KEYSTORE_TYPE: JCEKS <% } %> <% if (replication) { %>
ENABLE_MASTER: "true"
ENABLE_SLAVE: "false" <% } %>
COMPRESS_CONTENT: "<%=gzip%>"
mem_limit: 1200m
environment:
#Solr needs to know how to register itself with Alfresco
SOLR_ALFRESCO_HOST: "alfresco"
SOLR_ALFRESCO_PORT: "<%=alfrescoPort%>"
ALFRESCO_SECURE_COMMS: <%=secureComms%>
#Alfresco needs to know how to call solr
SOLR_SOLR_HOST: "solr6"
SOLR_SOLR_PORT: "8983"
#Create the default alfresco and archive cores
SOLR_CREATE_ALFRESCO_DEFAULTS: "alfresco,archive"
SOLR_JAVA_MEM: "-Xms1g -Xmx1g" <% if (httpMode == 'https') { %>
SOLR_SSL_TRUST_STORE: "/opt/<%=searchPath%>/keystore/ssl.repo.client.truststore"
SOLR_SSL_TRUST_STORE_PASSWORD: "kT9X6oe68t"
SOLR_SSL_TRUST_STORE_TYPE: "JCEKS"
SOLR_SSL_KEY_STORE: "/opt/<%=searchPath%>/keystore/ssl.repo.client.keystore"
SOLR_SSL_KEY_STORE_PASSWORD: "kT9X6oe68t"
SOLR_SSL_KEY_STORE_TYPE: "JCEKS"
SOLR_SSL_NEED_CLIENT_AUTH: "true"
SOLR_OPTS: "
-Dsolr.ssl.checkPeerName=false
-Dsolr.allow.unsafe.resourceloading=true
" <% } %>
ports:
- 8083:8983 <% if (httpMode == 'https') { %>
volumes:
- ./keystores/solr:/opt/<%=searchPath%>/keystore <% } %>
<% if (replication) { %>
solr6secondary:
build:
context: ./search
args:
SEARCH_TAG: ${SEARCH_TAG}
SOLR_HOSTNAME: solr6secondary
ALFRESCO_HOSTNAME: alfresco
ALFRESCO_COMMS: <%=secureComms%> <% if (httpMode == 'https') { %>
TRUSTSTORE_TYPE: JCEKS
KEYSTORE_TYPE: JCEKS <% } %> <% if (replication) { %>
ENABLE_MASTER: <% if (replication == 'master-master') { %>"true"<% } else { %>"false"<% } %>
ENABLE_SLAVE: <% if (replication == 'master-master') { %>"false"<% } else { %>"true"<% } %>
MASTER_HOST: solr6 <% } %>
COMPRESS_CONTENT: "<%=gzip%>"
mem_limit: 1200m
environment:
#Solr needs to know how to register itself with Alfresco
SOLR_ALFRESCO_HOST: "alfresco"
SOLR_ALFRESCO_PORT: "<%=alfrescoPort%>"
ALFRESCO_SECURE_COMMS: <%=secureComms%>
#Alfresco needs to know how to call solr
SOLR_SOLR_HOST: "solr6secondary"
SOLR_SOLR_PORT: "8983"
#Create the default alfresco and archive cores
SOLR_CREATE_ALFRESCO_DEFAULTS: "alfresco,archive"
SOLR_JAVA_MEM: "-Xms1g -Xmx1g" <% if (httpMode == 'https') { %>
SOLR_SSL_TRUST_STORE: "/opt/<%=searchPath%>/keystore/ssl.repo.client.truststore"
SOLR_SSL_TRUST_STORE_PASSWORD: "kT9X6oe68t"
SOLR_SSL_TRUST_STORE_TYPE: "JCEKS"
SOLR_SSL_KEY_STORE: "/opt/<%=searchPath%>/keystore/ssl.repo.client.keystore"
SOLR_SSL_KEY_STORE_PASSWORD: "kT9X6oe68t"
SOLR_SSL_KEY_STORE_TYPE: "JCEKS"
SOLR_SSL_NEED_CLIENT_AUTH: "true"
SOLR_OPTS: "
-Dsolr.ssl.checkPeerName=false
-Dsolr.allow.unsafe.resourceloading=true
" <% } %>
ports:
- 8084:8983 <% if (httpMode == 'https') { %>
volumes:
- ./keystores/solr:/opt/<%=searchPath%>/keystore <% } %>
<% } %>
transform-core-aio:
image: alfresco/alfresco-transform-core-aio:${TRANSFORM_CORE_AIO_TAG}
mem_limit: 1536m
environment:
JAVA_OPTS: " -Xms256m -Xmx1536m"
ports:
- 8090:8090
share:
build:
context: ./share
args:
SHARE_TAG: ${SHARE_TAG}
mem_limit: 1g
environment:
REPO_HOST: "alfresco"
REPO_PORT: "8080"
JAVA_OPTS: "
-Xms500m
-Xmx500m
-Dalfresco.host=localhost
-Dalfresco.port=8080
-Dalfresco.context=alfresco
-Dalfresco.protocol=http
"
postgres:
image: postgres:${POSTGRES_TAG}
mem_limit: 512m
environment:
- POSTGRES_PASSWORD=alfresco
- POSTGRES_USER=alfresco
- POSTGRES_DB=alfresco
command: postgres -c max_connections=300 -c log_min_messages=LOG
ports:
- 5432:5432
activemq:
image: alfresco/alfresco-activemq:${ACTIVE_MQ_TAG}
mem_limit: 1g
ports:
- 8161:8161 # Web Console
- 5672:5672 # AMQP
- 61616:61616 # OpenWire
- 61613:61613 # STOMP
content-app:
image: alfresco/alfresco-content-app:${ACA_TAG}
mem_limit: 256m
depends_on:
- alfresco
# HTTP proxy to provide HTTP Default port access to services
# SOLR API and SOLR Web Console are protected to avoid unauthenticated access
proxy:
image: nginx:stable-alpine
mem_limit: 128m
depends_on:
- alfresco
volumes:
- ./config/nginx.conf:/etc/nginx/nginx.conf
- ./config/nginx.htpasswd:/etc/nginx/conf.d/nginx.htpasswd <% if (httpWebMode == 'https') { %>
- ./config/cert/localhost.cer:/etc/nginx/localhost.cer
- ./config/cert/localhost.key:/etc/nginx/localhost.key <% } %>
ports:
- <%=port%>:<%=port%>
links:
- alfresco
- share
- solr6
- content-app
e2e-test/generator-alfresco-docker-compose/generators/app/templates/latest/docker-compose-ee.yml
Executable
+338
View File
@@ -0,0 +1,338 @@
# Using version 2 as 3 does not support resource constraint options (cpu_*, mem_* limits) for non swarm mode in Compose
version: "2"
services:
alfresco:
build:
context: ./alfresco
args:
ALFRESCO_TAG: ${ALFRESCO_TAG}
SOLR_COMMS: <%=secureComms%> <% if (httpMode == 'https') { %>
TRUSTSTORE_TYPE: JCEKS
TRUSTSTORE_PASS: kT9X6oe68t
KEYSTORE_TYPE: JCEKS
KEYSTORE_PASS: kT9X6oe68t <% } %>
COMPRESS_CONTENT: "<%=gzip%>"
mem_limit: 1800m
environment:
JAVA_OPTS: "
-Ddb.driver=org.postgresql.Driver
-Ddb.username=alfresco
-Ddb.password=alfresco
-Ddb.url=jdbc:postgresql://postgres:5432/alfresco
-Dsolr.host=<%=searchSolrHost%>
-Dsolr.port.ssl=8983
-Dsolr.secureComms=<%=secureComms%>
-Dsolr.baseUrl=/solr <% if (sharding == 'true') { %>
-Dsolr.useDynamicShardRegistration=true <% } %>
-Dindex.subsystem.name=solr6
-Daos.baseUrlOverwrite=http://localhost:8080/alfresco/aos
-Dmessaging.broker.url=\"failover:(nio://activemq:61616)?timeout=3000&jms.useCompression=true\"
-Ddeployment.method=DOCKER_COMPOSE
-Dmetadata-keystore.password=mp6yc0UD9e
-Dmetadata-keystore.aliases=metadata
-Dmetadata-keystore.metadata.password=mp6yc0UD9e
-Dmetadata-keystore.metadata.algorithm=AES <% if (httpMode == 'https') { %>
-Dencryption.keystore.type=pkcs12
-Dencryption.cipherAlgorithm=AES/CBC/PKCS5Padding
-Dencryption.keyAlgorithm=AES <% } %>
-Dtransform.service.enabled=true
-Dtransform.service.url=http://transform-router:8095
-Dsfs.url=http://shared-file-store:8099/
-DlocalTransform.core-aio.url=http://transform-core-aio:8090/
-Dalfresco-pdf-renderer.url=http://transform-core-aio:8090/
-Djodconverter.url=http://transform-core-aio:8090/
-Dimg.url=http://transform-core-aio:8090/
-Dtika.url=http://transform-core-aio:8090/
-Dtransform.misc.url=http://transform-core-aio:8090/
-Dcsrf.filter.enabled=false
-Dalfresco.restApi.basicAuthScheme=true
-Xms1500m -Xmx1500m
<% if (httpMode == 'https') { %>
-Dssl-keystore.password=kT9X6oe68t
-Dssl-keystore.aliases=ssl-alfresco-ca,ssl-repo
-Dssl-keystore.ssl-alfresco-ca.password=kT9X6oe68t
-Dssl-keystore.ssl-repo.password=kT9X6oe68t
-Dssl-truststore.password=kT9X6oe68t
-Dssl-truststore.aliases=alfresco-ca,ssl-repo-client
-Dssl-truststore.alfresco-ca.password=kT9X6oe68t
-Dssl-truststore.ssl-repo-client.password=kT9X6oe68t <% } %>
" <% if (httpMode == 'https') { %>
ports:
- 8443:8443
volumes:
- ./keystores/alfresco:/usr/local/tomcat/alf_data/keystore <% } %>
solr6:
build:
context: ./search
args:
SEARCH_TAG: ${SEARCH_TAG}
SOLR_HOSTNAME: solr6
ALFRESCO_HOSTNAME: alfresco
ALFRESCO_COMMS: <%=secureComms%> <% if (httpMode == 'https') { %>
TRUSTSTORE_TYPE: JCEKS
KEYSTORE_TYPE: JCEKS
PASSWORDS_AS_ENV_VARS: "true" <% } %> <% if (replication) { %>
ENABLE_MASTER: "true"
ENABLE_SLAVE: "false" <% } %> <% if (sharding == 'true') { %>
ENABLE_SHARDING: "true"
NUM_SHARDS: "2"
SHARD_ID: "0" <% if (shardingMethod == 'DB_ID_RANGE') { %>
SHARD_RANGE: "0-800" <% } %> <% if (shardingMethod == 'DATE') { %>
SHARD_KEY: "cm:created"
SHARD_DATE_GROUPING: "2" <% } %> <% if (shardingMethod == 'PROPERTY') { %>
SHARD_KEY: "shard:shardId" <% } %> <% if (shardingMethod == 'EXPLICIT_ID' || shardingMethod == 'EXPLICIT_ID_FALLBACK_LRIS') { %>
SHARD_KEY: "shard:shardId" <% } %>
SHARDING_METHOD: <%=shardingMethod%> <% } %>
COMPRESS_CONTENT: "<%=gzip%>"
mem_limit: 1200m
environment:
#Solr needs to know how to register itself with Alfresco
SOLR_ALFRESCO_HOST: "alfresco"
SOLR_ALFRESCO_PORT: "<%=alfrescoPort%>"
ALFRESCO_SECURE_COMMS: <%=secureComms%>
#Alfresco needs to know how to call solr
SOLR_SOLR_HOST: "solr6"
SOLR_SOLR_PORT: "8983"
#Create the default alfresco and archive cores
SOLR_CREATE_ALFRESCO_DEFAULTS: "alfresco,archive"
SOLR_JAVA_MEM: "-Xms1g -Xmx1g" <% if (httpMode == 'https') { %>
SOLR_SSL_TRUST_STORE: "/opt/<%=searchPath%>/keystore/ssl-repo-client.truststore"
SOLR_SSL_TRUST_STORE_PASSWORD: "kT9X6oe68t"
SOLR_SSL_TRUST_STORE_TYPE: "JCEKS"
SOLR_SSL_KEY_STORE: "/opt/<%=searchPath%>/keystore/ssl-repo-client.keystore"
SOLR_SSL_KEY_STORE_PASSWORD: "kT9X6oe68t"
SOLR_SSL_KEY_STORE_TYPE: "JCEKS"
SOLR_SSL_NEED_CLIENT_AUTH: "true" <% if (sharding == 'true') { %>
SOLR_SSL_CLIENT_KEY_STORE: "/opt/<%=searchPath%>/keystore/ssl-repo-client.keystore"
SOLR_SSL_CLIENT_KEY_STORE_PASSWORD: "kT9X6oe68t"
SOLR_SSL_CLIENT_KEY_STORE_TYPE: "JCEKS"
SOLR_SSL_CLIENT_TRUST_STORE: "/opt/<%=searchPath%>/keystore/ssl-repo-client.keystore"
SOLR_SSL_CLIENT_TRUST_STORE_PASSWORD: "kT9X6oe68t"
SOLR_SSL_CLIENT_TRUST_STORE_TYPE: "JCEKS" <% } %>
SOLR_OPTS: "
-Dsolr.ssl.checkPeerName=false
-Dsolr.allow.unsafe.resourceloading=true
-Dssl-keystore.password=kT9X6oe68t
-Dssl-keystore.aliases=ssl-alfresco-ca,ssl-repo-client
-Dssl-keystore.ssl-alfresco-ca.password=kT9X6oe68t
-Dssl-keystore.ssl-repo-client.password=kT9X6oe68t
-Dssl-truststore.password=kT9X6oe68t
-Dssl-truststore.aliases=ssl-alfresco-ca,ssl-repo,ssl-repo-client
-Dssl-truststore.ssl-alfresco-ca.password=kT9X6oe68t
-Dssl-truststore.ssl-repo.password=kT9X6oe68t
-Dssl-truststore.ssl-repo-client.password=kT9X6oe68t
" <% } %>
ports:
- 8083:8983 <% if (httpMode == 'https') { %>
volumes:
- ./keystores/solr:/opt/<%=searchPath%>/keystore <% } %>
<% if (sharding == 'true' || replication) { %>
solr6secondary:
build:
context: ./search
args:
SEARCH_TAG: ${SEARCH_TAG}
SOLR_HOSTNAME: solr6secondary
ALFRESCO_HOSTNAME: alfresco
ALFRESCO_COMMS: <%=secureComms%> <% if (httpMode == 'https') { %>
TRUSTSTORE_TYPE: JCEKS
KEYSTORE_TYPE: JCEKS
PASSWORDS_AS_ENV_VARS: "true" <% } %> <% if (replication) { %>
ENABLE_MASTER: <% if (replication == 'master-master') { %>"true"<% } else { %>"false"<% } %>
ENABLE_SLAVE: <% if (replication == 'master-master') { %>"false"<% } else { %>"true"<% } %>
MASTER_HOST: solr6 <% } %> <% if (sharding == 'true') { %>
ENABLE_SHARDING: "true"
NUM_SHARDS: "2"
SHARD_ID: "1" <% if (shardingMethod == 'DB_ID_RANGE') { %>
SHARD_RANGE: "801-40000" <% } %> <% if (shardingMethod == 'DATE') { %>
SHARD_KEY: "cm:created"
SHARD_DATE_GROUPING: "2" <% } %> <% if (shardingMethod == 'PROPERTY') { %>
SHARD_KEY: "shard:shardId" <% } %> <% if (shardingMethod == 'EXPLICIT_ID' || shardingMethod == 'EXPLICIT_ID_FALLBACK_LRIS') { %>
SHARD_KEY: "shard:shardId" <% } %>
SHARDING_METHOD: <%=shardingMethod%> <% } %>
COMPRESS_CONTENT: "<%=gzip%>"
mem_limit: 1200m
environment:
#Solr needs to know how to register itself with Alfresco
SOLR_ALFRESCO_HOST: "alfresco"
SOLR_ALFRESCO_PORT: "<%=alfrescoPort%>"
ALFRESCO_SECURE_COMMS: <%=secureComms%>
#Alfresco needs to know how to call solr
SOLR_SOLR_HOST: "solr6secondary"
SOLR_SOLR_PORT: "8983"
#Create the default alfresco and archive cores
SOLR_CREATE_ALFRESCO_DEFAULTS: "alfresco,archive"
SOLR_JAVA_MEM: "-Xms1g -Xmx1g" <% if (httpMode == 'https') { %>
SOLR_SSL_TRUST_STORE: "/opt/<%=searchPath%>/keystore/ssl-repo-client.truststore"
SOLR_SSL_TRUST_STORE_PASSWORD: "kT9X6oe68t"
SOLR_SSL_TRUST_STORE_TYPE: "JCEKS"
SOLR_SSL_KEY_STORE: "/opt/<%=searchPath%>/keystore/ssl-repo-client.keystore"
SOLR_SSL_KEY_STORE_PASSWORD: "kT9X6oe68t"
SOLR_SSL_KEY_STORE_TYPE: "JCEKS"
SOLR_SSL_NEED_CLIENT_AUTH: "true" <% if (sharding == 'true') { %>
SOLR_SSL_CLIENT_KEY_STORE: "/opt/<%=searchPath%>/keystore/ssl-repo-client.keystore"
SOLR_SSL_CLIENT_KEY_STORE_PASSWORD: "kT9X6oe68t"
SOLR_SSL_CLIENT_KEY_STORE_TYPE: "JCEKS"
SOLR_SSL_CLIENT_TRUST_STORE: "/opt/<%=searchPath%>/keystore/ssl-repo-client.keystore"
SOLR_SSL_CLIENT_TRUST_STORE_PASSWORD: "kT9X6oe68t"
SOLR_SSL_CLIENT_TRUST_STORE_TYPE: "JCEKS" <% } %>
SOLR_OPTS: "
-Dsolr.ssl.checkPeerName=false
-Dsolr.allow.unsafe.resourceloading=true
-Dssl-keystore.password=kT9X6oe68t
-Dssl-keystore.aliases=ssl-alfresco-ca,ssl-repo-client
-Dssl-keystore.ssl-alfresco-ca.password=kT9X6oe68t
-Dssl-keystore.ssl-repo-client.password=kT9X6oe68t
-Dssl-truststore.password=kT9X6oe68t
-Dssl-truststore.aliases=ssl-alfresco-ca,ssl-repo,ssl-repo-client
-Dssl-truststore.ssl-alfresco-ca.password=kT9X6oe68t
-Dssl-truststore.ssl-repo.password=kT9X6oe68t
-Dssl-truststore.ssl-repo-client.password=kT9X6oe68t
" <% } %>
ports:
- 8084:8983 <% if (httpMode == 'https') { %>
volumes:
- ./keystores/solr:/opt/<%=searchPath%>/keystore <% } %>
<% } %>
<% if (zeppelin == 'true') { %>
zeppelin:
build:
context: ./zeppelin
args:
ZEPPELIN_TAG: ${ZEPPELIN_TAG}
ALFRESCO_COMMS: <%=secureComms%>
mem_limit: 768m
environment: <% if (httpMode == 'https') { %>
REPO_PROTOCOL: "https" <% } %>
REPO_HOST: "alfresco"
REPO_PORT: "<%=alfrescoPort%>" <% if (httpMode == 'https') { %>
JAVA_OPTS: "
-Dalfresco.enable.ssl=true
-Dsolr.ssl.checkPeerName=false
-Djavax.net.ssl.keyStore=/zeppelin/keystore/ssl.repo.client.keystore
-Djavax.net.ssl.keyStorePassword=kT9X6oe68t
-Djavax.net.ssl.keyStoreType=JCEKS
-Djavax.net.ssl.trustStore=/zeppelin/keystore/ssl.repo.client.truststore
-Djavax.net.ssl.trustStorePassword=kT9X6oe68t
-Djavax.net.ssl.trustStoreType=JCEKS
" <% } %>
ports:
- 9090:9090 <% if (httpMode == 'https') { %>
volumes:
- ./keystores/zeppelin:/zeppelin/keystore <% } %>
<% } %>
transform-router:
mem_limit: 512m
image: quay.io/alfresco/alfresco-transform-router:${TRANSFORM_ROUTER_TAG}
environment:
JAVA_OPTS: " -Xms256m -Xmx512m"
ACTIVEMQ_URL: "nio://activemq:61616"
CORE_AIO_URL : "http://transform-core-aio:8090"
FILE_STORE_URL: "http://shared-file-store:8099/alfresco/api/-default-/private/sfs/versions/1/file"
ports:
- 8095:8095
links:
- activemq
transform-core-aio:
image: alfresco/alfresco-transform-core-aio:${TRANSFORM_CORE_AIO_TAG}
mem_limit: 1536m
environment:
JAVA_OPTS: " -Xms256m -Xmx1536m"
ACTIVEMQ_URL: "nio://activemq:61616"
FILE_STORE_URL: "http://shared-file-store:8099/alfresco/api/-default-/private/sfs/versions/1/file"
ports:
- 8090:8090
links:
- activemq
shared-file-store:
image: alfresco/alfresco-shared-file-store:${SHARED_FILE_STORE_TAG}
mem_limit: 512m
environment:
JAVA_OPTS: " -Xms256m -Xmx512m"
scheduler.content.age.millis: 86400000
scheduler.cleanup.interval: 86400000
ports:
- 8099:8099
volumes:
- shared-file-store-volume:/tmp/Alfresco/sfs
share:
build:
context: ./share
args:
SHARE_TAG: ${SHARE_TAG}
mem_limit: 1g
environment:
REPO_HOST: "alfresco"
REPO_PORT: "8080"
JAVA_OPTS: "
-Xms500m
-Xmx500m
-Dalfresco.host=localhost
-Dalfresco.port=8080
-Dalfresco.context=alfresco
-Dalfresco.protocol=http
"
postgres:
image: postgres:${POSTGRES_TAG}
mem_limit: 512m
environment:
- POSTGRES_PASSWORD=alfresco
- POSTGRES_USER=alfresco
- POSTGRES_DB=alfresco
command: postgres -c max_connections=300 -c log_min_messages=LOG
ports:
- 5432:5432
activemq:
image: alfresco/alfresco-activemq:${ACTIVE_MQ_TAG}
mem_limit: 1g
ports:
- 8161:8161 # Web Console
- 5672:5672 # AMQP
- 61616:61616 # OpenWire
- 61613:61613 # STOMP
content-app:
image: quay.io/alfresco/alfresco-digital-workspace:${DIGITAL_WORKSPACE_TAG}
mem_limit: 128m
environment:
BASEPATH: ./
# HTTP proxy to provide HTTP Default port access to services
# SOLR API and SOLR Web Console are protected to avoid unauthenticated access
proxy:
image: nginx:stable-alpine
mem_limit: 128m
depends_on:
- alfresco
volumes:
- ./config/nginx.conf:/etc/nginx/nginx.conf
- ./config/nginx.htpasswd:/etc/nginx/conf.d/nginx.htpasswd <% if (httpWebMode == 'https') { %>
- ./config/cert/localhost.cer:/etc/nginx/localhost.cer
- ./config/cert/localhost.key:/etc/nginx/localhost.key <% } %>
ports:
- <%=port%>:<%=port%>
links:
- alfresco
- share
- solr6
- content-app
volumes:
shared-file-store-volume:
driver_opts:
type: tmpfs
device: tmpfs
e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/solr/admin/SolrE2eAdminTest.java
+25 -32
View File
@@ -29,6 +29,7 @@ package org.alfresco.test.search.functional.searchServices.solr.admin;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import org.alfresco.rest.core.RestResponse;
import org.alfresco.search.TestGroup;
@@ -603,56 +604,48 @@ public class SolrE2eAdminTest extends AbstractE2EFunctionalTest
Assert.assertEquals(actionStatus, "scheduled");
}
/**
* FIX for every core.
* @throws Exception
*/
@Test(priority = 27, dependsOnMethods = "testPurge")
public void testFix() throws Exception
{
RestResponse response = restClient.withSolrAdminAPI().getAction("fix");
checkResponseStatusOk(response);
DEFAULT_CORE_NAMES.forEach(core -> {
List<String> txToReindex = response.getResponse().body().jsonPath().get("action." + core +".txToReindex");
Assert.assertTrue(txToReindex.size() >= 0, "Expected a list of transactions (or empty list) to be reindexed,");
List<String> aclToReindex = response.getResponse().body().jsonPath().get("action." + core + ".aclChangeSetToReindex");
Assert.assertTrue(aclToReindex.size() >= 0, "Expected a list of ACLs (or empty list) to be reindexed,");
});
String actionStatus = response.getResponse().body().jsonPath().get("action.status");
Assert.assertEquals(actionStatus, "scheduled");
}
/**
* FIX for specific core.
* @throws Exception
* The test checks the response structure in order to make sure the expected sections are present.
*
* We are not testing the content of each section because due to the underlying E2E infrastructure, we cannot know
* in advance the transactions that will be scheduled for reindexing.
*/
@Test(priority = 28)
public void testFixCore() throws Exception
public void testFixCore()
{
DEFAULT_CORE_NAMES.forEach(core -> {
try
{
RestResponse response = restClient.withParams("core=" + core).withSolrAdminAPI().getAction("fix");
checkResponseStatusOk(response);
List<String> txToReindex = response.getResponse().body().jsonPath().get("action." + core +".txToReindex");
Assert.assertTrue(txToReindex.size() >= 0, "Expected a list of transactions (or empty list) to be reindexed,");
List<String> aclToReindex = response.getResponse().body().jsonPath().get("action." + core + ".aclChangeSetToReindex");
Assert.assertTrue(aclToReindex.size() >= 0, "Expected a list of ACLs (or empty list) to be reindexed,");
Map<String, Object> txInIndexNotInDb = response.getResponse().body().jsonPath().get("action." + core +".txToReindex.txInIndexNotInDb");
Assert.assertNotNull(txInIndexNotInDb, "Expected a list of transactions (even empty) that are in index but not in the database to be reindexed,");
Map<String, Object> duplicatedTx = response.getResponse().body().jsonPath().get("action." + core +".txToReindex.duplicatedTxInIndex");
Assert.assertNotNull(duplicatedTx, "Expected a list of duplicated transactions (even empty) to be reindexed,");
Map<String, Object> missingTx = response.getResponse().body().jsonPath().get("action." + core +".txToReindex.missingTxInIndex");
Assert.assertNotNull(missingTx, "Expected a list of missing transactions (or empty list) to be reindexed,");
Map<String, Object> aclTxInIndexNotInDb = response.getResponse().body().jsonPath().get("action." + core + ".aclChangeSetToReindex.aclTxInIndexNotInDb");
Assert.assertNotNull(aclTxInIndexNotInDb, "Expected a list of ACLs (or empty list) to be reindexed,");
Map<String, Object> duplicatedAclTxInIndex = response.getResponse().body().jsonPath().get("action." + core + ".aclChangeSetToReindex.duplicatedAclTxInIndex");
Assert.assertNotNull(duplicatedAclTxInIndex, "Expected a list of ACLs (or empty list) to be reindexed,");
Map<String, Object> missingAclTxInIndex = response.getResponse().body().jsonPath().get("action." + core + ".aclChangeSetToReindex.missingAclTxInIndex");
Assert.assertNotNull(missingAclTxInIndex, "Expected a list of ACLs (or empty list) to be reindexed,");
String actionStatus = response.getResponse().body().jsonPath().get("action.status");
Assert.assertEquals(actionStatus, "scheduled");
Assert.assertEquals(actionStatus, "notScheduled");
}
catch (Exception e)
{
throw new RuntimeException(e);
}
});
}
search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoCoreAdminHandler.java
+303 -49
View File
@@ -54,6 +54,7 @@ import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.core.CoreContainer;
import org.apache.solr.core.SolrCore;
import org.apache.solr.core.SolrResourceLoader;
import org.apache.solr.handler.admin.CoreAdminHandler;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.response.SolrQueryResponse;
@@ -76,16 +77,19 @@ import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Properties;
import java.util.Set;
import java.util.TreeSet;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Consumer;
import java.util.function.LongToIntFunction;
import java.util.stream.Collectors;
import static java.util.Arrays.asList;
import static java.util.Arrays.stream;
import static java.util.Optional.of;
import static java.util.Optional.ofNullable;
import static org.alfresco.repo.search.adaptor.lucene.QueryConstants.FIELD_INACLTXID;
import static org.alfresco.repo.search.adaptor.lucene.QueryConstants.FIELD_INTXID;
import static org.alfresco.solr.HandlerOfResources.extractCustomProperties;
import static org.alfresco.solr.HandlerOfResources.getSafeBoolean;
import static org.alfresco.solr.HandlerOfResources.getSafeLong;
@@ -99,6 +103,7 @@ import static org.alfresco.solr.HandlerReportHelper.buildAclTxReport;
import static org.alfresco.solr.HandlerReportHelper.buildNodeReport;
import static org.alfresco.solr.HandlerReportHelper.buildTrackerReport;
import static org.alfresco.solr.HandlerReportHelper.buildTxReport;
import static org.alfresco.solr.utils.Utils.isNullOrEmpty;
import static org.alfresco.solr.utils.Utils.notNullOrEmpty;
/**
@@ -160,22 +165,39 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler
*/
private static final String ACTION_STATUS_SUCCESS = "success";
private static final String ACTION_STATUS_ERROR = "error";
private static final String ACTION_STATUS_SCHEDULED = "scheduled";
static final String ACTION_STATUS_SCHEDULED = "scheduled";
static final String ACTION_STATUS_NOT_SCHEDULED = "notScheduled";
static final String DRY_RUN_PARAMETER_NAME = "dryRun";
static final String FROM_TX_COMMIT_TIME_PARAMETER_NAME = "fromTxCommitTime";
static final String TO_TX_COMMIT_TIME_PARAMETER_NAME = "toTxCommitTime";
static final String MAX_TRANSACTIONS_TO_SCHEDULE_PARAMETER_NAME = "maxScheduledTransactions";
static final String MAX_TRANSACTIONS_TO_SCHEDULE_CONF_PROPERTY_NAME = "alfresco.admin.fix.maxScheduledTransactions";
static final String TX_IN_INDEX_NOT_IN_DB = "txInIndexNotInDb";
static final String DUPLICATED_TX_IN_INDEX = "duplicatedTxInIndex";
static final String MISSING_TX_IN_INDEX = "missingTxInIndex";
static final String ACL_TX_IN_INDEX_NOT_IN_DB = "aclTxInIndexNotInDb";
static final String DUPLICATED_ACL_TX_IN_INDEX = "duplicatedAclTxInIndex";
static final String MISSING_ACL_TX_IN_INDEX = "missingAclTxInIndex";
/**
* JSON/XML labels for the Action response
*/
private static final String ACTION_LABEL = "action";
private static final String ACTION_STATUS_LABEL = "status";
private static final String ACTION_ERROR_MESSAGE_LABEL = "errorMessage";
static final String ACTION_STATUS_LABEL = "status";
static final String ACTION_ERROR_MESSAGE_LABEL = "errorMessage";
static final String UNKNOWN_CORE_MESSAGE = "Unknown core:";
static final String UNPROCESSABLE_REQUEST_ON_SLAVE_NODES = "Requested action cannot be performed on slave nodes.";
private static final String ACTION_TX_TO_REINDEX = "txToReindex";
private static final String ACTION_ACL_CHANGE_SET_TO_REINDEX = "aclChangeSetToReindex";
private SolrTrackerScheduler scheduler;
private TrackerRegistry trackerRegistry;
private ConcurrentHashMap<String, InformationServer> informationServers;
TrackerRegistry trackerRegistry;
ConcurrentHashMap<String, InformationServer> informationServers;
private static List<String> CORE_PARAMETER_NAMES = asList(CoreAdminParams.CORE, "coreName", "index");
private final static List<String> CORE_PARAMETER_NAMES = asList(CoreAdminParams.CORE, "coreName", "index");
public AlfrescoCoreAdminHandler()
{
@@ -938,8 +960,7 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler
*
* Synchronous execution
*
* @param req Query Request without parameters
* - coreName, optional, the name of the core to be checked
* @param cname, optional, the name of the core to be checked
*
* @return Response including the action result:
* - status: success, when the core has been created
@@ -1366,7 +1387,6 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler
* - fromAclTx, optional: from ACL transaction Id to filter report results
* - toCalTx, optional: to ACL transaction Id to filter report results
*
* @param Response including the action result:
* - report.core: multiple Objects with the details of the report ("core" is the name of the Core)
*
* @throws JSONException
@@ -1497,7 +1517,6 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler
*
* @param params Query Request with following parameters:
* - core, optional: The name of the SOLR Core
* @param rsp Query Response including the action result:
* - action.status: scheduled, as it will be executed by Trackers on the next maintenance operation
* - core: list of Document Ids with error that are going to reindexed
*/
@@ -1594,66 +1613,140 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler
* - action.status: scheduled, as it will be executed by Trackers on the next maintenance operation
* - txToReindex: list of Transaction Ids that are going to be reindexed
* - aclChangeSetToReindex: list of ACL Change Set Ids that are going to be reindexed
* @throws JSONException
*/
private NamedList<Object> actionFIX(SolrParams params) throws JSONException
NamedList<Object> actionFIX(SolrParams params) throws JSONException
{
String requestedCoreName = coreName(params);
var wrapper = new Object()
{
NamedList<Object> response = new SimpleOrderedMap<>();;
final NamedList<Object> response = new SimpleOrderedMap<>();
};
if (isNullOrEmpty(requestedCoreName))
{
return wrapper.response;
}
if (!coreNames().contains(requestedCoreName))
{
wrapper.response.add(ACTION_ERROR_MESSAGE_LABEL, UNKNOWN_CORE_MESSAGE + requestedCoreName);
return wrapper.response;
}
if (!isMasterOrStandalone(requestedCoreName)) {
wrapper.response.add(ACTION_ERROR_MESSAGE_LABEL, UNPROCESSABLE_REQUEST_ON_SLAVE_NODES);
return wrapper.response;
}
Long fromTxCommitTime = params.getLong(FROM_TX_COMMIT_TIME_PARAMETER_NAME);
Long toTxCommitTime = params.getLong(TO_TX_COMMIT_TIME_PARAMETER_NAME);
boolean dryRun = params.getBool(DRY_RUN_PARAMETER_NAME, true);
int maxTransactionsToSchedule = getMaxTransactionToSchedule(params);
LOGGER.debug("FIX Admin request on core {}, parameters: " +
FROM_TX_COMMIT_TIME_PARAMETER_NAME + " = {}, " +
TO_TX_COMMIT_TIME_PARAMETER_NAME + " = {}, " +
DRY_RUN_PARAMETER_NAME + " = {}, " +
MAX_TRANSACTIONS_TO_SCHEDULE_PARAMETER_NAME + " = {}",
requestedCoreName,
ofNullable(fromTxCommitTime).map(Object::toString).orElse("N.A."),
ofNullable(toTxCommitTime).map(Object::toString).orElse("N.A."),
dryRun,
maxTransactionsToSchedule);
coreNames().stream()
.filter(coreName -> requestedCoreName == null || coreName.equals(requestedCoreName))
.filter(this::isMasterOrStandalone)
.forEach(coreName -> {
wrapper.response.add(coreName, fixOnSpecificCore(coreName));
});
.forEach(coreName ->
wrapper.response.add(
coreName,
fixOnSpecificCore(coreName, fromTxCommitTime, toTxCommitTime, dryRun, maxTransactionsToSchedule)));
if (wrapper.response.size() > 0)
{
wrapper.response.add(DRY_RUN_PARAMETER_NAME, dryRun);
ofNullable(fromTxCommitTime).ifPresent(value -> wrapper.response.add(FROM_TX_COMMIT_TIME_PARAMETER_NAME, value));
ofNullable(toTxCommitTime).ifPresent(value -> wrapper.response.add(TO_TX_COMMIT_TIME_PARAMETER_NAME, value));
wrapper.response.add(MAX_TRANSACTIONS_TO_SCHEDULE_PARAMETER_NAME, maxTransactionsToSchedule);
wrapper.response.add(ACTION_STATUS_LABEL, dryRun ? ACTION_STATUS_NOT_SCHEDULED : ACTION_STATUS_SCHEDULED);
}
wrapper.response.add(ACTION_STATUS_LABEL, ACTION_STATUS_SCHEDULED);
return wrapper.response;
}
private NamedList<Object> fixOnSpecificCore(String coreName)
/**
* Detects the transactions that need a FIX (i.e. reindexing) because the following reasons:
*
* <ul>
* <li>A transaction is in the index but not in repository</li>
* <li>A transaction is duplicated in the index</li>
* <li>A transaction is missing in the index</li>
* </ul>
*
* Depending on the dryRun parameter, other than collecting, this method could also schedule the transactions for
* reindexing.
*
* @param coreName the target core name.
* @param fromTxCommitTime the start commit time we consider for collecting transaction.
* @param toTxCommitTime the end commit time we consider for collecting transaction.
* @param dryRun a flag indicating if the collected transactions must be actually scheduled for reindexing.
* @param maxTransactionsToSchedule the maximum number of transactions to be scheduled for reindexing.
* @return a report about transactions that need to be fixed.
*/
NamedList<Object> fixOnSpecificCore(String coreName, Long fromTxCommitTime, Long toTxCommitTime, boolean dryRun, int maxTransactionsToSchedule)
{
try
{
// Gets Metadata health and fixes any problems
MetadataTracker metadataTracker = trackerRegistry.getTrackerForCore(coreName, MetadataTracker.class);
IndexHealthReport indexHealthReport = metadataTracker.checkIndex(null, null, null, null);
IOpenBitSet toReindex = indexHealthReport.getTxInIndexButNotInDb();
toReindex.or(indexHealthReport.getDuplicatedTxInIndex());
toReindex.or(indexHealthReport.getMissingTxFromIndex());
long current = -1;
// Goes through problems in the index
Set<Long> txToReindex = new TreeSet<>();
while ((current = toReindex.nextSetBit(current + 1)) != -1)
{
metadataTracker.addTransactionToReindex(current);
txToReindex.add(current);
}
final IndexHealthReport metadataTrackerIndexHealthReport =
metadataTracker.checkIndex(null, fromTxCommitTime, toTxCommitTime);
LOGGER.debug("FIX Admin action built the MetadataTracker Index Health Report on core {}, parameters: " +
FROM_TX_COMMIT_TIME_PARAMETER_NAME + " = {}, " +
TO_TX_COMMIT_TIME_PARAMETER_NAME + " = {}, " +
DRY_RUN_PARAMETER_NAME + " = {}, " +
MAX_TRANSACTIONS_TO_SCHEDULE_PARAMETER_NAME + " = {}",
coreName,
ofNullable(fromTxCommitTime).map(Object::toString).orElse("N.A."),
ofNullable(toTxCommitTime).map(Object::toString).orElse("N.A."),
dryRun,
maxTransactionsToSchedule);
// Gets the Acl health and fixes any problems
AclTracker aclTracker = trackerRegistry.getTrackerForCore(coreName, AclTracker.class);
indexHealthReport = aclTracker.checkIndex(null, null, null, null);
toReindex = indexHealthReport.getAclTxInIndexButNotInDb();
toReindex.or(indexHealthReport.getDuplicatedAclTxInIndex());
toReindex.or(indexHealthReport.getMissingAclTxFromIndex());
current = -1;
// Goes through the problems in the index
Set<Long> aclChangeSetToReindex = new TreeSet<>();
while ((current = toReindex.nextSetBit(current + 1)) != -1)
{
aclTracker.addAclChangeSetToReindex(current);
aclChangeSetToReindex.add(current);
}
final IndexHealthReport aclTrackerIndexHealthReport =
aclTracker.checkIndex(null, fromTxCommitTime, toTxCommitTime);
LOGGER.debug("FIX Admin action built the AclTracker Index Health Report on core {}, parameters: " +
FROM_TX_COMMIT_TIME_PARAMETER_NAME + " = {}, " +
TO_TX_COMMIT_TIME_PARAMETER_NAME + " = {}, " +
DRY_RUN_PARAMETER_NAME + " = {}, " +
MAX_TRANSACTIONS_TO_SCHEDULE_PARAMETER_NAME + " = {}",
coreName,
ofNullable(fromTxCommitTime).map(Object::toString).orElse("N.A."),
ofNullable(toTxCommitTime).map(Object::toString).orElse("N.A."),
dryRun,
maxTransactionsToSchedule);
NamedList<Object> response = new SimpleOrderedMap<>();
response.add(ACTION_TX_TO_REINDEX, txToReindex);
response.add(ACTION_ACL_CHANGE_SET_TO_REINDEX, aclChangeSetToReindex);
return response;
response.add(ACTION_TX_TO_REINDEX,
txToReindex(
coreName,
metadataTracker,
metadataTrackerIndexHealthReport,
dryRun ? txid -> {} : metadataTracker::addTransactionToReindex,
maxTransactionsToSchedule));
response.add(ACTION_ACL_CHANGE_SET_TO_REINDEX,
aclTxToReindex(
coreName,
aclTracker,
aclTrackerIndexHealthReport,
dryRun ? txid -> {} : aclTracker::addAclChangeSetToReindex,
maxTransactionsToSchedule));
return response;
}
catch(Exception exception)
{
@@ -1661,6 +1754,153 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler
}
}
/**
* Detects the transactions that need a FIX (i.e. reindexing) because the following reasons:
*
* <ul>
* <li>A transaction is in the index but not in repository</li>
* <li>A transaction is duplicated in the index</li>
* <li>A transaction is missing in the index</li>
* </ul>
*
* Note: the method, as a side effect, could also schedule the detected transactions for reindexing.
* That is controlled by the scheduler input param (which is directly connected with the FIX tool "dryRun" parameter).
*
* @param coreName the target core name.
* @param tracker the {@link MetadataTracker} instance associated with the target core.
* @param report the index healt report produced by the tracker.
* @param scheduler the controller which manages the actual transaction scheduling.
* @param maxTransactionsToSchedule the maximum number of transactions to schedule for reindexing.
* @return a report which includes the transactions that need a reindexing.
* @see <a href="https://issues.alfresco.com/jira/browse/SEARCH-2233">SEARCH-2233</a>
* @see <a href="https://issues.alfresco.com/jira/browse/SEARCH-2248">SEARCH-2248</a>
*/
NamedList<Object> txToReindex(
String coreName,
MetadataTracker tracker,
final IndexHealthReport report,
Consumer<Long> scheduler,
int maxTransactionsToSchedule)
{
final AtomicInteger globalLimit = new AtomicInteger(maxTransactionsToSchedule);
final LongToIntFunction retrieveTransactionRelatedNodesCountFromRepository =
txid -> notNullOrEmpty(tracker.getFullNodesForDbTransaction(txid)).size();
final LongToIntFunction retrieveTransactionRelatedNodesCountFromIndex =
txid -> of(getInformationServers().get(coreName))
.map(SolrInformationServer.class::cast)
.map(server -> server.getDocListSize(FIELD_INTXID + ":" + txid))
.orElse(0);
NamedList<Object> txToReindex = new SimpleOrderedMap<>();
txToReindex.add(TX_IN_INDEX_NOT_IN_DB,
manageTransactionsToBeFixed(
report.getTxInIndexButNotInDb(),
retrieveTransactionRelatedNodesCountFromIndex,
scheduler,
globalLimit));
txToReindex.add(DUPLICATED_TX_IN_INDEX,
manageTransactionsToBeFixed(
report.getDuplicatedTxInIndex(),
retrieveTransactionRelatedNodesCountFromIndex,
scheduler,
globalLimit));
txToReindex.add(MISSING_TX_IN_INDEX,
manageTransactionsToBeFixed(
report.getMissingTxFromIndex(),
retrieveTransactionRelatedNodesCountFromRepository,
scheduler,
globalLimit));
return txToReindex;
}
/**
* Detects the ACL transactions that need a FIX (i.e. reindexing) because the following reasons:
*
* <ul>
* <li>A transaction is in the index but not in repository</li>
* <li>A transaction is duplicated in the index</li>
* <li>A transaction is missing in the index</li>
* </ul>
*
* This method is almost the same as {@link #txToReindex(String, MetadataTracker, IndexHealthReport, Consumer, int)}.
* The main difference is the target tracker ({@link AclTracker} in this case, instead of {@link MetadataTracker}).
*
* Note: the method, as a side effect, could also schedule the detected transactions for reindexing.
* That is controlled by the scheduler input param (which is directly connected with the FIX tool "dryRun" parameter).
*
* @param coreName the target core name.
* @param tracker the {@link AclTracker} instance associated with the target core.
* @param report the index healt report produced by the tracker.
* @param scheduler the controller which manages the actual transaction scheduling.
* @return a report which includes the transactions that need a reindexing.
* @see <a href="https://issues.alfresco.com/jira/browse/SEARCH-2233">SEARCH-2233</a>
* @see <a href="https://issues.alfresco.com/jira/browse/SEARCH-2248">SEARCH-2248</a>
*/
NamedList<Object> aclTxToReindex(
String coreName,
AclTracker tracker,
final IndexHealthReport report,
Consumer<Long> scheduler,
int maxTransactionsToSchedule)
{
final AtomicInteger globalLimit = new AtomicInteger(maxTransactionsToSchedule);
final LongToIntFunction retrieveAclTransactionRelatedNodesCountFromRepository =
txid -> notNullOrEmpty(tracker.getAclsForDbAclTransaction(txid)).size();
final LongToIntFunction retrieveAclTransactionRelatedNodesCountFromIndex =
txid -> of(getInformationServers().get(coreName))
.map(SolrInformationServer.class::cast)
.map(server -> server.getDocListSize(FIELD_INACLTXID + ":" + txid))
.orElse(0);
NamedList<Object> aclTxToReindex = new SimpleOrderedMap<>();
aclTxToReindex.add(ACL_TX_IN_INDEX_NOT_IN_DB,
manageTransactionsToBeFixed(
report.getAclTxInIndexButNotInDb(),
retrieveAclTransactionRelatedNodesCountFromIndex,
scheduler,
globalLimit));
aclTxToReindex.add(DUPLICATED_ACL_TX_IN_INDEX,
manageTransactionsToBeFixed(
report.getDuplicatedAclTxInIndex(),
retrieveAclTransactionRelatedNodesCountFromIndex,
scheduler,
globalLimit));
aclTxToReindex.add(MISSING_ACL_TX_IN_INDEX,
manageTransactionsToBeFixed(
report.getMissingAclTxFromIndex(),
retrieveAclTransactionRelatedNodesCountFromRepository,
scheduler,
globalLimit));
return aclTxToReindex;
}
NamedList<Object> manageTransactionsToBeFixed(
IOpenBitSet transactions,
LongToIntFunction nodesCounter,
Consumer<Long> scheduler,
AtomicInteger limit)
{
final NamedList<Object> transactionsList = new SimpleOrderedMap<>();
long txid = -1;
while ((txid = transactions.nextSetBit(txid + 1)) != -1 && limit.decrementAndGet() >= 0)
{
transactionsList.add(String.valueOf(txid), nodesCounter.applyAsInt(txid));
scheduler.accept(txid);
}
return transactionsList;
}
/**
* Get detailed report for a core or for every core including information
* related with handlers and trackers.
@@ -1830,4 +2070,18 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler
.findFirst()
.orElse(null);
}
int getMaxTransactionToSchedule(SolrParams params)
{
String requestedCoreName = coreName(params);
return ofNullable(params.getInt(MAX_TRANSACTIONS_TO_SCHEDULE_PARAMETER_NAME))
.orElseGet(() ->
ofNullable(coreContainer)
.map(container -> container.getCore(requestedCoreName))
.map(SolrCore::getResourceLoader)
.map(SolrResourceLoader::getCoreProperties)
.map(conf -> conf.getProperty(MAX_TRANSACTIONS_TO_SCHEDULE_CONF_PROPERTY_NAME))
.map(Integer::parseInt)
.orElse(Integer.MAX_VALUE)); // Last fallback if we don't have a request param and a value in configuration
}
}
search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoSolrDataModel.java
+251 -2
View File
@@ -144,8 +144,9 @@ public class AlfrescoSolrDataModel implements QueryConstants
public enum FieldUse
{
FTS, // Term/Phrase/Range/Fuzzy/Prefix/Proximity/Wild
ID, // Exact/ExactRange - Comparison, In, Upper, Lower
FTS, // Term/Phrase/Range/Fuzzy/Prefix/Proximity
ID, // ExactRange - Comparison, In, Upper, Lower, Wildcard
EXACT, // Exact
FACET, // Field, Range, Query
MULTI_FACET, // Text fields will require cross language support to avoid tokenisation for facets
STATS, // Stats
@@ -719,6 +720,9 @@ public class AlfrescoSolrDataModel implements QueryConstants
case ID:
addIdentifierSearchFields(propertyDefinition, indexedField);
break;
case EXACT:
addExactSearchFields(propertyDefinition, indexedField);
break;
case MULTI_FACET:
addMultiSearchFields(propertyDefinition, indexedField);
break;
@@ -747,6 +751,251 @@ public class AlfrescoSolrDataModel implements QueryConstants
return indexedField;
}
/*
* Adds best completion fields in order of preference
*/
private void addCompletionFields(PropertyDefinition propertyDefinition, IndexedField indexedField)
{
if ((propertyDefinition.getIndexTokenisationMode() == IndexTokenisationMode.BOTH))
{
if(crossLocaleSearchDataTypes.contains(propertyDefinition.getDataType().getName()) || crossLocaleSearchProperties.contains(propertyDefinition.getName()))
{
indexedField.addField(getFieldForText(false, true, false, propertyDefinition), false, false);
}
else
{
indexedField.addField(getFieldForText(true, true, false, propertyDefinition), false, false);
}
}
else if ((propertyDefinition.getIndexTokenisationMode() == IndexTokenisationMode.TRUE))
{
if(crossLocaleSearchDataTypes.contains(propertyDefinition.getDataType().getName()) || crossLocaleSearchProperties.contains(propertyDefinition.getName()))
{
indexedField.addField(getFieldForText(false, true, false, propertyDefinition), false, false);
}
else
{
indexedField.addField(getFieldForText(true, true, false, propertyDefinition), false, false);
}
}
else if ((propertyDefinition.getIndexTokenisationMode() == IndexTokenisationMode.FALSE))
{
indexedField.addField(getFieldForText(false, false, false, propertyDefinition), false, false);
}
}
/*
* Adds best fts fields in order of preference
*/
private void addFullTextSearchFields( PropertyDefinition propertyDefinition , IndexedField indexedField)
{
if (((propertyDefinition.getIndexTokenisationMode() == IndexTokenisationMode.TRUE)
|| (propertyDefinition.getIndexTokenisationMode() == IndexTokenisationMode.BOTH))
&& !isIdentifierTextProperty(propertyDefinition.getName()))
{
indexedField.addField(getFieldForText(true, true, false, propertyDefinition), true, false);
if(crossLocaleSearchDataTypes.contains(propertyDefinition.getDataType().getName()) || crossLocaleSearchProperties.contains(propertyDefinition.getName()))
{
indexedField.addField(getFieldForText(false, true, false, propertyDefinition), false, false);
}
}
else
{
indexedField.addField(getFieldForText(true, false, false, propertyDefinition), true, false);
indexedField.addField(getFieldForText(false, false, false, propertyDefinition), false, false);
}
}
private void addHighlightSearchFields( PropertyDefinition propertyDefinition , IndexedField indexedField)
{
QName propertyName = propertyDefinition.getName();
QName propertyDataTypeQName = propertyDefinition.getDataType().getName();
String fieldName;
if(propertyDataTypeQName.equals(DataTypeDefinition.MLTEXT))
{
fieldName = getStoredMLTextField(propertyName);
}
else if(propertyDataTypeQName.equals(DataTypeDefinition.CONTENT))
{
fieldName = getStoredContentField(propertyName);
}
else
{
fieldName = getStoredTextField(propertyName);
}
indexedField.addField(fieldName, false, false);
}
/*
* If the untokenised version is available, go for it.
* If not it takes the tokenised versions available.
*/
private void addIdentifierSearchFields( PropertyDefinition propertyDefinition , IndexedField indexedField)
{
if ((propertyDefinition.getIndexTokenisationMode() == IndexTokenisationMode.FALSE)
|| (propertyDefinition.getIndexTokenisationMode() == IndexTokenisationMode.BOTH))
{
indexedField.addField(getFieldForText(true, false, false, propertyDefinition), true, false);
indexedField.addField(getFieldForText(false, false, false, propertyDefinition), false, false);
}
else
{
indexedField.addField(getFieldForText(true, true, false, propertyDefinition), true, false);
if(crossLocaleSearchDataTypes.contains(propertyDefinition.getDataType().getName()) || crossLocaleSearchProperties.contains(propertyDefinition.getName()))
{
indexedField.addField(getFieldForText(false, true, false, propertyDefinition), false, false);
}
}
}
/*
* If only the untokenised version is available, go for it.
* If not it takes the tokenised cross locale version.
*/
private void addExactSearchFields(PropertyDefinition propertyDefinition, IndexedField indexedField)
{
if ((propertyDefinition.getIndexTokenisationMode() == IndexTokenisationMode.FALSE)
|| !(propertyDefinition.getIndexTokenisationMode() == IndexTokenisationMode.BOTH))
{
indexedField.addField(getFieldForText(true, false, false, propertyDefinition), true, false);
indexedField.addField(getFieldForText(false, false, false, propertyDefinition), false, false);
}
else
{
if(crossLocaleSearchDataTypes.contains(propertyDefinition.getDataType().getName()) || crossLocaleSearchProperties.contains(propertyDefinition.getName()))
{
indexedField.addField(getFieldForText(false, true, false, propertyDefinition), false, false);
} else{
throw new UnsupportedOperationException("Exact Term search is not supported unless you configure the field <"+propertyDefinition.getName()+"> for cross locale search");
}
}
}
/*
* Adds best identifier fields in order of preference
*/
private void addFacetSearchFields(PropertyDefinition propertyDefinition, IndexedField indexedField)
{
if(propertyDefinition.getDataType().getName().equals(DataTypeDefinition.TEXT))
{
if (!isIdentifierTextProperty(propertyDefinition.getName()))
{
if(propertyDefinition.getFacetable() == Facetable.TRUE)
{
indexedField.addField(getFieldForText(false, false, false, propertyDefinition), false, false);
}
}
}
if ((propertyDefinition.getIndexTokenisationMode() == IndexTokenisationMode.FALSE)
|| (propertyDefinition.getIndexTokenisationMode() == IndexTokenisationMode.BOTH)
|| isIdentifierTextProperty(propertyDefinition.getName()))
{
indexedField.addField(getFieldForText(false, false, false, propertyDefinition), false, false);
}
else
{
if(crossLocaleSearchDataTypes.contains(propertyDefinition.getDataType().getName()) || crossLocaleSearchProperties.contains(propertyDefinition.getName()))
{
indexedField.addField(getFieldForText(false, true, false, propertyDefinition), false, false);
}
else
{
indexedField.addField(getFieldForText(true, true, false, propertyDefinition), false, false);
}
}
}
private void addMultiSearchFields( PropertyDefinition propertyDefinition , IndexedField indexedField)
{
if ((propertyDefinition.getIndexTokenisationMode() == IndexTokenisationMode.FALSE)
|| (propertyDefinition.getIndexTokenisationMode() == IndexTokenisationMode.BOTH)
|| isIdentifierTextProperty(propertyDefinition.getName()))
{
indexedField.addField(getFieldForText(false, false, false, propertyDefinition), false, false);
}
else
{
if(crossLocaleSearchDataTypes.contains(propertyDefinition.getDataType().getName()) || crossLocaleSearchProperties.contains(propertyDefinition.getName()))
{
indexedField.addField(getFieldForText(false, true, false, propertyDefinition), false, false);
}
else
{
indexedField.addField(getFieldForText(true, true, false, propertyDefinition), false, false);
}
}
}
private void addStatsSearchFields( PropertyDefinition propertyDefinition , IndexedField indexedField)
{
addFacetSearchFields(propertyDefinition, indexedField);
}
private void addSortSearchFields( PropertyDefinition propertyDefinition , IndexedField indexedField)
{
// Can only order on single valued fields
DataTypeDefinition dataTypeDefinition = propertyDefinition.getDataType();
if(dataTypeDefinition.getName().equals(DataTypeDefinition.TEXT))
{
if(!propertyDefinition.isMultiValued())
{
if ((propertyDefinition.getIndexTokenisationMode() == IndexTokenisationMode.FALSE)
|| (propertyDefinition.getIndexTokenisationMode() == IndexTokenisationMode.BOTH))
{
indexedField.addField(getFieldForText(false, false, true, propertyDefinition), false, true);
}
else if (isIdentifierTextProperty(propertyDefinition.getName()))
{
indexedField.addField(getFieldForText(false, false, false, propertyDefinition), false, false);
}
else
{
if(crossLocaleSearchDataTypes.contains(propertyDefinition.getDataType().getName()) || crossLocaleSearchProperties.contains(propertyDefinition.getName()))
{
indexedField.addField(getFieldForText(false, true, false, propertyDefinition), false, false);
}
else
{
indexedField.addField(getFieldForText(true, true, false, propertyDefinition), false, false);
}
}
}
}
if(dataTypeDefinition.getName().equals(DataTypeDefinition.MLTEXT))
{
if(!propertyDefinition.isMultiValued())
{
if ((propertyDefinition.getIndexTokenisationMode() == IndexTokenisationMode.FALSE)
|| (propertyDefinition.getIndexTokenisationMode() == IndexTokenisationMode.BOTH))
{
indexedField.addField(getFieldForText(false, false, true, propertyDefinition), false, true);
}
else
{
if(crossLocaleSearchDataTypes.contains(propertyDefinition.getDataType().getName()) || crossLocaleSearchProperties.contains(propertyDefinition.getName()))
{
indexedField.addField(getFieldForText(false, true, false, propertyDefinition), false, false);
}
else
{
indexedField.addField(getFieldForText(true, true, false, propertyDefinition), false, false);
}
}
}
}
}
public String getStoredTextField(QName propertyQName)
{
return getStoredTextField(propertyQName, null);
search-services/alfresco-search/src/main/java/org/alfresco/solr/HandlerReportHelper.java
+2 -2
View File
@@ -157,7 +157,7 @@ class HandlerReportHelper
{
// ACL
AclTracker aclTracker = trackerRegistry.getTrackerForCore(coreName, AclTracker.class);
IndexHealthReport aclReport = aclTracker.checkIndex(toTx, toAclTx, fromTime, toTime);
IndexHealthReport aclReport = aclTracker.checkIndex(toAclTx, fromTime, toTime);
NamedList<Object> ihr = new SimpleOrderedMap<>();
ihr.add("DB acl transaction count", aclReport.getDbAclTransactionCount());
ihr.add("Count of duplicated acl transactions in the index", aclReport.getDuplicatedAclTxInIndex()
@@ -187,7 +187,7 @@ class HandlerReportHelper
// Metadata
MetadataTracker metadataTracker = trackerRegistry.getTrackerForCore(coreName, MetadataTracker.class);
IndexHealthReport metaReport = metadataTracker.checkIndex(toTx, toAclTx, fromTime, toTime);
IndexHealthReport metaReport = metadataTracker.checkIndex(toTx, fromTime, toTime);
ihr.add("DB transaction count", metaReport.getDbTransactionCount());
ihr.add("Count of duplicated transactions in the index", metaReport.getDuplicatedTxInIndex()
.cardinality());
search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java
+1 -1
View File
@@ -3632,7 +3632,7 @@ public class SolrInformationServer implements InformationServer
return (NamedList) facetFields.get(field);
}
private int getDocListSize(String query)
public int getDocListSize(String query)
{
try (SolrQueryRequest request = this.newSolrQueryRequest())
{
search-services/alfresco-search/src/main/java/org/alfresco/solr/query/Solr4QueryParser.java
+228 -182
View File
@@ -150,12 +150,17 @@ import java.util.concurrent.Future;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import static org.apache.solr.search.QueryParsing.DEFTYPE;
/**
* @author Andy
*
*/
public class Solr4QueryParser extends QueryParser implements QueryConstants
{
private static final String CMIS_REQUEST_HANDLER = "cmis";
public Solr4QueryParser(SolrQueryRequest req, Version matchVersion, String f, Analyzer a,
FTSQueryParser.RerankPhase rerankPhase)
{
@@ -1480,13 +1485,18 @@ public class Solr4QueryParser extends QueryParser implements QueryConstants
BooleanQuery.Builder query = new BooleanQuery.Builder();
for (String fieldName : text)
{
Query part = getQuery.apply(fieldName);
if (part != null)
{
query.add(part, Occur.SHOULD);
} else
{
query.add(createNoMatchQuery(), Occur.SHOULD);
Query part;
try {
part = getQuery.apply(fieldName);
if (part != null)
{
query.add(part, Occur.SHOULD);
} else
{
query.add(createNoMatchQuery(), Occur.SHOULD);
}
} catch (UnsupportedOperationException e) {
logger.warn(e);
}
}
return query.build();
@@ -4836,46 +4846,52 @@ public class Solr4QueryParser extends QueryParser implements QueryConstants
AnalysisMode analysisMode, LuceneFunction luceneFunction, String expandedFieldName,
List<Locale> expandedLocales) throws ParseException
{
BooleanQuery.Builder booleanQuery = new BooleanQuery.Builder();
for (Locale locale : expandedLocales)
BooleanQuery.Builder booleanQueryBuilder = new BooleanQuery.Builder();
IndexedField indexedField = AlfrescoSolrDataModel.getInstance().getQueryableFields(pDef.getName(), null,
FieldUse.FTS);
if (isExactTermSearch(analysisMode))
{
if (locale.toString().length() == 0)
for (FieldInstance field : indexedField.getFields())
{
IndexedField indexedField = AlfrescoSolrDataModel.getInstance().getQueryableFields(pDef.getName(), null,
FieldUse.FTS);
if (!field.isLocalised())
{
Query subQuery = subQueryBuilder.getQuery(field.getField(), queryText, analysisMode,
luceneFunction);
if (subQuery != null)
{
booleanQueryBuilder.add(subQuery, Occur.SHOULD);
}
}
}
if (booleanQueryBuilder.build().clauses().size() == 0)
{
throw new UnsupportedOperationException("Exact Term search is not supported unless you configure the field <"+pDef.getName()+"> for cross locale search");
}
} else
{
for (Locale locale : expandedLocales)
{
boolean localisedSearch = locale.toString().length() != 0;
if (localisedSearch)
{
StringBuilder builder = new StringBuilder(queryText.length() + 10);
builder.append("\u0000").append(locale.toString()).append("\u0000").append(queryText);
}
for (FieldInstance field : indexedField.getFields())
{
if (!field.isLocalised())
if ((!localisedSearch && !field.isLocalised()) || (localisedSearch && field.isLocalised()))
{
Query subQuery = subQueryBuilder.getQuery(field.getField(), queryText, analysisMode,
luceneFunction);
if (subQuery != null)
{
booleanQuery.add(subQuery, Occur.SHOULD);
}
}
}
} else
{
StringBuilder builder = new StringBuilder(queryText.length() + 10);
builder.append("\u0000").append(locale.toString()).append("\u0000").append(queryText);
IndexedField indexedField = AlfrescoSolrDataModel.getInstance().getQueryableFields(pDef.getName(), null,
FieldUse.FTS);
for (FieldInstance field : indexedField.getFields())
{
if (field.isLocalised())
{
Query subQuery = subQueryBuilder.getQuery(field.getField(), builder.toString(), analysisMode,
luceneFunction);
if (subQuery != null)
{
booleanQuery.add(subQuery, Occur.SHOULD);
booleanQueryBuilder.add(subQuery, Occur.SHOULD);
}
}
}
}
}
return getNonEmptyBooleanQuery(booleanQuery.build());
return getNonEmptyBooleanQuery(booleanQueryBuilder.build());
}
protected void addLocaleSpecificUntokenisedMLOrTextFunction(String expandedFieldName, PropertyDefinition pDef,
@@ -4896,67 +4912,74 @@ public class Solr4QueryParser extends QueryParser implements QueryConstants
}
private FieldInstance getFieldInstance(String baseFieldName, PropertyDefinition pDef, Locale locale,
IndexTokenisationMode preferredIndexTokenisationMode)
IndexTokenisationMode preferredIndexTokenisationMode, AnalysisMode analysisMode)
{
if (pDef != null)
{
switch (preferredIndexTokenisationMode)
{
case BOTH:
throw new IllegalStateException("Preferred mode can not be BOTH");
case FALSE:
if (locale.toString().length() == 0)
{
IndexedField indexedField = AlfrescoSolrDataModel.getInstance().getQueryableFields(pDef.getName(),
null, FieldUse.ID);
for (FieldInstance field : indexedField.getFields())
case BOTH:
throw new IllegalStateException("Preferred mode can not be BOTH");
case FALSE:
if (locale.toString().length() == 0)
{
if (!field.isLocalised())
IndexedField indexedField = AlfrescoSolrDataModel.getInstance().getQueryableFields(pDef.getName(),
null, FieldUse.ID);
for (FieldInstance field : indexedField.getFields())
{
return field;
if (!field.isLocalised())
{
return field;
}
}
} else
{
IndexedField indexedField = AlfrescoSolrDataModel.getInstance().getQueryableFields(pDef.getName(),
null, FieldUse.ID);
for (FieldInstance field : indexedField.getFields())
{
if (field.isLocalised())
{
return field;
}
}
}
} else
break;
case TRUE:
{
IndexedField indexedField = AlfrescoSolrDataModel.getInstance().getQueryableFields(pDef.getName(),
null, FieldUse.ID);
for (FieldInstance field : indexedField.getFields())
IndexedField indexedField;
if (isExactTermSearch(analysisMode))
{
if (field.isLocalised())
indexedField = AlfrescoSolrDataModel.getInstance().getQueryableFields(pDef.getName(),
null, FieldUse.EXACT);
} else
{
indexedField = AlfrescoSolrDataModel.getInstance().getQueryableFields(pDef.getName(),
null, FieldUse.FTS);
}
if (locale.toString().length() == 0)
{
for (FieldInstance field : indexedField.getFields())
{
return field;
if (!field.isLocalised())
{
return field;
}
}
} else
{
for (FieldInstance field : indexedField.getFields())
{
if (field.isLocalised())
{
return field;
}
}
}
break;
}
break;
case TRUE:
if (locale.toString().length() == 0)
{
IndexedField indexedField = AlfrescoSolrDataModel.getInstance().getQueryableFields(pDef.getName(),
null, FieldUse.FTS);
for (FieldInstance field : indexedField.getFields())
{
if (!field.isLocalised())
{
return field;
}
}
} else
{
IndexedField indexedField = AlfrescoSolrDataModel.getInstance().getQueryableFields(pDef.getName(),
null, FieldUse.FTS);
for (FieldInstance field : indexedField.getFields())
{
if (field.isLocalised())
{
return field;
}
}
}
break;
}
return new FieldInstance("_dummy_", false, false);
return null;
}
@@ -4964,6 +4987,10 @@ public class Solr4QueryParser extends QueryParser implements QueryConstants
}
private boolean isExactTermSearch(AnalysisMode analysisMode) {
return analysisMode==AnalysisMode.IDENTIFIER && !this.solrParams.get(DEFTYPE).equals(CMIS_REQUEST_HANDLER);
}
protected void addLocaleSpecificUntokenisedTextRangeFunction(String expandedFieldName, PropertyDefinition pDef,
String lower, String upper, boolean includeLower, boolean includeUpper, LuceneFunction luceneFunction,
Builder booleanQuery, Locale locale, IndexTokenisationMode tokenisationMode) throws ParseException
@@ -5000,9 +5027,9 @@ public class Solr4QueryParser extends QueryParser implements QueryConstants
}
private void addMLTextOrTextAttributeQuery(String field, PropertyDefinition pDef, String queryText,
SubQuery subQueryBuilder, AnalysisMode analysisMode, LuceneFunction luceneFunction,
String expandedFieldName, IndexTokenisationMode tokenisationMode, Builder booleanQuery, Locale locale)
throws ParseException
SubQuery subQueryBuilder, AnalysisMode analysisMode, LuceneFunction luceneFunction,
String expandedFieldName, IndexTokenisationMode tokenisationMode, Builder booleanQuery, Locale locale)
throws ParseException
{
boolean lowercaseExpandedTerms = getLowercaseExpandedTerms();
@@ -5010,48 +5037,59 @@ public class Solr4QueryParser extends QueryParser implements QueryConstants
{
switch (tokenisationMode)
{
case BOTH:
switch (analysisMode)
{
default:
case DEFAULT:
addLocaleSpecificMLOrTextAttribute(pDef, queryText, subQueryBuilder, analysisMode, luceneFunction,
booleanQuery, locale, expandedFieldName, tokenisationMode, IndexTokenisationMode.TRUE);
if (ContentModel.PROP_NAME.equals(pDef.getName()))
case BOTH:
switch (analysisMode)
{
setLowercaseExpandedTerms(false);
addLocaleSpecificMLOrTextAttribute(pDef, queryText, subQueryBuilder, analysisMode, luceneFunction,
booleanQuery, locale, expandedFieldName, tokenisationMode, IndexTokenisationMode.FALSE);
}
default:
case DEFAULT:
addLocaleSpecificMLOrTextAttribute(pDef, queryText, subQueryBuilder, analysisMode, luceneFunction,
booleanQuery, locale, expandedFieldName, tokenisationMode, IndexTokenisationMode.TRUE);
if (ContentModel.PROP_NAME.equals(pDef.getName()))
{
setLowercaseExpandedTerms(false);
addLocaleSpecificMLOrTextAttribute(pDef, queryText, subQueryBuilder, analysisMode, luceneFunction,
booleanQuery, locale, expandedFieldName, tokenisationMode, IndexTokenisationMode.FALSE);
}
break;
case TOKENISE:
addLocaleSpecificMLOrTextAttribute(pDef, queryText, subQueryBuilder, analysisMode, luceneFunction,
booleanQuery, locale, expandedFieldName, tokenisationMode, IndexTokenisationMode.TRUE);
break;
case IDENTIFIER:
setLowercaseExpandedTerms(false);
if (isExactTermSearch(analysisMode))
{//with exact search we favour tokenization, specifically cross locale tokenization
addLocaleSpecificMLOrTextAttribute(pDef, queryText, subQueryBuilder, analysisMode, luceneFunction,
booleanQuery, locale, expandedFieldName, tokenisationMode, IndexTokenisationMode.TRUE);
} else
{
addLocaleSpecificMLOrTextAttribute(pDef, queryText, subQueryBuilder, analysisMode, luceneFunction,
booleanQuery, locale, expandedFieldName, tokenisationMode, IndexTokenisationMode.FALSE);
}
break;
case FUZZY:
case PREFIX:
case WILD:
case LIKE:
setLowercaseExpandedTerms(false);
addLocaleSpecificMLOrTextAttribute(pDef, queryText, subQueryBuilder, analysisMode, luceneFunction,
booleanQuery, locale, expandedFieldName, tokenisationMode, IndexTokenisationMode.FALSE);
break;
}
break;
case TOKENISE:
addLocaleSpecificMLOrTextAttribute(pDef, queryText, subQueryBuilder, analysisMode, luceneFunction,
booleanQuery, locale, expandedFieldName, tokenisationMode, IndexTokenisationMode.TRUE);
break;
case IDENTIFIER:
case FUZZY:
case PREFIX:
case WILD:
case LIKE:
case FALSE:
setLowercaseExpandedTerms(false);
addLocaleSpecificMLOrTextAttribute(pDef, queryText, subQueryBuilder, analysisMode, luceneFunction,
booleanQuery, locale, expandedFieldName, tokenisationMode, IndexTokenisationMode.FALSE);
break;
}
break;
case FALSE:
setLowercaseExpandedTerms(false);
addLocaleSpecificMLOrTextAttribute(pDef, queryText, subQueryBuilder, analysisMode, luceneFunction,
booleanQuery, locale, expandedFieldName, tokenisationMode, IndexTokenisationMode.FALSE);
break;
case TRUE:
default:
addLocaleSpecificMLOrTextAttribute(pDef, queryText, subQueryBuilder, analysisMode, luceneFunction,
booleanQuery, locale, expandedFieldName, tokenisationMode, IndexTokenisationMode.TRUE);
break;
case TRUE:
default:
addLocaleSpecificMLOrTextAttribute(pDef, queryText, subQueryBuilder, analysisMode, luceneFunction,
booleanQuery, locale, expandedFieldName, tokenisationMode, IndexTokenisationMode.TRUE);
break;
}
} finally
{
@@ -5071,22 +5109,26 @@ public class Solr4QueryParser extends QueryParser implements QueryConstants
}
private void addLocaleSpecificMLOrTextAttribute(PropertyDefinition pDef, String queryText, SubQuery subQueryBuilder,
AnalysisMode analysisMode, LuceneFunction luceneFunction, Builder booleanQuery, Locale locale,
String textFieldName, IndexTokenisationMode tokenisationMode,
IndexTokenisationMode preferredTokenisationMode) throws ParseException {
AnalysisMode analysisMode, LuceneFunction luceneFunction, Builder booleanQuery, Locale locale,
String textFieldName, IndexTokenisationMode tokenisationMode,
IndexTokenisationMode preferredTokenisationMode) throws ParseException
{
FieldInstance fieldInstance = getFieldInstance(textFieldName, pDef, locale, preferredTokenisationMode);
StringBuilder builder = new StringBuilder(queryText.length() + 10);
if (fieldInstance.isLocalised())
FieldInstance fieldInstance = getFieldInstance(textFieldName, pDef, locale, preferredTokenisationMode, analysisMode);
if (fieldInstance != null)
{
builder.append("\u0000").append(locale.toString()).append("\u0000");
}
builder.append(queryText);
Query subQuery = subQueryBuilder.getQuery(fieldInstance.getField(), builder.toString(), analysisMode,
luceneFunction);
if (subQuery != null)
{
booleanQuery.add(subQuery, Occur.SHOULD);
StringBuilder builder = new StringBuilder(queryText.length() + 10);
if (fieldInstance.isLocalised())
{
builder.append("\u0000").append(locale.toString()).append("\u0000");
}
builder.append(queryText);
Query subQuery = subQueryBuilder.getQuery(fieldInstance.getField(), builder.toString(), analysisMode,
luceneFunction);
if (subQuery != null)
{
booleanQuery.add(subQuery, Occur.SHOULD);
}
}
}
@@ -5131,76 +5173,80 @@ public class Solr4QueryParser extends QueryParser implements QueryConstants
}
private void addLocaleSpecificTextRange(String expandedFieldName, PropertyDefinition pDef, String part1,
String part2, boolean includeLower, boolean includeUpper, Builder booleanQuery, Locale locale,
AnalysisMode analysisMode, IndexTokenisationMode tokenisationMode,
IndexTokenisationMode preferredTokenisationMode) throws ParseException, IOException
String part2, boolean includeLower, boolean includeUpper, Builder booleanQuery, Locale locale,
AnalysisMode analysisMode, IndexTokenisationMode tokenisationMode,
IndexTokenisationMode preferredTokenisationMode) throws ParseException, IOException
{
FieldInstance fieldInstance = getFieldInstance(expandedFieldName, pDef, locale, preferredTokenisationMode);
String firstString = null;
if ((part1 != null) && !part1.equals("\u0000"))
FieldInstance fieldInstance = getFieldInstance(expandedFieldName, pDef, locale, preferredTokenisationMode, analysisMode);
if (fieldInstance != null)
{
if (fieldInstance.isLocalised())
String firstString = null;
if ((part1 != null) && !part1.equals("\u0000"))
{
firstString = getFirstTokenForRange(getLocalePrefixedText(part1, locale), fieldInstance);
if (firstString == null)
if (fieldInstance.isLocalised())
{
firstString = "{" + locale.getLanguage() + "}";
firstString = getFirstTokenForRange(getLocalePrefixedText(part1, locale), fieldInstance);
if (firstString == null)
{
firstString = "{" + locale.getLanguage() + "}";
}
} else
{
firstString = getFirstTokenForRange(part1, fieldInstance);
}
} else
{
firstString = getFirstTokenForRange(part1, fieldInstance);
if (fieldInstance.isLocalised())
{
firstString = "{" + locale.getLanguage() + "}";
} else
{
firstString = null;
}
}
} else
{
if (fieldInstance.isLocalised())
String lastString = null;
if ((part2 != null) && !part2.equals("\uffff"))
{
firstString = "{" + locale.getLanguage() + "}";
if (fieldInstance.isLocalised())
{
lastString = getFirstTokenForRange(getLocalePrefixedText(part2, locale), fieldInstance);
if (lastString == null)
{
lastString = "{" + locale.getLanguage() + "}\uffff";
}
} else
{
lastString = getFirstTokenForRange(part2, fieldInstance);
}
} else
{
if (fieldInstance.isLocalised())
{
lastString = "{" + locale.getLanguage() + "}\uffff";
} else
{
lastString = null;
}
}
// * is returned by stringField but it is not correct in case of range queries.
if (firstString != null && firstString.equals("*"))
{
firstString = null;
}
}
String lastString = null;
if ((part2 != null) && !part2.equals("\uffff"))
{
if (fieldInstance.isLocalised())
{
lastString = getFirstTokenForRange(getLocalePrefixedText(part2, locale), fieldInstance);
if (lastString == null)
{
lastString = "{" + locale.getLanguage() + "}\uffff";
}
} else
{
lastString = getFirstTokenForRange(part2, fieldInstance);
}
} else
{
if (fieldInstance.isLocalised())
{
lastString = "{" + locale.getLanguage() + "}\uffff";
} else
if (lastString != null && lastString.equals("*"))
{
lastString = null;
}
TermRangeQuery query = new TermRangeQuery(fieldInstance.getField(),
firstString == null ? null : new BytesRef(firstString),
lastString == null ? null : new BytesRef(lastString), includeLower, includeUpper);
booleanQuery.add(query, Occur.SHOULD);
}
// * is returned by stringField but it is not correct in case of range queries.
if (firstString != null && firstString.equals("*")){
firstString = null;
}
if (lastString != null && lastString.equals("*")){
lastString = null;
}
TermRangeQuery query = new TermRangeQuery(fieldInstance.getField(),
firstString == null ? null : new BytesRef(firstString),
lastString == null ? null : new BytesRef(lastString), includeLower, includeUpper);
booleanQuery.add(query, Occur.SHOULD);
}
private String getFirstTokenForRange(String string, FieldInstance field) throws IOException
search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/AclTracker.java
+122 -104
View File
@@ -56,8 +56,6 @@ import org.alfresco.solr.client.Acl;
import org.alfresco.solr.client.AclChangeSet;
import org.alfresco.solr.client.AclChangeSets;
import org.alfresco.solr.client.AclReaders;
import org.alfresco.solr.client.GetNodesParameters;
import org.alfresco.solr.client.Node;
import org.alfresco.solr.client.SOLRAPIClient;
import org.json.JSONException;
import org.slf4j.Logger;
@@ -73,21 +71,30 @@ public class AclTracker extends AbstractTracker
{
protected final static Logger LOGGER = LoggerFactory.getLogger(AclTracker.class);
private static final int ACL_CHANGE_SETS_FOUND_QUEUE_SIZE = 100;
private static final int DEFAULT_CHANGE_SET_ACLS_BATCH_SIZE = 2000;
private static final int DEFAULT_ACL_BATCH_SIZE = 100;
private static final int DEFAULT_ACL_TRACKER_MAX_PARALLELISM = 32;
private static final long DEFAULT_ACL_TRACKER_TIMESTEP = TIME_STEP_1_HR_IN_MS;
private static final long INITIAL_MAX_ACL_CHANGE_SET_ID = 2000L;
private static final int MAX_NUMBER_OF_ACL_CHANGE_SETS = 2000;
private static final long MAX_TIME_STEP = TIME_STEP_32_DAYS_IN_MS;
private int aclTrackerParallelism;
private int changeSetAclsBatchSize = DEFAULT_CHANGE_SET_ACLS_BATCH_SIZE;
private int aclBatchSize = DEFAULT_ACL_BATCH_SIZE;
private int changeSetAclsBatchSize;
private int aclBatchSize;
private long timeStep;
private int maxNumberOfAclChangeSets;
private ConcurrentLinkedQueue<Long> aclChangeSetsToReindex = new ConcurrentLinkedQueue<Long>();
private ConcurrentLinkedQueue<Long> aclChangeSetsToIndex = new ConcurrentLinkedQueue<Long>();
private ConcurrentLinkedQueue<Long> aclChangeSetsToPurge = new ConcurrentLinkedQueue<Long>();
private ConcurrentLinkedQueue<Long> aclsToReindex = new ConcurrentLinkedQueue<Long>();
private ConcurrentLinkedQueue<Long> aclsToIndex = new ConcurrentLinkedQueue<Long>();
private ConcurrentLinkedQueue<Long> aclsToPurge = new ConcurrentLinkedQueue<Long>();
private ConcurrentLinkedQueue<Long> aclChangeSetsToReindex = new ConcurrentLinkedQueue<>();
private ConcurrentLinkedQueue<Long> aclChangeSetsToIndex = new ConcurrentLinkedQueue<>();
private ConcurrentLinkedQueue<Long> aclChangeSetsToPurge = new ConcurrentLinkedQueue<>();
private ConcurrentLinkedQueue<Long> aclsToReindex = new ConcurrentLinkedQueue<>();
private ConcurrentLinkedQueue<Long> aclsToIndex = new ConcurrentLinkedQueue<>();
private ConcurrentLinkedQueue<Long> aclsToPurge = new ConcurrentLinkedQueue<>();
private DocRouter docRouter;
private ForkJoinPool forkJoinPool;
@@ -121,14 +128,21 @@ public class AclTracker extends AbstractTracker
super(p, client, coreName, informationServer, Tracker.Type.ACL);
changeSetAclsBatchSize = Integer.parseInt(p.getProperty("alfresco.changeSetAclsBatchSize",
String.valueOf(DEFAULT_CHANGE_SET_ACLS_BATCH_SIZE)));
aclBatchSize = Integer.parseInt(p.getProperty("alfresco.aclBatchSize", String.valueOf(DEFAULT_ACL_BATCH_SIZE)));
aclBatchSize = Integer.parseInt(p.getProperty("alfresco.aclBatchSize",
String.valueOf(DEFAULT_ACL_BATCH_SIZE)));
shardMethod = p.getProperty("shard.method", SHARD_METHOD_DBID);
docRouter = DocRouterFactory.getRouter(p, ShardMethodEnum.getShardMethod(shardMethod));
aclTrackerParallelism = Integer.parseInt(p.getProperty("alfresco.aclTrackerMaxParallelism",
aclTrackerParallelism = Integer.parseInt(p.getProperty("alfresco.acl.tracker.maxParallelism",
String.valueOf(DEFAULT_ACL_TRACKER_MAX_PARALLELISM)));
forkJoinPool = new ForkJoinPool(aclTrackerParallelism);
timeStep = Long.parseLong(p.getProperty("alfresco.acl.tracker.timestep",
String.valueOf(DEFAULT_ACL_TRACKER_TIMESTEP)));
maxNumberOfAclChangeSets = Integer.parseInt(p.getProperty("alfresco.acl.tracker.maxNumberOfAclChangeSets",
String.valueOf(MAX_NUMBER_OF_ACL_CHANGE_SETS)));
RUN_LOCK_BY_CORE.put(coreName, new Semaphore(1, true));
WRITE_LOCK_BY_CORE.put(coreName, new Semaphore(1, true));
}
@@ -168,8 +182,10 @@ public class AclTracker extends AbstractTracker
Long aclChangeSetId = aclChangeSetsToIndex.poll();
if (aclChangeSetId != null)
{
AclChangeSets aclChangeSets = client.getAclChangeSets(null, aclChangeSetId, null, aclChangeSetId+1, 1);
if ((aclChangeSets.getAclChangeSets().size() > 0) && aclChangeSetId.equals(aclChangeSets.getAclChangeSets().get(0).getId()))
AclChangeSets aclChangeSets = client.getAclChangeSets(null, aclChangeSetId,
null, aclChangeSetId+1, 1);
if ((aclChangeSets.getAclChangeSets().size() > 0) &&
aclChangeSetId.equals(aclChangeSets.getAclChangeSets().get(0).getId()))
{
AclChangeSet changeSet = aclChangeSets.getAclChangeSets().get(0);
List<Acl> acls = client.getAcls(Collections.singletonList(changeSet), null, Integer.MAX_VALUE);
@@ -184,9 +200,8 @@ public class AclTracker extends AbstractTracker
}
else
{
LOGGER.info(
"[CORE {}] - INDEX ACTION - AclChangeSetId {} was not found in database, it has NOT been reindexed",
coreName, aclChangeSetId);
LOGGER.info("[CORE {}] - INDEX ACTION - AclChangeSetId {} was not found in database, " +
"it has NOT been reindexed", coreName, aclChangeSetId);
}
}
checkShutdown();
@@ -227,11 +242,14 @@ public class AclTracker extends AbstractTracker
{
this.infoSrv.deleteByAclChangeSetId(aclChangeSetId);
AclChangeSets aclChangeSets = client.getAclChangeSets(null, aclChangeSetId, null, aclChangeSetId+1, 1);
if ((aclChangeSets.getAclChangeSets().size() > 0) && aclChangeSetId.equals(aclChangeSets.getAclChangeSets().get(0).getId()))
AclChangeSets aclChangeSets = client.getAclChangeSets(null, aclChangeSetId,
null, aclChangeSetId+1, 1);
if ((aclChangeSets.getAclChangeSets().size() > 0) &&
aclChangeSetId.equals(aclChangeSets.getAclChangeSets().get(0).getId()))
{
AclChangeSet changeSet = aclChangeSets.getAclChangeSets().get(0);
List<Acl> acls = client.getAcls(Collections.singletonList(changeSet), null, Integer.MAX_VALUE);
List<Acl> acls = client.getAcls(Collections.singletonList(changeSet),
null, Integer.MAX_VALUE);
for (Acl acl : acls)
{
List<AclReaders> readers = client.getAclReaders(Collections.singletonList(acl));
@@ -239,14 +257,14 @@ public class AclTracker extends AbstractTracker
}
this.infoSrv.indexAclTransaction(changeSet, true);
LOGGER.info("[CORE {}] - REINDEX ACTION - AclChangeSetId {} has been reindexed", coreName, aclChangeSetId);
LOGGER.info("[CORE {}] - REINDEX ACTION - AclChangeSetId {} has been reindexed",
coreName, aclChangeSetId);
requiresCommit = true;
}
else
{
LOGGER.info(
"[CORE {}] - REINDEX ACTION - AclChangeSetId {} was not found in database, it has NOT been reindexed",
coreName, aclChangeSetId);
LOGGER.info("[CORE {}] - REINDEX ACTION - AclChangeSetId {} was not found in database, " +
"it has NOT been reindexed", coreName, aclChangeSetId);
}
}
checkShutdown();
@@ -295,7 +313,6 @@ public class AclTracker extends AbstractTracker
}
checkShutdown();
}
}
protected void purgeAcls() throws IOException, JSONException
@@ -376,7 +393,8 @@ public class AclTracker extends AbstractTracker
state.setCheckedFirstAclTransactionTime(true);
LOGGER.info("[CORE {}] - No acl transactions found - no verification required", coreName);
firstChangeSets = client.getAclChangeSets(null, 0L, null, 2000L, 1);
firstChangeSets = client.getAclChangeSets(null, 0L,
null, INITIAL_MAX_ACL_CHANGE_SET_ID, 1);
if (!firstChangeSets.getAclChangeSets().isEmpty())
{
AclChangeSet firstChangeSet = firstChangeSets.getAclChangeSets().get(0);
@@ -388,19 +406,23 @@ public class AclTracker extends AbstractTracker
if (!state.isCheckedFirstAclTransactionTime())
{
firstChangeSets = client.getAclChangeSets(null, 0L, null, 2000L, 1);
firstChangeSets = client.getAclChangeSets(null, 0L,
null, INITIAL_MAX_ACL_CHANGE_SET_ID, 1);
if (!firstChangeSets.getAclChangeSets().isEmpty())
{
AclChangeSet firstAclChangeSet= firstChangeSets.getAclChangeSets().get(0);
long firstAclTxId = firstAclChangeSet.getId();
long firstAclTxCommitTime = firstAclChangeSet.getCommitTimeMs();
int setSize = this.infoSrv.getAclTxDocsSize(""+firstAclTxId, ""+firstAclTxCommitTime);
int setSize = this.infoSrv.getAclTxDocsSize(Long.toString(firstAclTxId),
Long.toString(firstAclTxCommitTime));
if (setSize == 0)
{
LOGGER.error("[CORE {}] First acl transaction was not found with the correct timestamp.", coreName);
LOGGER.error("SOLR has successfully connected to your repository however the SOLR indexes and repository database do not match.");
LOGGER.error("If this is a new or rebuilt database your SOLR indexes also need to be re-built to match the database.");
LOGGER.error("SOLR has successfully connected to your repository " +
"however the SOLR indexes and repository database do not match.");
LOGGER.error("If this is a new or rebuilt database your SOLR indexes " +
"also need to be re-built to match the database.");
LOGGER.error("You can also check your SOLR connection details in solrcore.properties.");
throw new AlfrescoRuntimeException("Initial acl transaction not found with correct timestamp");
}
@@ -421,7 +443,8 @@ public class AclTracker extends AbstractTracker
{
if (firstChangeSets == null)
{
firstChangeSets = client.getAclChangeSets(null, 0L, null, 2000L, 1);
firstChangeSets = client.getAclChangeSets(null, 0L,
null, INITIAL_MAX_ACL_CHANGE_SET_ID, 1);
}
setLastChangeSetIdAndCommitTimeInTrackerState(firstChangeSets.getAclChangeSets(), state);
@@ -432,19 +455,23 @@ public class AclTracker extends AbstractTracker
AclChangeSet maxAclTxInIndex = this.infoSrv.getMaxAclChangeSetIdAndCommitTimeInIndex();
if (maxAclTxInIndex.getCommitTimeMs() > maxChangeSetCommitTimeInRepo)
{
LOGGER.error("[CORE {}] Last acl transaction was found in index with timestamp later than that of repository.", coreName);
LOGGER.error("[CORE {}] Last acl transaction was found in index with " +
"timestamp later than that of repository.", coreName);
LOGGER.error("Max Acl Tx In Index: " + maxAclTxInIndex.getId() + ", In Repo: " + maxChangeSetIdInRepo);
LOGGER.error("Max Acl Tx Commit Time In Index: " + maxAclTxInIndex.getCommitTimeMs() + ", In Repo: "
+ maxChangeSetCommitTimeInRepo);
LOGGER.error("SOLR has successfully connected to your repository however the SOLR indexes and repository database do not match.");
LOGGER.error("If this is a new or rebuilt database your SOLR indexes also need to be re-built to match the database.");
LOGGER.error("SOLR has successfully connected to your repository" +
" however the SOLR indexes and repository database do not match.");
LOGGER.error("If this is a new or rebuilt database your SOLR indexes " +
"also need to be re-built to match the database.");
LOGGER.error("You can also check your SOLR connection details in solrcore.properties.");
throw new AlfrescoRuntimeException("Last acl transaction found in index with incorrect timestamp");
}
else
{
state.setCheckedLastAclTransactionTime(true);
LOGGER.info("[CORE {}] - Verified last acl transaction timestamp in index less than or equal to that of repository.", coreName);
LOGGER.info("[CORE {}] - Verified last acl transaction timestamp in index less" +
" than or equal to that of repository.", coreName);
}
}
}
@@ -467,7 +494,9 @@ public class AclTracker extends AbstractTracker
}
}
protected AclChangeSets getSomeAclChangeSets(BoundedDeque<AclChangeSet> changeSetsFound, Long fromCommitTime, long timeStep, int maxResults, long endTime) throws AuthenticationException, IOException, JSONException
protected AclChangeSets getSomeAclChangeSets(BoundedDeque<AclChangeSet> changeSetsFound, Long fromCommitTime,
long timeStep, int maxResults, long endTime)
throws AuthenticationException, IOException, JSONException
{
long actualTimeStep = timeStep;
@@ -477,15 +506,17 @@ public class AclTracker extends AbstractTracker
Long startTime = fromCommitTime == null ? Long.valueOf(0L) : fromCommitTime;
do
{
aclChangeSets = client.getAclChangeSets(startTime, null, startTime + actualTimeStep, null, maxResults);
aclChangeSets = client.getAclChangeSets(startTime, null,
startTime + actualTimeStep, null, maxResults);
startTime += actualTimeStep;
actualTimeStep *= 2;
if(actualTimeStep > TIME_STEP_32_DAYS_IN_MS)
if(actualTimeStep > MAX_TIME_STEP)
{
actualTimeStep = TIME_STEP_32_DAYS_IN_MS;
actualTimeStep = MAX_TIME_STEP;
}
}
while( ((aclChangeSets.getAclChangeSets().size() == 0) && (startTime < endTime)) || ((aclChangeSets.getAclChangeSets().size() > 0) && alreadyFoundChangeSets(changeSetsFound, aclChangeSets)));
while( ((aclChangeSets.getAclChangeSets().size() == 0) && (startTime < endTime)) ||
((aclChangeSets.getAclChangeSets().size() > 0) && alreadyFoundChangeSets(changeSetsFound, aclChangeSets)));
return aclChangeSets;
@@ -526,12 +557,13 @@ public class AclTracker extends AbstractTracker
trackerStats.addAclTime(time);
}
public IndexHealthReport checkIndex(Long toTx, Long toAclTx, Long fromTime, Long toTime)
public IndexHealthReport checkIndex(Long toAclTx, Long fromTime, Long toTime)
throws AuthenticationException, IOException, JSONException
{
// DB ACL TX Count
long firstChangeSetCommitTimex = 0;
AclChangeSets firstChangeSets = client.getAclChangeSets(null, 0L, null, 2000L, 1);
AclChangeSets firstChangeSets = client.getAclChangeSets(null, 0L,
null, INITIAL_MAX_ACL_CHANGE_SET_ID, 1);
if(firstChangeSets.getAclChangeSets().size() > 0)
{
AclChangeSet firstChangeSet = firstChangeSets.getAclChangeSets().get(0);
@@ -549,11 +581,11 @@ public class AclTracker extends AbstractTracker
Long minAclTxId = null;
long endTime = System.currentTimeMillis() + infoSrv.getHoleRetention();
AclChangeSets aclTransactions;
BoundedDeque<AclChangeSet> changeSetsFound = new BoundedDeque<>(100);
BoundedDeque<AclChangeSet> changeSetsFound = new BoundedDeque<>(ACL_CHANGE_SETS_FOUND_QUEUE_SIZE);
DO: do
{
aclTransactions = getSomeAclChangeSets(changeSetsFound, lastAclTxCommitTime, TIME_STEP_1_HR_IN_MS, 2000,
endTime);
aclTransactions = getSomeAclChangeSets(changeSetsFound,
lastAclTxCommitTime, timeStep, maxNumberOfAclChangeSets, endTime);
for (AclChangeSet set : aclTransactions.getAclChangeSets())
{
// include
@@ -597,7 +629,8 @@ public class AclTracker extends AbstractTracker
try
{
ArrayList<Long> answer = new ArrayList<>();
AclChangeSets changeSet = client.getAclChangeSets(null, acltxid, null, acltxid+1, 1);
AclChangeSets changeSet = client.getAclChangeSets(null, acltxid, null,
acltxid+1, 1);
List<Acl> acls = client.getAcls(changeSet.getAclChangeSets(), null, Integer.MAX_VALUE);
for (Acl acl : acls)
{
@@ -646,10 +679,9 @@ public class AclTracker extends AbstractTracker
{
long startElapsed = System.nanoTime();
boolean upToDate = false;
AclChangeSets aclChangeSets;
BoundedDeque<AclChangeSet> changeSetsFound = new BoundedDeque<AclChangeSet>(100);
BoundedDeque<AclChangeSet> changeSetsFound = new BoundedDeque<>(ACL_CHANGE_SETS_FOUND_QUEUE_SIZE);
long totalAclCount = 0;
int aclCount;
@@ -672,7 +704,7 @@ public class AclTracker extends AbstractTracker
Long fromCommitTime = getChangeSetFromCommitTime(changeSetsFound,
state.getLastChangeSetCommitTimeOnServer() == 0 ? state.getLastGoodChangeSetCommitTimeInIndex()
: state.getLastChangeSetCommitTimeOnServer());
aclChangeSets = getSomeAclChangeSets(changeSetsFound, fromCommitTime, TIME_STEP_1_HR_IN_MS, 2000,
aclChangeSets = getSomeAclChangeSets(changeSetsFound, fromCommitTime, timeStep, maxNumberOfAclChangeSets,
state.getTimeToStopIndexing());
if (aclChangeSets.getAclChangeSets().size() > 0)
@@ -690,53 +722,28 @@ public class AclTracker extends AbstractTracker
LOGGER.info("{}-[CORE {}] No ACL change set found after lastTxCommitTime {}",
Thread.currentThread().getId(), coreName, fromCommitTime);
}
// Ignore indexed ACL Change Sets
aclChangeSets = new AclChangeSets(aclChangeSets.getAclChangeSets().stream()
.filter(changeSet -> {
try
{
boolean isInIndex = (changeSet.getCommitTimeMs() <= state.getLastIndexedChangeSetCommitTime() &&
infoSrv.aclChangeSetInIndex(changeSet.getId(), true));
if (LOGGER.isTraceEnabled())
{
LOGGER.trace("{}-[CORE {}] Skipping change Set Id {} as it was already indexed",
Thread.currentThread().getId(), coreName, changeSet.getId());
}
return !isInIndex;
}
catch (IOException e)
{
LOGGER.warn(
"{}-[CORE {}] Error catched while checking if ACL Change Set {} was in index",
Thread.currentThread().getId(), coreName, changeSet.getId(), e);
return true;
}
})
.collect(Collectors.toList()));
// Make sure we do not go ahead of where we started - we will check the holes here
// correctly next time
if (aclChangeSets.getAclChangeSets().stream().anyMatch(changeSet -> changeSet.getCommitTimeMs() > state.getTimeToStopIndexing()))
if (aclChangeSets.getAclChangeSets()
.stream()
.anyMatch(changeSet -> changeSet.getCommitTimeMs() > state.getTimeToStopIndexing()))
{
break;
}
final AtomicInteger counter = new AtomicInteger();
Collection<List<AclChangeSet>> changeSetBatches = aclChangeSets.getAclChangeSets().stream()
.collect(Collectors.groupingBy(it -> counter.getAndAdd(it.getAclCount()) / changeSetAclsBatchSize)).values();
.peek(changeSetsFound::add)
.filter(this::isAclChangeSetAlreadyIndexed)
.collect(Collectors.groupingBy(it -> counter.getAndAdd(it.getAclCount()) / changeSetAclsBatchSize))
.values();
for (List<AclChangeSet> changeSetBatch : changeSetBatches)
{
aclCount = indexBatchOfChangeSets(changeSetBatch);
for (AclChangeSet indexed : changeSetBatch)
{
changeSetsFound.add(indexed);
}
// Update last committed transactions
setLastChangeSetIdAndCommitTimeInTrackerState(changeSetBatch, state);
indexAclChangeSetAfterWorker(changeSetBatch, state);
@@ -757,28 +764,47 @@ public class AclTracker extends AbstractTracker
}
}
while ((aclChangeSets.getAclChangeSets().size() > 0) && (upToDate == false));
while ((aclChangeSets.getAclChangeSets().size() > 0));
LOGGER.info("{}-[CORE {}] <end> Tracked {} ACLs", Thread.currentThread().getId(), coreName, totalAclCount);
}
private boolean isAclChangeSetAlreadyIndexed(AclChangeSet changeSet)
{
try
{
boolean isInIndex = (changeSet.getCommitTimeMs() <= state.getLastIndexedChangeSetCommitTime() &&
infoSrv.aclChangeSetInIndex(changeSet.getId(), true));
if (LOGGER.isTraceEnabled())
{
LOGGER.trace("{}-[CORE {}] Skipping change Set Id {} as it was already indexed",
Thread.currentThread().getId(), coreName, changeSet.getId());
}
return !isInIndex;
}
catch (IOException e)
{
LOGGER.warn(
"{}-[CORE {}] Error catched while checking if ACL Change Set {} was in index",
Thread.currentThread().getId(), coreName, changeSet.getId(), e);
return true;
}
}
private void setLastChangeSetIdAndCommitTimeInTrackerState(List<AclChangeSet> aclChangeSets, TrackerState state)
{
if (!aclChangeSets.isEmpty())
{
Long maxChangeSetCommitTime = aclChangeSets.stream().max(Comparator.comparing(AclChangeSet::getCommitTimeMs)).get().getCommitTimeMs();
if(maxChangeSetCommitTime != null)
{
state.setLastChangeSetCommitTimeOnServer(maxChangeSetCommitTime);
}
Long maxChangeSetId = aclChangeSets.stream().max(Comparator.comparing(AclChangeSet::getId)).get().getId();
if(maxChangeSetId != null)
{
state.setLastChangeSetIdOnServer(maxChangeSetId);
}
long maxChangeSetCommitTime =
aclChangeSets.stream().max(Comparator.comparing(AclChangeSet::getCommitTimeMs)).get().getCommitTimeMs();
state.setLastChangeSetCommitTimeOnServer(maxChangeSetCommitTime);
long maxChangeSetId = aclChangeSets.stream().max(Comparator.comparing(AclChangeSet::getId)).get().getId();
state.setLastChangeSetIdOnServer(maxChangeSetId);
}
}
@@ -804,15 +830,6 @@ public class AclTracker extends AbstractTracker
}
}
private int getAclCount(List<AclChangeSet> changeSetBatch)
{
int count = 0;
for (AclChangeSet set : changeSetBatch)
{
count += set.getAclCount();
}
return count;
}
/**
* Index ACLs from ACL Change Sets contained in changeSetBatch
@@ -822,7 +839,8 @@ public class AclTracker extends AbstractTracker
* @param changeSetBatch List of ACL Change Sets to be indexed
* @return List of ACL Change Set indexed and Count of ACL indexed
*/
private int indexBatchOfChangeSets(List<AclChangeSet> changeSetBatch) throws AuthenticationException, IOException, JSONException, ExecutionException, InterruptedException {
private int indexBatchOfChangeSets(List<AclChangeSet> changeSetBatch)
throws AuthenticationException, IOException, JSONException, ExecutionException, InterruptedException {
// Exclude ACL Change Set with no ACLs inside
List<AclChangeSet> nonEmptyChangeSets = changeSetBatch.stream()
.filter(set -> set.getAclCount() > 0)
search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/CascadeTracker.java
+2 -2
View File
@@ -92,10 +92,10 @@ public class CascadeTracker extends AbstractTracker implements Tracker
{
super(p, client, coreName, informationServer, Tracker.Type.CASCADE);
cascadeTrackerParallelism = Integer.parseInt(p.getProperty("alfresco.cascadeTrackerMaxParallelism",
cascadeTrackerParallelism = Integer.parseInt(p.getProperty("alfresco.cascade.tracker.maxParallelism",
String.valueOf(DEFAULT_CASCADE_TRACKER_MAX_PARALLELISM)));
cascadeBatchSize = Integer.parseInt(p.getProperty("alfresco.cascadeNodeBatchSize",
cascadeBatchSize = Integer.parseInt(p.getProperty("alfresco.cascade.tracker.nodeBatchSize",
String.valueOf(DEFAULT_CASCADE_NODE_BATCH_SIZE)));;
forkJoinPool = new ForkJoinPool(cascadeTrackerParallelism);
search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/ContentTracker.java
+1 -1
View File
@@ -83,7 +83,7 @@ public class ContentTracker extends AbstractTracker implements Tracker
contentUpdateBatchSize = Integer.parseInt(p.getProperty("alfresco.contentUpdateBatchSize",
String.valueOf(DEFAULT_CONTENT_UPDATE_BATCH_SIZE)));
contentTrackerParallelism = Integer.parseInt(p.getProperty("alfresco.contentTrackerMaxParallelism",
contentTrackerParallelism = Integer.parseInt(p.getProperty("alfresco.content.tracker.maxParallelism",
String.valueOf(DEFAULT_CONTENT_TRACKER_MAX_PARALLELISM)));
forkJoinPool = new ForkJoinPool(contentTrackerParallelism);
search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java
+152 -203
View File
@@ -49,7 +49,8 @@ import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
@@ -59,6 +60,7 @@ import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ForkJoinPool;
import java.util.concurrent.Semaphore;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
import static org.alfresco.repo.index.shard.ShardMethodEnum.DB_ID_RANGE;
@@ -71,26 +73,31 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
{
protected final static Logger LOGGER = LoggerFactory.getLogger(MetadataTracker.class);
private static final int METADATA_TRANSACTIONS_FOUND_QUEUE_SIZE = 100;
private static final int DEFAULT_METADATA_TRACKER_MAX_PARALLELISM = 32;
private static final int DEFAULT_TRANSACTION_DOCS_BATCH_SIZE = 2000;
private static final int DEFAULT_MAX_NUMBER_OF_TRANSACTIONS = 2000;
private static final int DEFAULT_NODE_BATCH_SIZE = 50;
private static final String DEFAULT_INITIAL_TRANSACTION_RANGE = "0-2000";
private static final long DEFAULT_METADATA_TRACKER_TIMESTEP = TIME_STEP_1_HR_IN_MS;
private static final long INITIAL_MAX_TXN_ID = 2000L;
private int matadataTrackerParallelism;
private int transactionDocsBatchSize;
private int nodeBatchSize;
private int maxNumberOfTransactions;
private long timeStep;
private ConcurrentLinkedQueue<Long> transactionsToReindex = new ConcurrentLinkedQueue<>();
private ConcurrentLinkedQueue<Long> transactionsToIndex = new ConcurrentLinkedQueue<>();
private ConcurrentLinkedQueue<Long> transactionsToPurge = new ConcurrentLinkedQueue<>();
private ConcurrentLinkedQueue<Long> nodesToReindex = new ConcurrentLinkedQueue<>();
private ConcurrentLinkedQueue<Long> nodesToIndex = new ConcurrentLinkedQueue<>();
private ConcurrentLinkedQueue<Long> nodesToPurge = new ConcurrentLinkedQueue<>();
private ConcurrentLinkedQueue<String> queriesToReindex = new ConcurrentLinkedQueue<>();
private final ConcurrentLinkedQueue<Long> transactionsToReindex = new ConcurrentLinkedQueue<>();
private final ConcurrentLinkedQueue<Long> transactionsToIndex = new ConcurrentLinkedQueue<>();
private final ConcurrentLinkedQueue<Long> transactionsToPurge = new ConcurrentLinkedQueue<>();
private final ConcurrentLinkedQueue<Long> nodesToReindex = new ConcurrentLinkedQueue<>();
private final ConcurrentLinkedQueue<Long> nodesToIndex = new ConcurrentLinkedQueue<>();
private final ConcurrentLinkedQueue<Long> nodesToPurge = new ConcurrentLinkedQueue<>();
private final ConcurrentLinkedQueue<String> queriesToReindex = new ConcurrentLinkedQueue<>();
private final boolean isRunningInProduction = !Boolean.parseBoolean(System.getProperty("alfresco.test", "false"));
private final boolean isRunningInProduction =
!Boolean.parseBoolean(System.getProperty("alfresco.test", "false"));
private ForkJoinPool forkJoinPool;
@@ -160,11 +167,18 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
String.valueOf(DEFAULT_TRANSACTION_DOCS_BATCH_SIZE)));
nodeBatchSize = Integer.parseInt(p.getProperty("alfresco.nodeBatchSize",
String.valueOf(DEFAULT_NODE_BATCH_SIZE)));
maxNumberOfTransactions = Integer.parseInt(p.getProperty("alfresco.maxNumberOfTransactions", String.valueOf(DEFAULT_MAX_NUMBER_OF_TRANSACTIONS)));
matadataTrackerParallelism = Integer.parseInt(p.getProperty("alfresco.metadataTrackerMaxParallelism",
maxNumberOfTransactions = Integer.parseInt(p.getProperty("alfresco.metadata.tracker.maxNumberOfTransactions",
String.valueOf(DEFAULT_MAX_NUMBER_OF_TRANSACTIONS)));
matadataTrackerParallelism = Integer.parseInt(p.getProperty("alfresco.metadata.tracker.maxParallelism",
String.valueOf(DEFAULT_METADATA_TRACKER_MAX_PARALLELISM)));
String[] minTxninitialRangeString = p.getProperty("solr.initial.transaction.range", DEFAULT_INITIAL_TRANSACTION_RANGE).split("-");
timeStep = Long.parseLong(p.getProperty("alfresco.metadata.tracker.timestep",
String.valueOf(DEFAULT_METADATA_TRACKER_TIMESTEP)));
String[] minTxninitialRangeString =
p.getProperty("solr.initial.transaction.range", DEFAULT_INITIAL_TRANSACTION_RANGE)
.split("-");
cascadeTrackerEnabled = informationServer.cascadeTrackingEnabled();
minTxnIdRange = new Pair<>(Long.valueOf(minTxninitialRangeString[0]), Long.valueOf(minTxninitialRangeString[1]));
forkJoinPool = new ForkJoinPool(matadataTrackerParallelism);
@@ -179,12 +193,13 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
// Try invoking getNextTxCommitTime service
try
{
client.getNextTxCommitTime(coreName, 0l);
client.getNextTxCommitTime(coreName, 0L);
nextTxCommitTimeServiceAvailable = true;
}
catch (NoSuchMethodException e)
{
LOGGER.warn("nextTxCommitTimeService is not available. Upgrade your ACS Repository version in order to use this feature: {} ", e.getMessage());
LOGGER.warn("nextTxCommitTimeService is not available. " +
"Upgrade your ACS Repository version in order to use this feature: {} ", e.getMessage());
}
catch (Exception e)
{
@@ -196,12 +211,13 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
{
try
{
client.getTxIntervalCommitTime(coreName, 0l, 0l);
client.getTxIntervalCommitTime(coreName, 0L, 0L);
txIntervalCommitTimeServiceAvailable = true;
}
catch (NoSuchMethodException e)
{
LOGGER.warn("txIntervalCommitTimeServiceAvailable is not available. Upgrade your ACS Repository version " +
LOGGER.warn("txIntervalCommitTimeServiceAvailable is not available. " +
"Upgrade your ACS Repository version " +
"to use this feature with DB_ID_RANGE sharding: {} ", e.getMessage());
}
catch (Exception e)
@@ -219,8 +235,8 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
}
@Override
protected void doTrack(String iterationId) throws AuthenticationException, IOException, JSONException, EncoderException
{
protected void doTrack(String iterationId)
throws AuthenticationException, IOException, JSONException {
// MetadataTracker must wait until ModelTracker has run
ModelTracker modelTracker = this.infoSrv.getAdminHandler().getTrackerRegistry().getModelTracker();
if (modelTracker != null && modelTracker.hasModels())
@@ -255,7 +271,7 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
queriesToReindex.size() > 0;
}
private void trackRepository() throws IOException, AuthenticationException, JSONException, EncoderException
private void trackRepository() throws IOException, AuthenticationException, JSONException
{
checkShutdown();
@@ -303,7 +319,8 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
state.setCheckedFirstTransactionTime(true);
LOGGER.info("No transactions found - no verification required");
firstTransactions = client.getTransactions(null, minTxnIdRange.getFirst(), null, minTxnIdRange.getSecond(), 1);
firstTransactions = client.getTransactions(null, minTxnIdRange.getFirst(),
null, minTxnIdRange.getSecond(), 1);
if (!firstTransactions.getTransactions().isEmpty())
{
Transaction firstTransaction = firstTransactions.getTransactions().get(0);
@@ -321,7 +338,7 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
// are skipped if they are not related with the range of the Shard.
// Getting the minCommitTime for the Shard is enough in order to check
// that the first transaction is present.
long minCommitTime = 0l;
long minCommitTime = 0L;
if (docRouter instanceof DBIDRangeRouter && txIntervalCommitTimeServiceAvailable)
{
try
@@ -333,28 +350,34 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
}
catch (NoSuchMethodException e)
{
LOGGER.warn("txIntervalCommitTimeServiceAvailable is not available. If you are using DB_ID_RANGE shard method, "
+ "upgrade your ACS Repository version in order to use the skip transactions feature: {} ", e.getMessage());
LOGGER.warn("txIntervalCommitTimeServiceAvailable is not available." +
" If you are using DB_ID_RANGE shard method, "
+ "upgrade your ACS Repository version in order to use the skip transactions feature: {} ",
e.getMessage());
}
}
// When a Shard with DB_ID_RANGE method is empty, minCommitTime is -1.
// No firstTransaction checking is required for this case.
if (minCommitTime != -1l) {
if (minCommitTime != -1L) {
firstTransactions = client.getTransactions(minCommitTime, 0L, null, 2000l, 1);
firstTransactions = client.getTransactions(minCommitTime, 0L,
null, INITIAL_MAX_TXN_ID, 1);
if (!firstTransactions.getTransactions().isEmpty())
{
Transaction firstTransaction = firstTransactions.getTransactions().get(0);
long firstTxId = firstTransaction.getId();
long firstTransactionCommitTime = firstTransaction.getCommitTimeMs();
int setSize = this.infoSrv.getTxDocsSize(""+firstTxId, ""+firstTransactionCommitTime);
int setSize = this.infoSrv.getTxDocsSize(Long.toString(firstTxId),
Long.toString(firstTransactionCommitTime));
if (setSize == 0)
{
LOGGER.error("First transaction was not found with the correct timestamp.");
LOGGER.error("SOLR has successfully connected to your repository however the SOLR indexes and repository database do not match.");
LOGGER.error("If this is a new or rebuilt database your SOLR indexes also need to be re-built to match the database.");
LOGGER.error("SOLR has successfully connected to your repository however the SOLR indexes" +
" and repository database do not match.");
LOGGER.error("If this is a new or rebuilt database your SOLR indexes also need to be " +
"re-built to match the database.");
LOGGER.error("You can also check your SOLR connection details in solrcore.properties.");
throw new AlfrescoRuntimeException("Initial transaction not found with correct timestamp");
}
@@ -376,7 +399,8 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
{
if (firstTransactions == null)
{
firstTransactions = client.getTransactions(null, minTxnIdRange.getFirst(), null, minTxnIdRange.getSecond(), 1);
firstTransactions = client.getTransactions(null, minTxnIdRange.getFirst(),
null, minTxnIdRange.getSecond(), 1);
}
setLastTxCommitTimeAndTxIdInTrackerState(firstTransactions);
@@ -391,8 +415,10 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
LOGGER.error("Max Tx In Index: " + maxTxInIndex.getId() + ", In Repo: " + maxTxnIdInRepo);
LOGGER.error("Max Tx Commit Time In Index: " + maxTxInIndex.getCommitTimeMs() + ", In Repo: "
+ maxTxnCommitTimeInRepo);
LOGGER.error("SOLR has successfully connected to your repository however the SOLR indexes and repository database do not match.");
LOGGER.error("If this is a new or rebuilt database your SOLR indexes also need to be re-built to match the database.");
LOGGER.error("SOLR has successfully connected to your repository however the SOLR indexes" +
" and repository database do not match.");
LOGGER.error("If this is a new or rebuilt database your SOLR indexes also need to " +
"be re-built to match the database.");
LOGGER.error("You can also check your SOLR connection details in solrcore.properties.");
throw new AlfrescoRuntimeException("Last transaction found in index with incorrect timestamp");
}
@@ -416,20 +442,22 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
Long transactionId = transactionsToIndex.poll();
if (transactionId != null)
{
Transactions transactions = client.getTransactions(null, transactionId, null, transactionId+1, 1);
if ((transactions.getTransactions().size() > 0) && (transactionId.equals(transactions.getTransactions().get(0).getId())))
Transactions transactions = client.getTransactions(null, transactionId,
null, transactionId + 1, 1);
if ((transactions.getTransactions().size() > 0) &&
(transactionId.equals(transactions.getTransactions().get(0).getId())))
{
Transaction info = transactions.getTransactions().get(0);
GetNodesParameters gnp = new GetNodesParameters();
ArrayList<Long> txs = new ArrayList<Long>();
ArrayList<Long> txs = new ArrayList<>();
txs.add(info.getId());
gnp.setTransactionIds(txs);
gnp.setStoreProtocol(storeRef.getProtocol());
gnp.setStoreIdentifier(storeRef.getIdentifier());
updateShardProperty();
shardProperty.ifPresent(p -> gnp.setShardProperty(p));
shardProperty.ifPresent(gnp::setShardProperty);
gnp.setCoreName(coreName);
@@ -455,27 +483,14 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
}
else
{
LOGGER.info("INDEX ACTION - Transaction {} was not found in database, it has NOT been reindexed", transactionId);
}
}
if (docCount > batchCount)
{
if(this.infoSrv.getRegisteredSearcherCount() < getMaxLiveSearchers())
{
checkShutdown();
long endElapsed = System.nanoTime();
trackerStats.addElapsedNodeTime(docCount, endElapsed-startElapsed);
startElapsed = endElapsed;
docCount = 0;
requiresCommit = false;
LOGGER.info("INDEX ACTION - Transaction {} was not found in database, it has NOT been reindexed",
transactionId);
}
}
}
if (requiresCommit || (docCount > 0))
if (requiresCommit)
{
checkShutdown();
//this.infoSrv.commit();
long endElapsed = System.nanoTime();
trackerStats.addElapsedNodeTime(docCount, endElapsed - startElapsed);
}
@@ -520,13 +535,15 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
// make sure it is cleaned out so we do not miss deletes
this.infoSrv.deleteByTransactionId(transactionId);
Transactions transactions = client.getTransactions(null, transactionId, null, transactionId+1, 1);
if ((transactions.getTransactions().size() > 0) && (transactionId.equals(transactions.getTransactions().get(0).getId())))
Transactions transactions = client.getTransactions(null, transactionId,
null, transactionId+1, 1);
if ((transactions.getTransactions().size() > 0) &&
(transactionId.equals(transactions.getTransactions().get(0).getId())))
{
Transaction info = transactions.getTransactions().get(0);
this.infoSrv.dirtyTransaction(info.getId());
GetNodesParameters gnp = new GetNodesParameters();
ArrayList<Long> txs = new ArrayList<Long>();
ArrayList<Long> txs = new ArrayList<>();
txs.add(info.getId());
gnp.setTransactionIds(txs);
gnp.setStoreProtocol(storeRef.getProtocol());
@@ -550,7 +567,8 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
}
else
{
LOGGER.info("REINDEX ACTION - Transaction {} was not found in database, it has NOT been reindexed", transactionId);
LOGGER.info("REINDEX ACTION - Transaction {} was not found in database, it has NOT been reindexed",
transactionId);
}
}
@@ -629,7 +647,7 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
}
private void purgeTransactions() throws IOException, AuthenticationException, JSONException
private void purgeTransactions() throws IOException, JSONException
{
boolean requiresCommit = false;
while (transactionsToPurge.peek() != null)
@@ -651,7 +669,7 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
}
}
private void purgeNodes() throws IOException, AuthenticationException, JSONException
private void purgeNodes() throws IOException, JSONException
{
while (nodesToPurge.peek() != null)
{
@@ -715,22 +733,21 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
}
protected Transactions getSomeTransactions(BoundedDeque<Transaction> txnsFound, Long fromCommitTime, long timeStep,
int maxResults, long endTime) throws AuthenticationException, IOException, JSONException, EncoderException, NoSuchMethodException
int maxResults, long endTime)
throws AuthenticationException, IOException, JSONException, EncoderException, NoSuchMethodException
{
long actualTimeStep = timeStep;
ShardState shardstate = getShardState();
Transactions transactions;
// step forward in time until we find something or hit the time bound
// max id unbounded
Long startTime = fromCommitTime == null ? 0L : fromCommitTime;
long startTime = fromCommitTime == null ? 0L : fromCommitTime;
if(startTime == 0)
{
return client.getTransactions(startTime,
null,
startTime + actualTimeStep,
startTime + timeStep,
null,
maxResults,
shardstate);
@@ -738,8 +755,9 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
do
{
transactions = client.getTransactions(startTime, null, startTime + actualTimeStep, null, maxResults, shardstate);
startTime += actualTimeStep;
transactions = client.getTransactions(startTime, null, startTime + timeStep,
null, maxResults, shardstate);
startTime += timeStep;
// If no transactions are found, advance the time window to the next available transaction commit time
if (nextTxCommitTimeServiceAvailable && transactions.getTransactions().size() == 0)
@@ -749,7 +767,8 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
{
LOGGER.info("{}-[CORE {}] Advancing transactions from {} to {}",
Thread.currentThread().getId(), coreName, startTime, nextTxCommitTime);
transactions = client.getTransactions(nextTxCommitTime, null, nextTxCommitTime + actualTimeStep, null, maxResults, shardstate);
transactions = client.getTransactions(nextTxCommitTime, null,
nextTxCommitTime + timeStep, null, maxResults, shardstate);
}
}
@@ -807,7 +826,7 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
fromCommitTime = shardMinCommitTime;
}
Transactions transactions = getSomeTransactions(txnsFound, fromCommitTime, TIME_STEP_1_HR_IN_MS, maxNumberOfTransactions,
Transactions transactions = getSomeTransactions(txnsFound, fromCommitTime, timeStep, maxNumberOfTransactions,
state.getTimeToStopIndexing());
@@ -819,7 +838,7 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
latestTransaction.setCommitTimeMs(transactions.getMaxTxnCommitTime());
latestTransaction.setId(transactions.getMaxTxnId());
transactions = new Transactions(
Arrays.asList(latestTransaction),
Collections.singletonList(latestTransaction),
transactions.getMaxTxnCommitTime(),
transactions.getMaxTxnId());
}
@@ -827,81 +846,49 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
return transactions;
}
/**
* Remove transactions already present in SOLR index
*
* @param transactions List of transactions to be indexed
* @return List of transactions not indexed in SOLR index
*/
private Transactions removeIndexedTransactions(Transactions transactions)
private boolean isTransactionIndexed(Transaction transaction)
{
return new Transactions(transactions.getTransactions().stream()
.filter(transaction -> {
try
{
boolean isInIndex = (transaction.getCommitTimeMs() <= state.getLastIndexedTxCommitTime() &&
infoSrv.txnInIndex(transaction.getId(), true));
if (LOGGER.isTraceEnabled())
{
LOGGER.trace("{}-[CORE {}] Skipping Transaction Id {} as it was already indexed",
Thread.currentThread().getId(), coreName, transaction.getId());
}
return !isInIndex;
}
catch (IOException e)
{
LOGGER.warn(
"{}-[CORE {}] Error catched while checking if Transaction Id {} was in index",
Thread.currentThread().getId(), coreName, transaction.getId(), e);
return true;
}
})
.collect(Collectors.toList()));
try
{
boolean isInIndex = (transaction.getCommitTimeMs() <= state.getLastIndexedTxCommitTime() &&
infoSrv.txnInIndex(transaction.getId(), true));
if (LOGGER.isTraceEnabled())
{
LOGGER.trace("{}-[CORE {}] Skipping Transaction Id {} as it was already indexed",
Thread.currentThread().getId(), coreName, transaction.getId());
}
return !isInIndex;
}
catch (IOException e)
{
LOGGER.warn(
"{}-[CORE {}] Error catched while checking if Transaction Id {} was in index",
Thread.currentThread().getId(), coreName, transaction.getId(), e);
return true;
}
}
//fixme remove
/**
* Keep only transactions previous to node transaction Id
*
* @param transactions List of transactions from Repository
* @param node Last Node indexed in the cycle
* @return Filtered list of transactions
*/
private Transactions filterTransactionsByNode(Transactions transactions, Node node)
{
return new Transactions(transactions.getTransactions().stream()
.filter(transaction -> {
return transaction.getId() < node.getTxnId();
})
.collect(Collectors.toList()));
}
/**
* Indexing new transactions from repository in batches of "transactionDocsBatchSize" size.
*
* Additionally, the nodes inside a transaction batch are indexed in batches of "nodeBatchSize" size.
*
* @throws AuthenticationException
* @throws IOException
* @throws JSONException
* @throws EncoderException
*/
protected void trackTransactions() throws AuthenticationException, IOException, JSONException, EncoderException
protected void trackTransactions() throws IOException, JSONException
{
long startElapsed = System.nanoTime();
boolean upToDate = false;
Transactions transactions;
BoundedDeque<Transaction> txnsFound = new BoundedDeque<Transaction>(100);
BoundedDeque<Transaction> txnsFound = new BoundedDeque<>(METADATA_TRANSACTIONS_FOUND_QUEUE_SIZE);
int totalUpdatedDocs = 0;
LOGGER.info("{}-[CORE {}] Starting metadata tracker execution", Thread.currentThread().getId(), coreName);
do
{
try
{
/*
@@ -918,7 +905,8 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
this.state = getTrackerState();
Long fromCommitTime = getTxFromCommitTime(txnsFound,
state.getLastIndexedTxCommitTime() == 0 ? state.getLastGoodTxCommitTimeInIndex() : state.getLastIndexedTxCommitTime());
state.getLastIndexedTxCommitTime() == 0 ? state.getLastGoodTxCommitTimeInIndex()
: state.getLastIndexedTxCommitTime());
// Get transaction list to be indexed
if (docRouter instanceof DBIDRangeRouter && txIntervalCommitTimeServiceAvailable)
@@ -927,12 +915,10 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
}
else
{
transactions = getSomeTransactions(txnsFound, fromCommitTime, TIME_STEP_1_HR_IN_MS, maxNumberOfTransactions,
transactions = getSomeTransactions(txnsFound, fromCommitTime, timeStep, maxNumberOfTransactions,
state.getTimeToStopIndexing());
}
// Remove transactions already indexed
transactions = removeIndexedTransactions(transactions);
if (transactions.getTransactions().size() > 0)
{
@@ -949,48 +935,45 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
LOGGER.info("{}-[CORE {}] No transaction found after lastTxCommitTime {}",
Thread.currentThread().getId(),
coreName,
((txnsFound.size() > 0) ? txnsFound.getLast().getCommitTimeMs() : state.getLastIndexedTxCommitTime()));
((txnsFound.size() > 0) ? txnsFound.getLast().getCommitTimeMs()
: state.getLastIndexedTxCommitTime()));
}
// Group the transactions in batches of transactionDocsBatchSize (or less)
List<List<Transaction>> txBatches = new ArrayList<>();
List<Transaction> txBatch = new ArrayList<>();
for (Transaction info : transactions.getTransactions()) {
if (LOGGER.isTraceEnabled())
{
LOGGER.trace("{}-[CORE {}] Tracking {} Transactions. Current Transaction Id to be indexed: {}",
Thread.currentThread().getId(), coreName, transactions.getTransactions().size(), info.getId());
}
// Make sure we do not go ahead of where we started - we will check the holes here
// correctly next time
if (info.getCommitTimeMs() > state.getTimeToStopIndexing()) {
upToDate = true;
break;
}
txBatch.add(info);
if (getUpdateAndDeleteCount(txBatch) > transactionDocsBatchSize) {
txBatches.add(txBatch);
txBatch = new ArrayList<>();
}
}
if (!txBatch.isEmpty())
// Make sure we do not go ahead of where we started - we will check the holes here
// correctly next time
if (transactions.getTransactions()
.stream()
.anyMatch(transaction -> transaction.getCommitTimeMs() > state.getTimeToStopIndexing()))
{
txBatches.add(txBatch);
break;
}
long transaction_number = transactions.getTransactions().size();
final AtomicInteger counter = new AtomicInteger();
Collection<List<Transaction>> txBatches = transactions.getTransactions().stream()
.peek(txnsFound::add)
.filter(this::isTransactionIndexed)
.peek(transaction -> {
if (LOGGER.isTraceEnabled())
{
LOGGER.trace("{}-[CORE {}] Tracking {} Transactions. Current Transaction Id to be indexed: {}",
Thread.currentThread().getId(), coreName, transaction_number, transaction.getId());
}
})
.collect(Collectors.groupingBy(transaction -> counter.getAndAdd(
(int) (transaction.getDeletes() + transaction.getUpdates())) / transactionDocsBatchSize))
.values();
// Index batches of transactions and the nodes updated or deleted within the transaction
for (List<Transaction> batch : txBatches)
{
// Index nodes contained in the transactions
int docCount = indexBatchOfTransactions(batch, totalUpdatedDocs);
int docCount = indexBatchOfTransactions(batch);
totalUpdatedDocs += docCount;
// Add the transactions as found to avoid processing them again in the next iteration
batch.forEach(transaction -> txnsFound.add(transaction));
batch.forEach(txnsFound::add);
// Index the transactions
indexTransactionsAfterWorker(batch);
@@ -999,10 +982,7 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
startElapsed = endElapsed;
}
setLastTxCommitTimeAndTxIdInTrackerState(transactions);
}
catch(Exception e)
{
@@ -1014,7 +994,7 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
}
}
while ((transactions.getTransactions().size() > 0) && (upToDate == false));
while ((transactions.getTransactions().size() > 0));
LOGGER.info("{}-[CORE {}] Tracked {} DOCs", Thread.currentThread().getId(), coreName, totalUpdatedDocs);
}
@@ -1062,20 +1042,6 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
txsIndexed.clear();
}
/**
* Return the number of updated and deleted nodes in a list of transactions
* @param txs List of transactions
* @return Number of updated and deleted nodes
*/
private long getUpdateAndDeleteCount(List<Transaction> txs)
{
long count = 0;
for (Transaction tx : txs)
{
count += (tx.getUpdates() + tx.getDeletes());
}
return count;
}
/**
* Index a batch of transactions.
@@ -1084,23 +1050,20 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
* the metadata of the nodes in smaller invocations to Repository
*
* @param txBatch Batch of transactions to be indexed
* @param indexedNodes Number of nodes indexed in this Tracker execution
*
* @return Number of nodes indexed and last node indexed
*
* @throws AuthenticationException
* @throws IOException
* @throws JSONException
*/
private int indexBatchOfTransactions(List<Transaction> txBatch, int indexedNodes) throws AuthenticationException, IOException, JSONException, ExecutionException, InterruptedException {
private int indexBatchOfTransactions(List<Transaction> txBatch)
throws AuthenticationException, IOException, JSONException, ExecutionException, InterruptedException {
// Skip transactions without modifications (updates, deletes)
ArrayList<Transaction> nonEmptyTxs = new ArrayList<>(txBatch.size());
ArrayList<Long> txIds = new ArrayList<>();
for (Transaction tx : txBatch)
{
if (tx.getUpdates() > 0 || tx.getDeletes() > 0)
{
nonEmptyTxs.add(tx);
txIds.add(tx.getId());
}
}
@@ -1118,22 +1081,18 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
if (LOGGER.isDebugEnabled())
{
LOGGER.debug("{}-[CORE {}] Found {} Nodes to be indexed from Transactions: {}", Thread.currentThread().getId(),
coreName, nodes.size(), txIds);
LOGGER.debug("{}-[CORE {}] Found {} Nodes to be indexed from Transactions: {}",
Thread.currentThread().getId(), coreName, nodes.size(), txIds);
}
// Group the nodes in batches of nodeBatchSize (or less)
List<List<Node>> nodeBatches = Lists.partition(nodes, nodeBatchSize);
Integer processedNodes = forkJoinPool.submit(() ->
return forkJoinPool.submit(() ->
nodeBatches.parallelStream().map(batch -> {
new NodeIndexWorker(batch, infoSrv).run();
return batch.size();
}).reduce(0, Integer::sum)).get();
return processedNodes;
}
@@ -1260,7 +1219,7 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
try
{
GetNodesParameters gnp = new GetNodesParameters();
ArrayList<Long> txs = new ArrayList<Long>();
ArrayList<Long> txs = new ArrayList<>();
txs.add(txid);
gnp.setTransactionIds(txs);
gnp.setStoreProtocol(storeRef.getProtocol());
@@ -1268,26 +1227,19 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
gnp.setCoreName(coreName);
return client.getNodes(gnp, Integer.MAX_VALUE);
}
catch (IOException e)
{
throw new AlfrescoRuntimeException("Failed to get nodes", e);
}
catch (JSONException e)
{
throw new AlfrescoRuntimeException("Failed to get nodes", e);
}
catch (AuthenticationException e)
catch (IOException | AuthenticationException | JSONException e)
{
throw new AlfrescoRuntimeException("Failed to get nodes", e);
}
}
public IndexHealthReport checkIndex(Long toTx, Long toAclTx, Long fromTime, Long toTime)
public IndexHealthReport checkIndex(Long toTx, Long fromTime, Long toTime)
throws IOException, AuthenticationException, JSONException, EncoderException, NoSuchMethodException
{
// DB TX Count
long firstTransactionCommitTime = 0;
Transactions firstTransactions = client.getTransactions(null, 0L, null, 2000l, 1);
Transactions firstTransactions = client.getTransactions(null, 0L,
null, INITIAL_MAX_TXN_ID, 1);
if(firstTransactions.getTransactions().size() > 0)
{
Transaction firstTransaction = firstTransactions.getTransactions().get(0);
@@ -1295,7 +1247,7 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
}
IOpenBitSet txIdsInDb = infoSrv.getOpenBitSetInstance();
Long lastTxCommitTime = Long.valueOf(firstTransactionCommitTime);
long lastTxCommitTime = firstTransactionCommitTime;
if (fromTime != null)
{
lastTxCommitTime = fromTime;
@@ -1304,24 +1256,24 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
Long minTxId = null;
Transactions transactions;
BoundedDeque<Transaction> txnsFound = new BoundedDeque<Transaction>(100);
BoundedDeque<Transaction> txnsFound = new BoundedDeque<>(METADATA_TRANSACTIONS_FOUND_QUEUE_SIZE);
long endTime = System.currentTimeMillis() + infoSrv.getHoleRetention();
DO: do
{
transactions = getSomeTransactions(txnsFound, lastTxCommitTime, TIME_STEP_1_HR_IN_MS, maxNumberOfTransactions, endTime);
transactions = getSomeTransactions(txnsFound, lastTxCommitTime, timeStep, maxNumberOfTransactions, endTime);
for (Transaction info : transactions.getTransactions())
{
// include
if (toTime != null)
{
if (info.getCommitTimeMs() > toTime.longValue())
if (info.getCommitTimeMs() > toTime)
{
break DO;
}
}
if (toTx != null)
{
if (info.getId() > toTx.longValue())
if (info.getId() > toTx)
{
break DO;
}
@@ -1382,9 +1334,6 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker
infoSrv.clearProcessedTransactions();
}
/**
* @param query
*/
public void addQueryToReindex(String query)
{
this.queriesToReindex.offer(query);
search-services/alfresco-search/src/main/resources/solr/instance/templates/rerank/conf/solrcore.properties
+5
View File
@@ -11,6 +11,11 @@ enable.alfresco.tracking=true
#data.dir.store=workspace/SpacesStore
#alfresco.stores=workspace://SpacesStore
# Max transaction to schedule for reindexing in the admin FIX tool.
# If the admin fix action is called with a "maxScheduledTransactions" request parameter
# then that is used (i.e. the value configured in this file is ignored).
alfresco.admin.fix.maxScheduledTransactions=500
#
# Properties loaded during alfresco tracking
#
search-services/alfresco-search/src/test/java/org/alfresco/solr/AlfrescoCoreAdminHandlerTest.java
+512
View File
@@ -0,0 +1,512 @@
/*
* #%L
* Alfresco Search Services
* %%
* Copyright (C) 2005 - 2020 Alfresco Software Limited
* %%
* This file is part of the Alfresco software.
* If the software was purchased under a paid Alfresco license, the terms of
* the paid license agreement will prevail. Otherwise, the software is
* provided under the following open source license terms:
*
* Alfresco is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Alfresco is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
* #L%
*/
package org.alfresco.solr;
import org.alfresco.solr.adapters.IOpenBitSet;
import org.alfresco.solr.adapters.SolrOpenBitSetAdapter;
import org.alfresco.solr.tracker.AclTracker;
import org.alfresco.solr.tracker.IndexHealthReport;
import org.alfresco.solr.tracker.MetadataTracker;
import org.alfresco.solr.tracker.TrackerRegistry;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.core.CoreContainer;
import org.apache.solr.core.SolrCore;
import org.apache.solr.core.SolrResourceLoader;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mock;
import org.mockito.junit.MockitoJUnitRunner;
import java.util.Properties;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Consumer;
import static java.util.Optional.of;
import static java.util.stream.IntStream.range;
import static org.alfresco.solr.AlfrescoCoreAdminHandler.ACL_TX_IN_INDEX_NOT_IN_DB;
import static org.alfresco.solr.AlfrescoCoreAdminHandler.ACTION_ERROR_MESSAGE_LABEL;
import static org.alfresco.solr.AlfrescoCoreAdminHandler.ACTION_STATUS_LABEL;
import static org.alfresco.solr.AlfrescoCoreAdminHandler.ACTION_STATUS_NOT_SCHEDULED;
import static org.alfresco.solr.AlfrescoCoreAdminHandler.ALFRESCO_CORE_NAME;
import static org.alfresco.solr.AlfrescoCoreAdminHandler.ARCHIVE_CORE_NAME;
import static org.alfresco.solr.AlfrescoCoreAdminHandler.DRY_RUN_PARAMETER_NAME;
import static org.alfresco.solr.AlfrescoCoreAdminHandler.DUPLICATED_ACL_TX_IN_INDEX;
import static org.alfresco.solr.AlfrescoCoreAdminHandler.DUPLICATED_TX_IN_INDEX;
import static org.alfresco.solr.AlfrescoCoreAdminHandler.FROM_TX_COMMIT_TIME_PARAMETER_NAME;
import static org.alfresco.solr.AlfrescoCoreAdminHandler.MAX_TRANSACTIONS_TO_SCHEDULE_PARAMETER_NAME;
import static org.alfresco.solr.AlfrescoCoreAdminHandler.MAX_TRANSACTIONS_TO_SCHEDULE_CONF_PROPERTY_NAME;
import static org.alfresco.solr.AlfrescoCoreAdminHandler.MISSING_ACL_TX_IN_INDEX;
import static org.alfresco.solr.AlfrescoCoreAdminHandler.MISSING_TX_IN_INDEX;
import static org.alfresco.solr.AlfrescoCoreAdminHandler.TO_TX_COMMIT_TIME_PARAMETER_NAME;
import static org.alfresco.solr.AlfrescoCoreAdminHandler.TX_IN_INDEX_NOT_IN_DB;
import static org.alfresco.solr.AlfrescoCoreAdminHandler.UNKNOWN_CORE_MESSAGE;
import static org.alfresco.solr.AlfrescoCoreAdminHandler.UNPROCESSABLE_REQUEST_ON_SLAVE_NODES;
import static org.apache.solr.common.params.CoreAdminParams.CORE;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@RunWith(MockitoJUnitRunner.class)
public class AlfrescoCoreAdminHandlerTest
{
private AlfrescoCoreAdminHandler admin;
@Mock
TrackerRegistry registry;
private ModifiableSolrParams params;
@Before
public void setUp()
{
admin = new AlfrescoCoreAdminHandler();
admin.trackerRegistry = registry;
when(registry.getCoreNames()).thenReturn(Set.of(ALFRESCO_CORE_NAME, ARCHIVE_CORE_NAME));
params = new ModifiableSolrParams();
}
@Test
public void noTargetCoreInParams()
{
assertEquals(0, params.size());
NamedList<Object> actionResponse = admin.actionFIX(params);
assertEquals(0, actionResponse.size());
}
@Test
public void unknownTargetCoreInParams()
{
String invalidCoreName = "thisIsAnInvalidOrAtLeastUnknownCoreName";
params.set(CORE, invalidCoreName);
NamedList<Object> actionResponse = admin.actionFIX(params);
assertEquals( 1, actionResponse.size());
assertEquals(UNKNOWN_CORE_MESSAGE + invalidCoreName, actionResponse.get(ACTION_ERROR_MESSAGE_LABEL));
}
@Test
public void fixOnSlaveNodeHasNoEffect()
{
params.set(CORE, ALFRESCO_CORE_NAME);
assertFalse(admin.isMasterOrStandalone(ALFRESCO_CORE_NAME));
NamedList<Object> actionResponse = admin.actionFIX(params);
assertEquals( 1, actionResponse.size());
assertEquals(UNPROCESSABLE_REQUEST_ON_SLAVE_NODES, actionResponse.get(ACTION_ERROR_MESSAGE_LABEL));
}
@Test
public void maxTransactionScheduledParameterIsNotNull()
{
int expectedMaxTransactionToSchedule = 12876;
params.set(CORE, ALFRESCO_CORE_NAME);
params.set(MAX_TRANSACTIONS_TO_SCHEDULE_PARAMETER_NAME, expectedMaxTransactionToSchedule);
admin = new AlfrescoCoreAdminHandler();
assertEquals(expectedMaxTransactionToSchedule, admin.getMaxTransactionToSchedule(params));
}
@Test
public void maxTransactionScheduledIsNull_shouldBeGatheredFromCoreProperties()
{
params.set(CORE, ALFRESCO_CORE_NAME);
int expectedMaxTransactionToSchedule = 17892;
Properties coreProperties = new Properties();
coreProperties.setProperty(
MAX_TRANSACTIONS_TO_SCHEDULE_CONF_PROPERTY_NAME,
String.valueOf(expectedMaxTransactionToSchedule));
CoreContainer coreContainer = mock(CoreContainer.class);
SolrCore core = mock(SolrCore.class);
SolrResourceLoader resourceLoader = mock(SolrResourceLoader.class);
when(coreContainer.getCore(ALFRESCO_CORE_NAME)).thenReturn(core);
when(core.getResourceLoader()).thenReturn(resourceLoader);
when(resourceLoader.getCoreProperties()).thenReturn(coreProperties);
admin = new AlfrescoCoreAdminHandler(coreContainer);
assertEquals(expectedMaxTransactionToSchedule, admin.getMaxTransactionToSchedule(params));
}
@Test
public void maxTransactionScheduledParameterAndConfigurationIsNull_shouldGetTheHardCodedDefault()
{
params.set(CORE, ALFRESCO_CORE_NAME);
Properties coreProperties = new Properties();
CoreContainer coreContainer = mock(CoreContainer.class);
SolrCore core = mock(SolrCore.class);
SolrResourceLoader resourceLoader = mock(SolrResourceLoader.class);
when(coreContainer.getCore(ALFRESCO_CORE_NAME)).thenReturn(core);
when(core.getResourceLoader()).thenReturn(resourceLoader);
when(resourceLoader.getCoreProperties()).thenReturn(coreProperties);
admin = new AlfrescoCoreAdminHandler(coreContainer);
assertEquals(Integer.MAX_VALUE, admin.getMaxTransactionToSchedule(params));
}
@Test
public void masterOrStandaloneNode_implicitDryRunParameterIsEchoed()
{
admin = new AlfrescoCoreAdminHandler() {
@Override
NamedList<Object> fixOnSpecificCore(
String coreName,
Long fromTxCommitTime,
Long toTxCommitTime,
boolean dryRun,
int maxTransactionsToSchedule) {
return new NamedList<>(); // dummy entry
}
@Override
boolean isMasterOrStandalone(String coreName)
{
return true;
}
};
admin.trackerRegistry = registry;
params.set(CORE, ALFRESCO_CORE_NAME);
NamedList<Object> actionResponse = admin.actionFIX(params);
assertEquals(true, actionResponse.get(DRY_RUN_PARAMETER_NAME));
assertEquals(ACTION_STATUS_NOT_SCHEDULED, actionResponse.get(ACTION_STATUS_LABEL));
}
@Test
public void masterOrStandaloneNode_explicitDryRunParameterIsEchoed()
{
assertThatExplicitParameterIsEchoed(
DRY_RUN_PARAMETER_NAME,
true);
assertThatExplicitParameterIsEchoed(
DRY_RUN_PARAMETER_NAME,
false);
}
@Test
public void masterOrStandaloneNode_explicitFromCommitTimeParameterIsEchoed()
{
assertThatExplicitParameterIsEchoed(
FROM_TX_COMMIT_TIME_PARAMETER_NAME,
System.currentTimeMillis());
}
@Test
public void masterOrStandaloneNode_explicitToCommitTimeParameterIsEchoed()
{
assertThatExplicitParameterIsEchoed(
TO_TX_COMMIT_TIME_PARAMETER_NAME,
System.currentTimeMillis());
}
@Test
public void masterOrStandaloneNode_explicitMaxTransactionsToScheduleParameterIsEchoed()
{
assertThatExplicitParameterIsEchoed(
MAX_TRANSACTIONS_TO_SCHEDULE_PARAMETER_NAME,
Integer.MAX_VALUE);
}
@Test
public void manageTransactionsToBeFixed_shouldRespectTheInputGlobalLimit()
{
AtomicInteger limit = new AtomicInteger(4);
AtomicInteger transactionCount = new AtomicInteger();
IOpenBitSet transactions = new SolrOpenBitSetAdapter();
range(1, 7).forEach(transactions::set);
Consumer<Long> counter = tx -> transactionCount.incrementAndGet();
admin.manageTransactionsToBeFixed(transactions, tx -> 0, counter, limit);
assertEquals(-1, limit.get());
assertEquals(4, transactionCount.get());
}
@Test
public void subsequentInvocationsToManageTransactionsToBeFixed_shouldRespectTheInputGlobalLimit()
{
// Limit is set to max 13 transactions
AtomicInteger limit = new AtomicInteger(13);
AtomicInteger transactionCount = new AtomicInteger();
// First transaction set contains 6 transactions
IOpenBitSet firstTransactionSet = new SolrOpenBitSetAdapter();
range(1, 7).forEach(firstTransactionSet::set);
Consumer<Long> counter = tx -> transactionCount.incrementAndGet();
admin.manageTransactionsToBeFixed(firstTransactionSet, tx -> 0, counter, limit);
assertEquals(7, limit.get());
assertEquals(6, transactionCount.get());
// Second transaction set contains 9 transactions (more than the remaining transactions to process)
IOpenBitSet secondTransactionSet = new SolrOpenBitSetAdapter();
range(10, 21).forEach(secondTransactionSet::set);
admin.manageTransactionsToBeFixed(secondTransactionSet, tx -> 0, counter, limit);
assertEquals("Global transaction limit should have been exceeded", -1, limit.get());
assertEquals(13, transactionCount.get());
// Third transaction set contains 10 transactions, it should be completely ignored as we already exceeded the
// global limit above
IOpenBitSet thirdTransactionSet = new SolrOpenBitSetAdapter();
range(31, 42).forEach(thirdTransactionSet::set);
Consumer<Long> thisShoulndtBeInvoked = tx -> { throw new RuntimeException("We should never be here, as the global limit has been already exceeded."); };
admin.manageTransactionsToBeFixed(thirdTransactionSet, tx -> 0, thisShoulndtBeInvoked, limit);
assertEquals( -2, limit.get());
assertEquals(13, transactionCount.get());
}
@Test
public void noAclTransactionToReindex_shouldReturnAnEmptyResponse()
{
IndexHealthReport emptyReport = mock(IndexHealthReport.class);
when(emptyReport.getAclTxInIndexButNotInDb()).thenReturn(new SolrOpenBitSetAdapter());
when(emptyReport.getDuplicatedAclTxInIndex()).thenReturn(new SolrOpenBitSetAdapter());
when(emptyReport.getMissingAclTxFromIndex()).thenReturn(new SolrOpenBitSetAdapter());
NamedList<Object> subReport = admin.aclTxToReindex(ALFRESCO_CORE_NAME, mock(AclTracker.class), emptyReport, tx -> {}, Integer.MAX_VALUE);
assertEquals(
Long.valueOf(0L),
of(subReport.get(ACL_TX_IN_INDEX_NOT_IN_DB))
.map(NamedList.class::cast)
.map(NamedList::size)
.map(Number::longValue)
.orElseThrow(() -> new RuntimeException(ACL_TX_IN_INDEX_NOT_IN_DB + " section not found in response.")));
assertEquals(
Long.valueOf(0L),
of(subReport.get(DUPLICATED_ACL_TX_IN_INDEX))
.map(NamedList.class::cast)
.map(NamedList::size)
.map(Number::longValue)
.orElseThrow(() -> new RuntimeException(DUPLICATED_ACL_TX_IN_INDEX + " section not found in response.")));
assertEquals(
Long.valueOf(0L),
of(subReport.get(MISSING_ACL_TX_IN_INDEX))
.map(NamedList.class::cast)
.map(NamedList::size)
.map(Number::longValue)
.orElseThrow(() -> new RuntimeException(MISSING_ACL_TX_IN_INDEX + " section not found in response.")));
}
@Test
public void noTransactionToReindex_shouldReturnAnEmptyResponse()
{
IndexHealthReport emptyReport = mock(IndexHealthReport.class);
when(emptyReport.getTxInIndexButNotInDb()).thenReturn(new SolrOpenBitSetAdapter());
when(emptyReport.getDuplicatedTxInIndex()).thenReturn(new SolrOpenBitSetAdapter());
when(emptyReport.getMissingTxFromIndex()).thenReturn(new SolrOpenBitSetAdapter());
NamedList<Object> subReport = admin.txToReindex(ALFRESCO_CORE_NAME, mock(MetadataTracker.class), emptyReport, tx -> {}, Integer.MAX_VALUE);
assertEquals(
Long.valueOf(0L),
of(subReport.get(TX_IN_INDEX_NOT_IN_DB))
.map(NamedList.class::cast)
.map(NamedList::size)
.map(Number::longValue)
.orElseThrow(() -> new RuntimeException(TX_IN_INDEX_NOT_IN_DB + " section not found in response.")));
assertEquals(
Long.valueOf(0L),
of(subReport.get(DUPLICATED_TX_IN_INDEX))
.map(NamedList.class::cast)
.map(NamedList::size)
.map(Number::longValue)
.orElseThrow(() -> new RuntimeException(DUPLICATED_TX_IN_INDEX + " section not found in response.")));
assertEquals(
Long.valueOf(0L),
of(subReport.get(MISSING_TX_IN_INDEX))
.map(NamedList.class::cast)
.map(NamedList::size)
.map(Number::longValue)
.orElseThrow(() -> new RuntimeException(MISSING_TX_IN_INDEX + " section not found in response.")));
}
@Test
public void maxTransactionsGlobalLimitShouldBeAppliedInCascade()
{
SolrInformationServer server = mock(SolrInformationServer.class);
when(server.getDocListSize(anyString())).thenReturn(0);
ConcurrentHashMap<String, InformationServer> informationServers = new ConcurrentHashMap<>();
informationServers.put(ALFRESCO_CORE_NAME, server);
admin.informationServers = informationServers;
IOpenBitSet txInIndexButNotInDb = new SolrOpenBitSetAdapter();
IOpenBitSet duplicatedTxInIndex = new SolrOpenBitSetAdapter();
IOpenBitSet missingTxFromIndex = new SolrOpenBitSetAdapter();
range(1, 10).forEach(txInIndexButNotInDb::set);
range(21, 32).forEach(duplicatedTxInIndex::set);
range(50, 61).forEach(missingTxFromIndex::set);
int maxTransactionToSchedule = (int) (txInIndexButNotInDb.cardinality() +
duplicatedTxInIndex.cardinality() +
missingTxFromIndex.cardinality() -
5);
IndexHealthReport emptyReport = mock(IndexHealthReport.class);
when(emptyReport.getTxInIndexButNotInDb()).thenReturn(txInIndexButNotInDb);
when(emptyReport.getDuplicatedTxInIndex()).thenReturn(duplicatedTxInIndex);
when(emptyReport.getMissingTxFromIndex()).thenReturn(missingTxFromIndex);
NamedList<Object> subReport = admin.txToReindex(ALFRESCO_CORE_NAME, mock(MetadataTracker.class), emptyReport, tx -> {}, maxTransactionToSchedule);
assertEquals(
Long.valueOf(txInIndexButNotInDb.cardinality()),
of(subReport.get(TX_IN_INDEX_NOT_IN_DB))
.map(NamedList.class::cast)
.map(NamedList::size)
.map(Number::longValue)
.orElseThrow(() -> new RuntimeException(TX_IN_INDEX_NOT_IN_DB + " section not found in response.")));
assertEquals(
Long.valueOf(duplicatedTxInIndex.cardinality()),
of(subReport.get(DUPLICATED_TX_IN_INDEX))
.map(NamedList.class::cast)
.map(NamedList::size)
.map(Number::longValue)
.orElseThrow(() -> new RuntimeException(DUPLICATED_TX_IN_INDEX + " section not found in response.")));
assertEquals(
Long.valueOf(missingTxFromIndex.cardinality() - 5),
of(subReport.get(MISSING_TX_IN_INDEX))
.map(NamedList.class::cast)
.map(NamedList::size)
.map(Number::longValue)
.orElseThrow(() -> new RuntimeException(MISSING_TX_IN_INDEX + " section not found in response.")));
}
@Test
public void maxAclTransactionsGlobalLimitShouldBeAppliedInCascade()
{
SolrInformationServer server = mock(SolrInformationServer.class);
when(server.getDocListSize(anyString())).thenReturn(0);
ConcurrentHashMap<String, InformationServer> informationServers = new ConcurrentHashMap<>();
informationServers.put(ALFRESCO_CORE_NAME, server);
admin.informationServers = informationServers;
IOpenBitSet txInIndexButNotInDb = new SolrOpenBitSetAdapter();
IOpenBitSet duplicatedTxInIndex = new SolrOpenBitSetAdapter();
IOpenBitSet missingTxFromIndex = new SolrOpenBitSetAdapter();
range(1, 10).forEach(txInIndexButNotInDb::set);
range(21, 32).forEach(duplicatedTxInIndex::set);
range(50, 61).forEach(missingTxFromIndex::set);
int maxTransactionToSchedule = (int) (txInIndexButNotInDb.cardinality() +
duplicatedTxInIndex.cardinality() +
missingTxFromIndex.cardinality() -
5);
IndexHealthReport emptyReport = mock(IndexHealthReport.class);
when(emptyReport.getAclTxInIndexButNotInDb()).thenReturn(txInIndexButNotInDb);
when(emptyReport.getDuplicatedAclTxInIndex()).thenReturn(duplicatedTxInIndex);
when(emptyReport.getMissingAclTxFromIndex()).thenReturn(missingTxFromIndex);
NamedList<Object> subReport = admin.aclTxToReindex(ALFRESCO_CORE_NAME, mock(AclTracker.class), emptyReport, tx -> {}, maxTransactionToSchedule);
assertEquals(
Long.valueOf(txInIndexButNotInDb.cardinality()),
of(subReport.get(ACL_TX_IN_INDEX_NOT_IN_DB))
.map(NamedList.class::cast)
.map(NamedList::size)
.map(Number::longValue)
.orElseThrow(() -> new RuntimeException(ACL_TX_IN_INDEX_NOT_IN_DB + " section not found in response.")));
assertEquals(
Long.valueOf(duplicatedTxInIndex.cardinality()),
of(subReport.get(DUPLICATED_ACL_TX_IN_INDEX))
.map(NamedList.class::cast)
.map(NamedList::size)
.map(Number::longValue)
.orElseThrow(() -> new RuntimeException(DUPLICATED_ACL_TX_IN_INDEX + " section not found in response.")));
assertEquals(
Long.valueOf(missingTxFromIndex.cardinality() - 5),
of(subReport.get(MISSING_ACL_TX_IN_INDEX))
.map(NamedList.class::cast)
.map(NamedList::size)
.map(Number::longValue)
.orElseThrow(() -> new RuntimeException(MISSING_ACL_TX_IN_INDEX + " section not found in response.")));
}
private <T> void assertThatExplicitParameterIsEchoed(String parameterName, T parameterValue)
{
admin = new AlfrescoCoreAdminHandler() {
@Override
NamedList<Object> fixOnSpecificCore(
String coreName,
Long fromTxCommitTime,
Long toTxCommitTime,
boolean dryRun,
int maxTransactionsToSchedule) {
return new NamedList<>(); // dummy entry
}
@Override
boolean isMasterOrStandalone(String coreName) {
return true;
}
};
admin.trackerRegistry = registry;
params.set(CORE, ALFRESCO_CORE_NAME);
params.set(parameterName, parameterValue.toString());
NamedList<Object> actionResponse = admin.actionFIX(params);
assertEquals(parameterValue, actionResponse.get(parameterName));
}
}
search-services/alfresco-search/src/test/java/org/alfresco/solr/AlfrescoSolrUtils.java
+15 -12
View File
@@ -135,18 +135,17 @@ public class AlfrescoSolrUtils
*/
public static Transaction getTransaction(int deletes, int updates)
{
long txnCommitTime = System.currentTimeMillis();
Transaction transaction = new Transaction();
transaction.setCommitTimeMs(txnCommitTime);
transaction.setId(generateId());
transaction.setDeletes(deletes);
transaction.setUpdates(updates);
return transaction;
return getTransaction(deletes, updates, generateId());
}
public static Transaction getTransaction(int deletes, int updates, long id)
{
long txnCommitTime = System.currentTimeMillis();
return getTransaction(deletes, updates, id, System.currentTimeMillis());
}
public static Transaction getTransaction(int deletes, int updates, long id, long timestamp)
{
long txnCommitTime = timestamp;
Transaction transaction = new Transaction();
transaction.setCommitTimeMs(txnCommitTime);
transaction.setId(id);
@@ -309,6 +308,7 @@ public class AlfrescoSolrUtils
Acl acl = new Acl(aclChangeSet.getId(), aclId);
return acl;
}
/**
* Get an AclChangeSet
* @param aclCount
@@ -316,14 +316,17 @@ public class AlfrescoSolrUtils
*/
public static AclChangeSet getAclChangeSet(int aclCount)
{
AclChangeSet aclChangeSet = new AclChangeSet(generateId(), System.currentTimeMillis(), aclCount);
return aclChangeSet;
return new AclChangeSet(generateId(), System.currentTimeMillis(), aclCount);
}
public static AclChangeSet getAclChangeSet(int aclCount, long id)
{
AclChangeSet aclChangeSet = new AclChangeSet(id, System.currentTimeMillis(), aclCount);
return aclChangeSet;
return new AclChangeSet(id, System.currentTimeMillis(), aclCount);
}
public static AclChangeSet getAclChangeSet(int aclCount, long id, long timestamp)
{
return new AclChangeSet(id, timestamp, aclCount);
}
private static AtomicLong id = new AtomicLong(System.currentTimeMillis());
search-services/alfresco-search/src/test/java/org/alfresco/solr/SolrITInitializer.java
+6 -6
View File
@@ -29,7 +29,6 @@ package org.alfresco.solr;
import static org.alfresco.solr.AlfrescoSolrUtils.createCoreUsingTemplate;
import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope;
import com.carrotsearch.randomizedtesting.annotations.ThreadLeakLingering;
import org.alfresco.solr.basics.RandomSupplier;
import org.alfresco.solr.client.SOLRAPIQueueClient;
import org.apache.commons.io.FileUtils;
@@ -98,7 +97,10 @@ import java.util.concurrent.atomic.AtomicInteger;
public abstract class SolrITInitializer extends SolrTestCaseJ4
{
private static final Logger LOGGER = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
protected static final int DEFAULT_CONNECTION_TIMEOUT1 = DEFAULT_CONNECTION_TIMEOUT;
protected static final int CLIENT_SO_TIMEOUT = 90000;
protected final static int INDEX_TIMEOUT = 100000;
private static AtomicInteger nodeCnt;
protected static boolean useExplicitNodeNames;
@@ -115,8 +117,6 @@ public abstract class SolrITInitializer extends SolrTestCaseJ4
//Standalone Tests
protected static SolrCore defaultCore;
protected static final int clientConnectionTimeout = DEFAULT_CONNECTION_TIMEOUT;
protected static final int clientSoTimeout = 90000;
protected static final String id = "id";
@@ -501,8 +501,8 @@ public abstract class SolrITInitializer extends SolrTestCaseJ4
try
{
HttpSolrClient client = new HttpSolrClient(url);
client.setConnectionTimeout(clientConnectionTimeout);
client.setSoTimeout(clientSoTimeout);
client.setConnectionTimeout(DEFAULT_CONNECTION_TIMEOUT1);
client.setSoTimeout(CLIENT_SO_TIMEOUT);
client.setDefaultMaxConnectionsPerHost(100);
client.setMaxTotalConnections(100);
return client;
search-services/alfresco-search/src/test/java/org/alfresco/solr/query/afts/requestHandler/AFTSDefaultTextQueryIT.java
+395 -18
View File
@@ -28,16 +28,19 @@ package org.alfresco.solr.query.afts.requestHandler;
import org.alfresco.service.cmr.repository.NodeRef;
import org.alfresco.service.namespace.QName;
import org.alfresco.solr.client.ContentPropertyValue;
import org.alfresco.solr.AlfrescoSolrDataModel;
import org.alfresco.solr.client.MLTextPropertyValue;
import org.alfresco.solr.client.PropertyValue;
import org.alfresco.solr.client.StringPropertyValue;
import org.alfresco.solr.dataload.TestDataProvider;
import org.apache.solr.SolrTestCaseJ4;
import org.junit.After;
import org.junit.BeforeClass;
import org.junit.Test;
import java.lang.reflect.Field;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Locale;
import java.util.Map;
@@ -64,10 +67,17 @@ import static com.google.common.collect.ImmutableMap.of;
@SolrTestCaseJ4.SuppressSSL
public class AFTSDefaultTextQueryIT extends AbstractRequestHandlerIT
{
// Following data is used for restoring alfrescoSolrDataModel after text execution
private HashSet<QName> oldCrossLocaleProperties;
private HashSet<QName> oldCrossLocaleTypes;
static TestDataProvider dataProvider;
@BeforeClass
public static void beforeClass() throws Exception
{
TestDataProvider dataProvider = new TestDataProvider(h);
dataProvider = new TestDataProvider(h);
List<Map<String, String>> data = asList(
of("name", "test1",
@@ -93,7 +103,6 @@ public class AFTSDefaultTextQueryIT extends AbstractRequestHandlerIT
TEST_ROOT_NODEREF = dataProvider.getRootNode();
range(0, data.size())
.forEach(dbId -> {
@@ -124,34 +133,402 @@ public class AFTSDefaultTextQueryIT extends AbstractRequestHandlerIT
});
}
private void prepareExactSearchData() throws Exception {
TestDataProvider dataProvider = AFTSDefaultTextQueryIT.dataProvider;
List<Map<String, String>> exactSearchData = asList(
of("name", "Running",
"description", "Running is a sport is a nice activity",
"content", "when you are running you are doing an amazing sport",
"title", "Running jumping",
"creator", "Alex"),
of("name", "Run",
"description", "you are supposed to run jump",
"content", "after many runs you are tired and if you jump it happens the same",
"title", "Run : a philosophy",
"creator", "Alex"),
of("name", "Poetry",
"description", "a document about poetry and jumpers",
"content", "poetry is unrelated to sport",
"title", "Running jumping twice jumpers",
"creator", "Alex"),
of("name", "Jump",
"description", "a document about jumps",
"content", "runnings jumpings",
"title", "Running",
"creator", "Alex"),
of("name", "Running jumping",
"description", "runners jumpers runs everywhere",
"content", "run is Good as jump",
"title", "Running the art of jumping",
"creator", "Alex"));
range(0, exactSearchData.size())
.forEach(dbId -> {
Map<String, String> record = exactSearchData.get(dbId);
String name = record.get("name");
String description = record.get("description");
String content = record.get("content");
String title = record.get("title");
String creator = record.get("creator");
Map<QName, PropertyValue> properties = new HashMap<>();
properties.put(PROP_NAME, new StringPropertyValue(name));
properties.put(PROP_DESCRIPTION, new MLTextPropertyValue(Map.of(Locale.getDefault(), description)));
properties.put(PROP_TITLE, new MLTextPropertyValue(Map.of(Locale.getDefault(), title)));
properties.put(PROP_CREATOR, new StringPropertyValue(creator));
properties.put(PROP_RATING_SCHEME, new StringPropertyValue(title));
Map<QName, String> contents = Map.of(PROP_CONTENT, content);
addNode(getCore(),
dataModel, 2, dbId+10, 1,
TYPE_CONTENT, null, properties, contents,
"the_owner_of_this_node_is" + name,
null,
new NodeRef[]{ TEST_ROOT_NODEREF },
new String[]{ "/" + dataProvider.qName("a_qname_for_node_" + name) },
dataProvider.newNodeRef(), true);
});
}
private void changeCrossLocaleEnabledFields() throws NoSuchFieldException, IllegalAccessException {
Field crossLocalePropertiesField = AlfrescoSolrDataModel
.getInstance()
.getClass()
.getDeclaredField("crossLocaleSearchProperties");
crossLocalePropertiesField.setAccessible(true);
Field crossLocaleTypesField = AlfrescoSolrDataModel
.getInstance()
.getClass()
.getDeclaredField("crossLocaleSearchDataTypes");
crossLocaleTypesField.setAccessible(true);
HashSet<QName> crossLocaleProperties = (HashSet<QName>) crossLocalePropertiesField.get(AlfrescoSolrDataModel.getInstance());
HashSet<QName> crossLocaleTypes = (HashSet<QName>) crossLocaleTypesField.get(AlfrescoSolrDataModel.getInstance());
oldCrossLocaleTypes = (HashSet<QName>) crossLocaleTypes.clone();
oldCrossLocaleProperties = (HashSet<QName>) crossLocaleProperties.clone();
crossLocaleTypes.clear();
crossLocaleProperties.clear();
crossLocaleProperties.add(PROP_TITLE);
}
@After
public void restoreAlfrescoSolrDataModel() throws NoSuchFieldException, IllegalAccessException {
if (oldCrossLocaleTypes != null) {
Field crossLocaleTypesField = AlfrescoSolrDataModel
.getInstance()
.getClass()
.getDeclaredField("crossLocaleSearchDataTypes");
crossLocaleTypesField.setAccessible(true);
crossLocaleTypesField.set(AlfrescoSolrDataModel.getInstance(), oldCrossLocaleTypes);
}
if (oldCrossLocaleProperties != null) {
Field crossLocalePropertiesField = AlfrescoSolrDataModel
.getInstance()
.getClass()
.getDeclaredField("crossLocaleSearchProperties");
crossLocalePropertiesField.setAccessible(true);
crossLocalePropertiesField.set(AlfrescoSolrDataModel.getInstance(), oldCrossLocaleProperties);
}
}
/**
* Test exact search is working.
*/
@Test
public void exactSearch_singleTerm_shouldReturnResultsContainingExactTerm() throws Exception {
prepareExactSearchData();
/*
* Out of the 5 'run corpus' documents
* 2 results are expected:
*
* - "name", "Run",
* "description", "you are supposed to run jump"
*
* - "name", "Running jumping",
* "content", "run is Good as jump",
*/
assertResponseCardinality("=run", 2);
/*
* No result for runner, one record has runners,
* you can see the difference between exact search and not
*/
assertResponseCardinality("=runner", 0);
assertResponseCardinality("runner", 1);
/*
* Out of the 5 'run corpus' documents
* 4 results are expected, only one doc is no fit for the query
*
* "name", "Run",
* "description", "you are supposed to run jump",
* "content", "after many runs you are tired and if you jump it happens the same",
* "title", "Run : a philosophy",
* "creator", "Alex"
*/
assertResponseCardinality("=running", 4);
}
@Test
public void exactSearch_singleTermInFieldWithNoCrossLocaleEnabled_shouldReturnException() throws Exception {
/*
* Cross Locale is Disabled for: content, cm_description, name
* Cross Locale is Enabled for : cm_title
*/
changeCrossLocaleEnabledFields();
prepareExactSearchData();
assertResponseCardinality("=cm_title:run", 1);
assertResponseException("=content:run", "Exact Term search is not supported unless you configure the field <{http://www.alfresco.org/model/content/1.0}content> for cross locale search");
assertResponseException("=cm_description:run", "Exact Term search is not supported unless you configure the field <{http://www.alfresco.org/model/content/1.0}description> for cross locale search");
assertResponseCardinality("=run", 1);
restoreAlfrescoSolrDataModel();
}
@Test
public void exactSearch_singleTermInFieldWithOnlyUnTokenizedAnalysis_shouldReturnFullFieldValueMatch() throws Exception {
/**
* cm_ratingScheme is a copy field un-tokenized of Title, so it has the exact same content but not analysed.
* This means we produce just a token in the index, exactly as the full content.
* We can't expect any search to work except full exact value search
*/
prepareExactSearchData();
/*
* Out of the 5 'run corpus' documents
* 0 results are expected:
* there is no result that have cm_ratingScheme:"running"
*
*/
assertResponseCardinality("=cm_ratingScheme:running", 0);
assertResponseCardinality("=cm_title:running", 4);
/*
* Out of the 5 'run corpus' documents
* 1 result is expected:
*
* - "name", "Jump",
* ...
* "title", "Running"
*
*/
assertResponseCardinality("=cm_ratingScheme:Running", 1);
assertResponseCardinality("=cm_title:Running", 4);
/*
* Out of the 5 'run corpus' documents
* 0 results are expected:
* there is no result that have exactly cm_ratingScheme:"Run"
* The closest we have is record Run (cm_ratingScheme:"Run : a philosophy")
* As you can see we don't have a full match, so it's not in the results.
*
*/
assertResponseCardinality("=cm_ratingScheme:Run", 0);
assertResponseCardinality("=cm_title:Run", 1);
}
/**
* Test exact search is working.
*/
@Test
public void defaultExactQueryTest()
{
public void exactSearch_multiTerm_shouldReturnResultsContainingExactTerm() throws Exception {
prepareExactSearchData();
/*
* Out of the 5 'run corpus' documents
* 3 results are expected:
*
* - "name", "Run",
* "description", "you are supposed to run jump",
*
* - "name", "Running jumping",
* "description", "runners jumpers runs everywhere",
* "content", "run is Good as jump",
*
* - "name", "Jump",
* "description", "a document about jumps",
*
*/
assertResponseCardinality("=run =jump", 3);
/*
* 3 results expected:
* record 1 ("test" in name, description and content)
* record 2 ("test" in name, and content)
* record 3 ("test" in description)
* No result for runner or jumper, one record has runners,
* and another record has jumpers
*
* - "name", "Poetry",
* "description", "a document about poetry and jumpers",
* - "name", "Running jumping",
* "description", "runners jumpers run everywhere",
*
* you can see the difference between exact search and not
*/
assertResponseCardinality("test", 3);
assertResponseCardinality("=runner =jumper", 0);
assertResponseCardinality("runner jumper", 2);
/*
* 3 results expected:
* record 2 ("Other" in title )
* record 3 ("other" in content)
* record 4 ("other" in description)
* Out of the 5 'run corpus' documents
* 4 results are expected:
* Only one doc does't fit:
* - "name", "Run",
* "description", "you are supposed to run jump",
* "content", "after many runs you are tired and if you jump it happens the same",
* "title", "Run : a philosophy",
* "creator", "Alex"),
*
*
*/
assertResponseCardinality("Other", 3);
assertResponseCardinality("=running =jumping", 4);
}
@Test
public void exactSearch_multiTermInFieldWithNoCrossLocaleEnabled_shouldReturnException() throws Exception {
/*
* Cross Locale is Disabled for: content, cm_description, name
* Cross Locale is Enabled for : cm_title
*/
changeCrossLocaleEnabledFields();
prepareExactSearchData();
assertResponseCardinality("=cm_title:run =cm_title:jump", 1);
assertResponseException("=content:run =content:jump", "Exact Term search is not supported unless you configure the field <{http://www.alfresco.org/model/content/1.0}content> for cross locale search");
assertResponseException("=cm_description:run =cm_description:jump", "Exact Term search is not supported unless you configure the field <{http://www.alfresco.org/model/content/1.0}description> for cross locale search");
assertResponseCardinality("=run =jump", 1);
restoreAlfrescoSolrDataModel();
}
@Test
public void exactSearch_multiTermInFieldWithOnlyUnTokenizedAnalysis_shouldReturnFullFieldValueMatch() throws Exception {
/**
* cm_ratingScheme is a copy field un-tokenized of Title, so it has the exact same content but not analysed.
* This means we produce just a token in the index, exactly as the full content.
* We can't expect any search to work except full exact value search
*/
prepareExactSearchData();
/*
* Out of the 5 'run corpus' documents
* 0 results are expected:
* there is no result that have cm_ratingScheme:"running" or "jumpers"
*
*/
assertResponseCardinality("=cm_ratingScheme:running =cm_ratingScheme:jumpers", 0);
assertResponseCardinality("=cm_title:running =cm_title:jumpers", 4);
}
/**
* Test exact search is working.
*/
@Test
public void exactSearch_exactPhrase_shouldReturnResultsContainingExactPhrase() throws Exception {
prepareExactSearchData();
/*
* Out of the 5 'run corpus' documents
* 1 results are expected:
*
* - "name", "Run",
* "description", "you are supposed to run jump",
*
*/
assertResponseCardinality("=\"run jump\"", 1);
/*
* No results expected because creator should not be considered in default text search.
* No result for runner jumper, one record has runners jumpers,
* you can see the difference between exact search and not
*
* "name", "Running jumping",
* "description", "runners jumpers run everywhere",
*/
assertResponseCardinality("Giovanni", 0);
assertResponseCardinality("=\"runner jumper\"", 0);
assertResponseCardinality("\"runner jumper\"", 1);
/*
* Out of the 5 'run corpus' documents
* 3 results are expected:
*
* - "name", "Running",
* ...
* "title", "Running jumping",
*
* - "name", "Poetry",
* "title", "Running jumping twice jumpers"
*
* - "name", "Running jumping",
*/
assertResponseCardinality("=\"running jumping\"", 3);
assertResponseCardinality("\"running jumping\"", 5);
}
@Test
public void exactSearch_phraseInFieldWithNoCrossLocaleEnabled_shouldReturnException() throws Exception {
/*
* Cross Locale is Disabled for: content, cm_description, name
* Cross Locale is Enabled for : cm_title
*/
changeCrossLocaleEnabledFields();
prepareExactSearchData();
assertResponseCardinality("=cm_title:\"running jumping\"", 2);
assertResponseException("=content:\"running jumping\"", "Exact Term search is not supported unless you configure the field <{http://www.alfresco.org/model/content/1.0}content> for cross locale search");
assertResponseException("=cm_description:\"running jumping\"", "Exact Term search is not supported unless you configure the field <{http://www.alfresco.org/model/content/1.0}description> for cross locale search");
assertResponseCardinality("=\"running jumping\"", 2);
restoreAlfrescoSolrDataModel();
}
@Test
public void exactSearch_phraseInFieldWithOnlyUnTokenizedAnalysis_shouldReturnFullFieldValueMatch() throws Exception {
/**
* cm_ratingScheme is a copy field un-tokenized of Title, so it has the exact same content but not analysed.
* This means we produce just a token in the index, exactly as the full content.
* We can't expect any search to work except full exact value search
*/
prepareExactSearchData();
/*
* Out of the 5 'run corpus' documents
* 0 results are expected:
* the closest we got was this one, but it is uppercase
* - "name", "Running",
* "title", "Running jumping",
*
*/
assertResponseCardinality("=cm_ratingScheme:\"running jumping\"", 0);
assertResponseCardinality("=cm_title:\"running jumping\"", 2);
/*
* Out of the 5 'run corpus' documents
* 1 results are expected:
* - "name", "Running",
* "title", "Running jumping",
*
*/
assertResponseCardinality("=cm_ratingScheme:\"Running jumping\"", 1);
assertResponseCardinality("=cm_title:\"Running jumping\"", 2);
/*
* Out of the 5 'run corpus' documents
* 0 results are expected:
* the closest we got was this one, but it is uppercase
* - "name", "Poetry",
* "title", "Running jumping twice jumpers",
*
*/
assertResponseCardinality("=cm_ratingScheme:\"Running jumping twice\"", 0);
assertResponseCardinality("=cm_title:\"Running jumping twice\"", 1);
}
@@ -175,7 +552,7 @@ public class AFTSDefaultTextQueryIT extends AbstractRequestHandlerIT
* record 3 ("other" in content)
* record 4 ("other" in description)
*/
assertResponseCardinality("other to otherz ", 3);
assertResponseCardinality("[other to otherz]", 3);
}
search-services/alfresco-search/src/test/java/org/alfresco/solr/query/afts/requestHandler/AbstractRequestHandlerIT.java
+13
View File
@@ -37,6 +37,9 @@ import java.util.stream.Stream;
import static java.util.Arrays.asList;
import static java.util.stream.IntStream.range;
import static org.hamcrest.CoreMatchers.is;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.junit.Assert.fail;
public abstract class AbstractRequestHandlerIT extends AbstractAlfrescoSolrIT
{
@@ -52,6 +55,16 @@ public abstract class AbstractRequestHandlerIT extends AbstractAlfrescoSolrIT
assertQ(areq(params("rows", "20", "qt", "/afts", "fq", "{!afts}AUTHORITY_FILTER_FROM_JSON", "q", query), json), "*[count(//doc)="+num+"]");
}
void assertResponseException(String query, String exceptionMessage) {
try {
h.query(areq(params("rows", "20", "qt", "/afts", "q", query), null));
fail("No Exception Thrown");
} catch (Exception e) {
assertThat(e.getMessage(), is(exceptionMessage));
}
}
void assertResponseCardinality(String query, int num)
{
assertQ(areq(params("rows", "20", "qt", "/afts", "q", query), null), "*[count(//doc)="+num+"]");
search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedAlfrescoSolrTrackerRaceIT.java
+33 -23
View File
@@ -66,7 +66,7 @@ public class DistributedAlfrescoSolrTrackerRaceIT extends AbstractAlfrescoDistri
@BeforeClass
public static void initData() throws Throwable
{
initSolrServers(2, "DistributedAlfrescoSolrTrackerRaceTest", null);
initSolrServers(2, "DistributedAlfrescoSolrTrackerRaceIT", null);
}
@AfterClass
@@ -76,25 +76,23 @@ public class DistributedAlfrescoSolrTrackerRaceIT extends AbstractAlfrescoDistri
}
@Test
public void testTracker() throws Exception
{
public void testTracker() throws Exception {
putHandleDefaults();
AclChangeSet aclChangeSet = getAclChangeSet(1);
AclChangeSet aclChangeSet = getAclChangeSet(2, 1, System.currentTimeMillis() -
AbstractTracker.TIME_STEP_32_DAYS_IN_MS);
Acl acl = getAcl(aclChangeSet);
Acl acl2 = getAcl(aclChangeSet);
AclReaders aclReaders = getAclReaders(aclChangeSet, acl, singletonList("joel"), singletonList("phil"), null);
AclReaders aclReaders2 = getAclReaders(aclChangeSet, acl2, singletonList("jim"), singletonList("phil"), null);
AclReaders aclReaders =
getAclReaders(aclChangeSet, acl, singletonList("joel"), singletonList("phil"), null);
AclReaders aclReaders2 =
getAclReaders(aclChangeSet, acl2, singletonList("jim"), singletonList("phil"), null);
// Transaction between [1-2000] is required, when greater value checking the core will fail
Transaction txn = getTransaction(0, 2, 1);
long txnCommitTimeMs = txn.getCommitTimeMs();
// Subtract from the commit time to go beyond hole retention
long backdatedCommitTimeMs = txnCommitTimeMs - 4600000;
txn.setCommitTimeMs(backdatedCommitTimeMs);
Transaction txn = getTransaction(0, 2, 1,
System.currentTimeMillis() - AbstractTracker.TIME_STEP_32_DAYS_IN_MS);
//Next create two nodes to update for the transaction
Node folderNode = getNode(txn, acl, Node.SolrApiNodeStatus.UPDATED);
@@ -103,9 +101,12 @@ public class DistributedAlfrescoSolrTrackerRaceIT extends AbstractAlfrescoDistri
// Next, create the node metadata for each node.
// Note: the error node metadata will cause an exception.
NodeMetaData folderMetaData = getNodeMetaData(folderNode, txn, acl, "mike", null, false);
NodeMetaData fileMetaData = getNodeMetaData(fileNode, txn, acl, "mike", ancestors(folderMetaData.getNodeRef()), false);
NodeMetaData errorMetaData = getNodeMetaData(errorNode, txn, acl, "lisa", ancestors(folderMetaData.getNodeRef()), true);
NodeMetaData folderMetaData =
getNodeMetaData(folderNode, txn, acl, "mike", null, false);
NodeMetaData fileMetaData = getNodeMetaData(fileNode, txn, acl, "mike",
ancestors(folderMetaData.getNodeRef()), false);
NodeMetaData errorMetaData = getNodeMetaData(errorNode, txn, acl, "lisa",
ancestors(folderMetaData.getNodeRef()), true);
// Index the transaction, nodes, and nodeMetaDatas.
// Note that the content is automatically created by the test framework.
@@ -113,25 +114,34 @@ public class DistributedAlfrescoSolrTrackerRaceIT extends AbstractAlfrescoDistri
indexAclChangeSet(aclChangeSet, asList(acl, acl2), asList(aclReaders, aclReaders2));
BooleanQuery.Builder builder = new BooleanQuery.Builder();
builder.add(new BooleanClause(new TermQuery(new Term(QueryConstants.FIELD_SOLR4_ID, "TRACKER!STATE!ACLTX")), BooleanClause.Occur.MUST));
builder.add(new BooleanClause(LegacyNumericRangeQuery.newLongRange(QueryConstants.FIELD_S_ACLTXID, aclChangeSet.getId(), aclChangeSet.getId() + 1, true, false), BooleanClause.Occur.MUST));
builder.add(new BooleanClause(new TermQuery(new Term(QueryConstants.FIELD_SOLR4_ID, "TRACKER!STATE!ACLTX")),
BooleanClause.Occur.MUST));
builder.add(new BooleanClause(LegacyNumericRangeQuery.newLongRange(QueryConstants.FIELD_S_ACLTXID,
aclChangeSet.getId(), aclChangeSet.getId() + 1, true, false),
BooleanClause.Occur.MUST));
BooleanQuery waitForQuery = builder.build();
waitForDocCountAllCores(waitForQuery, 1, 80000);
waitForDocCountAllCores(waitForQuery, 1, INDEX_TIMEOUT);
// This ACL should have one record in each core with DBID sharding
waitForDocCountAllCores(new TermQuery(new Term(QueryConstants.FIELD_READER, "jim")), 1, 80000);
waitForDocCountAllCores(new TermQuery(new Term(QueryConstants.FIELD_READER, "jim")), 1, INDEX_TIMEOUT);
// We should have 2 document in totals (1 folder and 1 file)
waitForDocCount(new TermQuery(new Term("content@s___t@{http://www.alfresco.org/model/content/1.0}content", "world")), 2, 100000);
waitForDocCount(new TermQuery(new Term("content@s___t@{http://www.alfresco.org/model/content/1.0}content",
"world")), 2, INDEX_TIMEOUT);
// There should be 1 with the folder node identifier.
waitForDocCount(new TermQuery(new Term("content@s___t@{http://www.alfresco.org/model/content/1.0}content", Long.toString(folderNode.getId()))), 1, 80000);
waitForDocCount(new TermQuery(new Term("content@s___t@{http://www.alfresco.org/model/content/1.0}content",
Long.toString(folderNode.getId()))), 1, INDEX_TIMEOUT);
// There should be 1 with the file node identifier.
waitForDocCount(new TermQuery(new Term("content@s___t@{http://www.alfresco.org/model/content/1.0}content", Long.toString(fileNode.getId()))), 1, 80000);
waitForDocCount(new TermQuery(new Term("content@s___t@{http://www.alfresco.org/model/content/1.0}content",
Long.toString(fileNode.getId()))), 1, INDEX_TIMEOUT);
// and last but not least, the error node shouldn't be in the index.
waitForDocCount(new TermQuery(new Term("content@s___t@{http://www.alfresco.org/model/content/1.0}content", Long.toString(errorNode.getId()))), 0, 80000);
waitForDocCount(new TermQuery(new Term("content@s___t@{http://www.alfresco.org/model/content/1.0}content",
Long.toString(errorNode.getId()))), 0, INDEX_TIMEOUT);
// This will run the same query on the control client and the cluster and compare the result.
query(getDefaultTestClient(),
search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedAlfrescoTrackerWithDelayedTransactionsIT.java
+169
View File
@@ -0,0 +1,169 @@
/*
* #%L
* Alfresco Search Services
* %%
* Copyright (C) 2005 - 2020 Alfresco Software Limited
* %%
* This file is part of the Alfresco software.
* If the software was purchased under a paid Alfresco license, the terms of
* the paid license agreement will prevail. Otherwise, the software is
* provided under the following open source license terms:
*
* Alfresco is free software: you can redistribute it and/or modify
* it under the terms of the GNU Lesser General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Alfresco is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with Alfresco. If not, see <http://www.gnu.org/licenses/>.
* #L%
*/
package org.alfresco.solr.tracker;
import org.alfresco.repo.search.adaptor.lucene.QueryConstants;
import org.alfresco.solr.AbstractAlfrescoDistributedIT;
import org.alfresco.solr.client.Acl;
import org.alfresco.solr.client.AclChangeSet;
import org.alfresco.solr.client.AclReaders;
import org.alfresco.solr.client.Node;
import org.alfresco.solr.client.NodeMetaData;
import org.alfresco.solr.client.Transaction;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.LegacyNumericRangeQuery;
import org.apache.lucene.search.TermQuery;
import org.apache.solr.SolrTestCaseJ4;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
import java.util.List;
import static java.util.Arrays.asList;
import static java.util.Collections.singletonList;
import static org.alfresco.solr.AlfrescoSolrUtils.ancestors;
import static org.alfresco.solr.AlfrescoSolrUtils.getAcl;
import static org.alfresco.solr.AlfrescoSolrUtils.getAclChangeSet;
import static org.alfresco.solr.AlfrescoSolrUtils.getAclReaders;
import static org.alfresco.solr.AlfrescoSolrUtils.getNode;
import static org.alfresco.solr.AlfrescoSolrUtils.getNodeMetaData;
import static org.alfresco.solr.AlfrescoSolrUtils.getTransaction;
import static org.alfresco.solr.AlfrescoSolrUtils.indexAclChangeSet;
@SolrTestCaseJ4.SuppressSSL
public class DistributedAlfrescoTrackerWithDelayedTransactionsIT extends AbstractAlfrescoDistributedIT {
private static NodeMetaData folderMetaData;
private static AclChangeSet aclChangeSet;
@BeforeClass
public static void initData() throws Throwable
{
initializeDataBeforeServerCreation();
initSolrServers(2, "DistributedAlfrescoTrackerWithDelayedTransactionsIT",
null);
}
@AfterClass
public static void destroyData()
{
dismissSolrServers();
}
private static void initializeDataBeforeServerCreation()
{
putHandleDefaults();
aclChangeSet = getAclChangeSet(2, 1,
System.currentTimeMillis() - AbstractTracker.TIME_STEP_32_DAYS_IN_MS);
Acl acl = getAcl(aclChangeSet);
Acl acl2 = getAcl(aclChangeSet);
AclReaders aclReaders =
getAclReaders(aclChangeSet, acl, singletonList("joel"), singletonList("phil"), null);
AclReaders aclReaders2 =
getAclReaders(aclChangeSet, acl2, singletonList("jim"), singletonList("phil"), null);
// Transaction between [1-2000] is required, when greater value checking the core will fail
Transaction txn = getTransaction(0, 2, 1,
System.currentTimeMillis() - AbstractTracker.TIME_STEP_32_DAYS_IN_MS);
//Next create two nodes to update for the transaction
Node folderNode = getNode(txn, acl, Node.SolrApiNodeStatus.UPDATED);
Node fileNode = getNode(txn, acl, Node.SolrApiNodeStatus.UPDATED);
folderMetaData = getNodeMetaData(folderNode, txn, acl, "mike", null, false);
NodeMetaData fileMetaData = getNodeMetaData(fileNode, txn, acl, "mike",
ancestors(folderMetaData.getNodeRef()), false);
indexAclChangeSet(aclChangeSet, asList(acl, acl2), asList(aclReaders, aclReaders2));
indexTransaction(txn, List.of(folderNode, fileNode), List.of(folderMetaData, fileMetaData));
}
@Test
public void testTracker() throws Exception {
BooleanQuery.Builder builder = new BooleanQuery.Builder();
builder.add(new BooleanClause(new TermQuery(new Term(QueryConstants.FIELD_SOLR4_ID, "TRACKER!STATE!ACLTX")),
BooleanClause.Occur.MUST));
builder.add(new BooleanClause(LegacyNumericRangeQuery.newLongRange(QueryConstants.FIELD_S_ACLTXID,
aclChangeSet.getId(), aclChangeSet.getId() + 1, true, false),
BooleanClause.Occur.MUST));
BooleanQuery waitForQuery = builder.build();
waitForDocCountAllCores(waitForQuery, 1, INDEX_TIMEOUT);
// This ACL should have one record in each core with DBID sharding
waitForDocCountAllCores(new TermQuery(new Term(QueryConstants.FIELD_READER, "jim")),
1, INDEX_TIMEOUT);
// We should have 2 document in totals (1 folder and 1 file)
waitForDocCount(new TermQuery(new Term("content@s___t@{http://www.alfresco.org/model/content/1.0}content",
"world")), 2, INDEX_TIMEOUT);
// This will run the same query on the control client and the cluster and compare the result.
query(getDefaultTestClient(),
true,
"{\"locales\":[\"en\"], \"templates\": [{\"name\":\"t1\", \"template\":\"%cm:content\"}]}",
params("q", "t1:world", "qt", "/afts", "shards.qt", "/afts", "start", "0", "rows", "6", "sort", "id asc"));
// Index new ACL after 32 days
AclChangeSet lateAclChangeSet = getAclChangeSet(1, 2);
Acl lateAcl = getAcl(lateAclChangeSet);
AclReaders lateAclReaders =
getAclReaders(lateAclChangeSet, lateAcl, singletonList("elia"), singletonList("phil"), null);
indexAclChangeSet(lateAclChangeSet, asList(lateAcl), asList(lateAclReaders));
// Index new transaction after 32 days
Transaction lateTransaction = getTransaction(0, 1);
Node lateNode = getNode(lateTransaction, lateAcl, Node.SolrApiNodeStatus.UPDATED);
NodeMetaData lateNodeMetaData = getNodeMetaData(lateNode, lateTransaction, lateAcl, "elia",
ancestors(folderMetaData.getNodeRef()), false);
// Check new Acl has been indexed
builder = new BooleanQuery.Builder();
builder.add(new BooleanClause(new TermQuery(new Term(QueryConstants.FIELD_SOLR4_ID, "TRACKER!STATE!ACLTX")),
BooleanClause.Occur.MUST));
builder.add(new BooleanClause(LegacyNumericRangeQuery.newLongRange(QueryConstants.FIELD_S_ACLTXID,
lateAclChangeSet.getId(), lateAclChangeSet.getId() + 1, true, false),
BooleanClause.Occur.MUST));
waitForQuery = builder.build();
waitForDocCountAllCores(waitForQuery, 1, INDEX_TIMEOUT);
// Check the new node has been indexed
indexTransaction(lateTransaction, List.of(lateNode), List.of(lateNodeMetaData));
waitForDocCount(new TermQuery(new Term("content@s___t@{http://www.alfresco.org/model/content/1.0}content",
"world")), 3, INDEX_TIMEOUT);
}
}
search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedCascadeTrackerIT.java
+2 -4
View File
@@ -77,8 +77,6 @@ public class DistributedCascadeTrackerIT extends AbstractAlfrescoDistributedIT
private final String pathChild1 = "pathChild2";
private final int timeout = 100000;
@Before
public void initData() throws Throwable
{
@@ -139,7 +137,7 @@ public class DistributedCascadeTrackerIT extends AbstractAlfrescoDistributedIT
*/
indexParentFolderWithCascade();
waitForDocCount(params("qt", "/afts", "q", "PATH:" + cascadingFirstChild), 1, timeout);
waitForDocCount(params("qt", "/afts", "q", "PATH:" + cascadingFirstChild), 1, INDEX_TIMEOUT);
// Check if the path is updated for both the nodes
assertShardCount(0, params("qt", "/afts", "q", "PATH:" + cascadingFirstChild), 1);
@@ -184,7 +182,7 @@ public class DistributedCascadeTrackerIT extends AbstractAlfrescoDistributedIT
/*
* Get sure the nodes are indexed correctly in the shards
*/
waitForDocCount(new TermQuery(new Term("content@s___t@{http://www.alfresco.org/model/content/1.0}content", "world")), 3, timeout);
waitForDocCount(new TermQuery(new Term("content@s___t@{http://www.alfresco.org/model/content/1.0}content", "world")), 3, INDEX_TIMEOUT);
assertShardCount(0, new TermQuery(new Term("content@s___t@{http://www.alfresco.org/model/content/1.0}content", "world")), 2);
assertShardCount(1, new TermQuery(new Term("content@s___t@{http://www.alfresco.org/model/content/1.0}content", "world")), 1);
}