From 3debea0e11aa8ac5481b1f3d00c0c6ed7f2b98f9 Mon Sep 17 00:00:00 2001 From: Angel Borroy Date: Tue, 2 Jun 2020 13:39:24 +0200 Subject: [PATCH 1/5] SEARCH-1915: Fix 6.2.x latest versions for Enterprise and Community --- .../generators/app/templates/6.2/.env | 18 ++-- .../app/templates/6.2/docker-compose-ce.yml | 12 +-- .../app/templates/6.2/docker-compose-ee.yml | 85 +++---------------- 3 files changed, 27 insertions(+), 88 deletions(-) diff --git a/e2e-test/generator-alfresco-docker-compose/generators/app/templates/6.2/.env b/e2e-test/generator-alfresco-docker-compose/generators/app/templates/6.2/.env index 352f5b342..af5ab5ebb 100755 --- a/e2e-test/generator-alfresco-docker-compose/generators/app/templates/6.2/.env +++ b/e2e-test/generator-alfresco-docker-compose/generators/app/templates/6.2/.env @@ -1,15 +1,13 @@ -ALFRESCO_TAG=latest -SHARE_TAG=6.2.0 +ALFRESCO_TAG=6.2.1 +ALFRESCO_CE_TAG=6.2.0-ga +SHARE_TAG=6.2.1 POSTGRES_TAG=11.4 -TRANSFORM_ROUTER_TAG=1.1.0 -PDF_RENDERER_TAG=2.1.0 -IMAGE_MAGICK_TAG=2.1.0 -LIBREOFFICE_TAG=2.1.0 -TIKA_TAG=2.1.0 -TRANSFORM_MISC_TAG=2.1.0 -SHARED_FILE_STORE_TAG=0.5.3 +TRANSFORM_ROUTER_TAG=1.2.0 +TRANSFORM_CORE_AIO_TAG=2.2.1 +TRANSFORMER_TAG=2.1.0 +SHARED_FILE_STORE_TAG=0.7.0 ACTIVE_MQ_TAG=5.15.8 -DIGITAL_WORKSPACE_TAG=1.3.0 +DIGITAL_WORKSPACE_TAG=1.5.0 ACS_NGINX_TAG=3.0.1 ACS_COMMUNITY_NGINX_TAG=1.0.0 SEARCH_TAG=latest diff --git a/e2e-test/generator-alfresco-docker-compose/generators/app/templates/6.2/docker-compose-ce.yml b/e2e-test/generator-alfresco-docker-compose/generators/app/templates/6.2/docker-compose-ce.yml index c6f466991..0773340eb 100755 --- a/e2e-test/generator-alfresco-docker-compose/generators/app/templates/6.2/docker-compose-ce.yml +++ b/e2e-test/generator-alfresco-docker-compose/generators/app/templates/6.2/docker-compose-ce.yml @@ -7,7 +7,7 @@ services: build: context: ./alfresco args: - ALFRESCO_TAG: ${ALFRESCO_TAG} + ALFRESCO_TAG: ${ALFRESCO_CE_TAG} SOLR_COMMS: <%=secureComms%> <% if (httpMode == 'https') { %> TRUSTSTORE_TYPE: JCEKS TRUSTSTORE_PASS: kT9X6oe68t @@ -141,7 +141,7 @@ services: <% } %> alfresco-pdf-renderer: - image: alfresco/alfresco-pdf-renderer:${PDF_RENDERER_TAG} + image: alfresco/alfresco-pdf-renderer:${TRANSFORMER_TAG} mem_limit: 1g environment: JAVA_OPTS: " -Xms256m -Xmx512m" @@ -149,7 +149,7 @@ services: - 8090:8090 imagemagick: - image: alfresco/alfresco-imagemagick:${IMAGE_MAGICK_TAG} + image: alfresco/alfresco-imagemagick:${TRANSFORMER_TAG} mem_limit: 1g environment: JAVA_OPTS: " -Xms256m -Xmx512m" @@ -157,7 +157,7 @@ services: - 8091:8090 libreoffice: - image: alfresco/alfresco-libreoffice:${LIBREOFFICE_TAG} + image: alfresco/alfresco-libreoffice:${TRANSFORMER_TAG} mem_limit: 1g environment: JAVA_OPTS: " -Xms256m -Xmx512m" @@ -165,7 +165,7 @@ services: - 8092:8090 tika: - image: alfresco/alfresco-tika:${TIKA_TAG} + image: alfresco/alfresco-tika:${TRANSFORMER_TAG} mem_limit: 1g environment: JAVA_OPTS: " -Xms256m -Xmx512m" @@ -173,7 +173,7 @@ services: - 8093:8090 transform-misc: - image: alfresco/alfresco-transform-misc:${TRANSFORM_MISC_TAG} + image: alfresco/alfresco-transform-misc:${TRANSFORMER_TAG} mem_limit: 1g environment: JAVA_OPTS: " -Xms256m -Xmx512m" diff --git a/e2e-test/generator-alfresco-docker-compose/generators/app/templates/6.2/docker-compose-ee.yml b/e2e-test/generator-alfresco-docker-compose/generators/app/templates/6.2/docker-compose-ee.yml index 964f94de1..1efb7dd31 100755 --- a/e2e-test/generator-alfresco-docker-compose/generators/app/templates/6.2/docker-compose-ee.yml +++ b/e2e-test/generator-alfresco-docker-compose/generators/app/templates/6.2/docker-compose-ee.yml @@ -34,20 +34,12 @@ services: -Dtransform.service.enabled=true -Dtransform.service.url=http://transform-router:8095 -Dsfs.url=http://shared-file-store:8099/ - - -Dlocal.transform.service.enabled=true - -DlocalTransform.pdfrenderer.url=http://alfresco-pdf-renderer:8090/ - -DlocalTransform.imagemagick.url=http://imagemagick:8090/ - -DlocalTransform.libreoffice.url=http://libreoffice:8090/ - -DlocalTransform.tika.url=http://tika:8090/ - -DlocalTransform.misc.url=http://transform-misc:8090/ - - -Dlegacy.transform.service.enabled=true - -Dalfresco-pdf-renderer.url=http://alfresco-pdf-renderer:8090/ - -Djodconverter.url=http://libreoffice:8090/ - -Dimg.url=http://imagemagick:8090/ - -Dtika.url=http://tika:8090/ - -Dtransform.misc.url=http://transform-misc:8090/ + -DlocalTransform.core-aio.url=http://transform-core-aio:8090/ + -Dalfresco-pdf-renderer.url=http://transform-core-aio:8090/ + -Djodconverter.url=http://transform-core-aio:8090/ + -Dimg.url=http://transform-core-aio:8090/ + -Dtika.url=http://transform-core-aio:8090/ + -Dtransform.misc.url=http://transform-core-aio:8090/ -Dcsrf.filter.enabled=false -Dalfresco.restApi.basicAuthScheme=true @@ -207,75 +199,24 @@ services: environment: JAVA_OPTS: " -Xms256m -Xmx512m" ACTIVEMQ_URL: "nio://activemq:61616" - IMAGEMAGICK_URL: "http://imagemagick:8090" - PDF_RENDERER_URL : "http://alfresco-pdf-renderer:8090" - LIBREOFFICE_URL : "http://libreoffice:8090" - TIKA_URL : "http://tika:8090" + CORE_AIO_URL : "http://transform-core-aio:8090" FILE_STORE_URL: "http://shared-file-store:8099/alfresco/api/-default-/private/sfs/versions/1/file" ports: - 8095:8095 links: - activemq - alfresco-pdf-renderer: - image: alfresco/alfresco-pdf-renderer:${PDF_RENDERER_TAG} - mem_limit: 1g + transform-core-aio: + image: alfresco/alfresco-transform-core-aio:${TRANSFORM_CORE_AIO_TAG} + mem_limit: 1536m environment: - JAVA_OPTS: " -Xms256m -Xmx512m" + JAVA_OPTS: " -Xms256m -Xmx1536m" ACTIVEMQ_URL: "nio://activemq:61616" FILE_STORE_URL: "http://shared-file-store:8099/alfresco/api/-default-/private/sfs/versions/1/file" ports: - 8090:8090 links: - - activemq - - imagemagick: - image: alfresco/alfresco-imagemagick:${IMAGE_MAGICK_TAG} - mem_limit: 1g - environment: - JAVA_OPTS: " -Xms256m -Xmx512m" - ACTIVEMQ_URL: "nio://activemq:61616" - FILE_STORE_URL: "http://shared-file-store:8099/alfresco/api/-default-/private/sfs/versions/1/file" - ports: - - 8091:8090 - links: - - activemq - - libreoffice: - image: alfresco/alfresco-libreoffice:${LIBREOFFICE_TAG} - mem_limit: 1g - environment: - JAVA_OPTS: " -Xms256m -Xmx512m" - ACTIVEMQ_URL: "nio://activemq:61616" - FILE_STORE_URL: "http://shared-file-store:8099/alfresco/api/-default-/private/sfs/versions/1/file" - ports: - - 8092:8090 - links: - - activemq - - tika: - image: alfresco/alfresco-tika:${TIKA_TAG} - mem_limit: 1g - environment: - JAVA_OPTS: " -Xms256m -Xmx512m" - ACTIVEMQ_URL: "nio://activemq:61616" - FILE_STORE_URL: "http://shared-file-store:8099/alfresco/api/-default-/private/sfs/versions/1/file" - ports: - - 8093:8090 - links: - - activemq - - transform-misc: - image: alfresco/alfresco-transform-misc:${TRANSFORM_MISC_TAG} - mem_limit: 1g - environment: - JAVA_OPTS: " -Xms256m -Xmx512m" - ACTIVEMQ_URL: "nio://activemq:61616" - FILE_STORE_URL: "http://shared-file-store:8099/alfresco/api/-default-/private/sfs/versions/1/file" - ports: - - 8094:8090 - links: - - activemq + - activemq shared-file-store: image: alfresco/alfresco-shared-file-store:${SHARED_FILE_STORE_TAG} @@ -320,7 +261,7 @@ services: activemq: image: alfresco/alfresco-activemq:${ACTIVE_MQ_TAG} - mem_limit: 512m + mem_limit: 1g ports: - 8161:8161 # Web Console - 5672:5672 # AMQP From 18a7d5af3b5679b7bf8ad90b566e1d1979df9281 Mon Sep 17 00:00:00 2001 From: Angel Borroy Date: Tue, 2 Jun 2020 16:38:27 +0200 Subject: [PATCH 2/5] SEARCH-1915: Support for ACS 7.x (named temporary as "latest") Created the new env var password specification for cryptographic stores when using mTLS in latest. Added new cryptographic stores set according to ACS 7.x criteria --- .../README.md | 22 +- .../generators/app/index.js | 23 +- .../app/templates/images/search/Dockerfile | 29 +- .../keystores/{ => 6.x}/alfresco/keystore | Bin .../alfresco/keystore-passwords.properties | 0 .../ssl-keystore-passwords.properties | 0 .../ssl-truststore-passwords.properties | 0 .../keystores/{ => 6.x}/alfresco/ssl.keystore | Bin .../{ => 6.x}/alfresco/ssl.truststore | Bin .../keystores/{ => 6.x}/client/browser.p12 | Bin .../solr/ssl-keystore-passwords.properties | 0 .../solr/ssl-truststore-passwords.properties | 0 .../{ => 6.x}/solr/ssl.repo.client.keystore | Bin .../{ => 6.x}/solr/ssl.repo.client.truststore | Bin .../zeppelin/ssl.repo.client.keystore | Bin .../zeppelin/ssl.repo.client.truststore | Bin .../templates/keystores/7.x/alfresco/keystore | Bin 0 -> 327 bytes .../keystores/7.x/alfresco/ssl.keystore | Bin 0 -> 3288 bytes .../keystores/7.x/alfresco/ssl.truststore | Bin 0 -> 1854 bytes .../keystores/7.x/client/browser.p12 | Bin 0 -> 3033 bytes .../7.x/solr/ssl-repo-client.keystore | Bin 0 -> 3298 bytes .../7.x/solr/ssl-repo-client.truststore | Bin 0 -> 2883 bytes .../7.x/zeppelin/ssl.repo.client.keystore | Bin 0 -> 3265 bytes .../7.x/zeppelin/ssl.repo.client.truststore | Bin 0 -> 2850 bytes .../generators/app/templates/latest/.env | 14 + .../templates/latest/docker-compose-ce.yml | 206 +++++++++++ .../templates/latest/docker-compose-ee.yml | 338 ++++++++++++++++++ 27 files changed, 603 insertions(+), 29 deletions(-) rename e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/{ => 6.x}/alfresco/keystore (100%) rename e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/{ => 6.x}/alfresco/keystore-passwords.properties (100%) rename e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/{ => 6.x}/alfresco/ssl-keystore-passwords.properties (100%) rename e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/{ => 6.x}/alfresco/ssl-truststore-passwords.properties (100%) rename e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/{ => 6.x}/alfresco/ssl.keystore (100%) rename e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/{ => 6.x}/alfresco/ssl.truststore (100%) rename e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/{ => 6.x}/client/browser.p12 (100%) rename e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/{ => 6.x}/solr/ssl-keystore-passwords.properties (100%) rename e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/{ => 6.x}/solr/ssl-truststore-passwords.properties (100%) rename e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/{ => 6.x}/solr/ssl.repo.client.keystore (100%) rename e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/{ => 6.x}/solr/ssl.repo.client.truststore (100%) rename e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/{ => 6.x}/zeppelin/ssl.repo.client.keystore (100%) rename e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/{ => 6.x}/zeppelin/ssl.repo.client.truststore (100%) create mode 100644 e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/7.x/alfresco/keystore create mode 100644 e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/7.x/alfresco/ssl.keystore create mode 100644 e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/7.x/alfresco/ssl.truststore create mode 100644 e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/7.x/client/browser.p12 create mode 100644 e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/7.x/solr/ssl-repo-client.keystore create mode 100644 e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/7.x/solr/ssl-repo-client.truststore create mode 100644 e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/7.x/zeppelin/ssl.repo.client.keystore create mode 100644 e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/7.x/zeppelin/ssl.repo.client.truststore create mode 100755 e2e-test/generator-alfresco-docker-compose/generators/app/templates/latest/.env create mode 100755 e2e-test/generator-alfresco-docker-compose/generators/app/templates/latest/docker-compose-ce.yml create mode 100755 e2e-test/generator-alfresco-docker-compose/generators/app/templates/latest/docker-compose-ee.yml diff --git a/e2e-test/generator-alfresco-docker-compose/README.md b/e2e-test/generator-alfresco-docker-compose/README.md index 00a45e386..68c0ab08f 100644 --- a/e2e-test/generator-alfresco-docker-compose/README.md +++ b/e2e-test/generator-alfresco-docker-compose/README.md @@ -17,8 +17,10 @@ $ tree generators/app/templates/ │   ├── .env │   ├── docker-compose-ce.yml │   └── docker-compose-ee.yml -├── empty -│   └── empty +├── latest +│   ├── .env +│   ├── docker-compose-ce.yml +│   └── docker-compose-ee.yml├── empty ├── images │   ├── alfresco │   │   ├── Dockerfile @@ -42,7 +44,7 @@ $ tree generators/app/templates/ │   │   └── sharding-share-config-custom.xml │   └── zeppelin │   └── Dockerfile -└── keystores +└── keystores :: 6.x / 7.x ├── alfresco ├── client ├── solr @@ -50,8 +52,8 @@ $ tree generators/app/templates/ ``` * `.env` includes default values for Docker Compose environment variables -* `docker-compose-ce.yml` is the base Docker Compose Template for Alfresco Community deployment (for ACS 6.2 and ACS 6.1) -* `docker-compose-ee.yml` is the base Docker Compose Template for Alfresco Enterprise deployment (for ACS 6.2 and ACS 6.1) +* `docker-compose-ce.yml` is the base Docker Compose Template for Alfresco Community deployment (for latest, ACS 6.2 and ACS 6.1) +* `docker-compose-ee.yml` is the base Docker Compose Template for Alfresco Enterprise deployment (for latest, ACS 6.2 and ACS 6.1) * `alfresco` includes a Dockerfile template to start Alfresco Repository * `model` includes a default content model (Sharding Explicit Routing or empty). This `empty` file is required for Dockerfile to work, so it should not be deleted. * `share` includes a Dockerfile template to start Share Web Application @@ -59,7 +61,7 @@ $ tree generators/app/templates/ * `search` includes a Dockerfile template to start Search Services and Insight Engine * `config` includes configuration for HTTP Web Proxy (NGINX) * `zeppelin` includes a Dockerfile template to start Zeppelin with SSL -* `keystores` includes every truststore and keystore required for SSL configuration +* `keystores` includes every truststore and keystore required for SSL configuration. Included different stores for ACS 6.x and ACS 7.x, as there were changes between this versions ## Installation @@ -80,15 +82,17 @@ $ yo alfresco-docker-compose ## ACS Version -Currently supported ACS Versions are `6.2` and `6.1` +Currently supported ACS Versions are `latest`, `6.2` and `6.1` This is the first choice to be selected when the generator is executed. ``` ? Which ACS version do you want to use? 6.1 -❯ 6.2 + 6.2 +❯ latest ``` + ## AGS Version Currently supported AGS Version is `latest` (based in ACS 6.1). @@ -197,7 +201,7 @@ $ yo alfresco-docker-compose --acsVersion=6.2 --alfrescoVersion=community --http **Parameter names reference** -`--acsVersion`: default 6.2, but could be set to 6.1 +`--acsVersion`: default latest, but could be set to 6.2 or 6.1 `--ags:`: only available when acsVersion=6.1 `--alfrescoVersion`: community or enterprise `--httpMode`: http or https diff --git a/e2e-test/generator-alfresco-docker-compose/generators/app/index.js b/e2e-test/generator-alfresco-docker-compose/generators/app/index.js index 2058411fc..a53d3d5c0 100644 --- a/e2e-test/generator-alfresco-docker-compose/generators/app/index.js +++ b/e2e-test/generator-alfresco-docker-compose/generators/app/index.js @@ -21,8 +21,8 @@ module.exports = class extends Generator { type: 'list', name: 'acsVersion', message: 'Which ACS version do you want to use?', - choices: [ "6.1", "6.2" ], - default: '6.2' + choices: [ '6.1', '6.2', 'latest' ], + default: 'latest' }, { whenFunction: response => response.acsVersion == '6.1', @@ -143,10 +143,8 @@ module.exports = class extends Generator { // Generate boilerplate from "templates" folder writing() { - var dockerComposeTemplateDirectory = '6.2'; - if (this.props.acsVersion.startsWith('6.1')) { - dockerComposeTemplateDirectory = '6.1'; - } + // Set base template directory: 6.1, 6.2, latest + var dockerComposeTemplateDirectory = this.props.acsVersion; // Docker Compose environment variables values this.fs.copyTpl( @@ -181,10 +179,10 @@ module.exports = class extends Generator { (this.props.alfrescoVersion == 'community' ? (this.props.ags ? 'alfresco/alfresco-governance-share-community' : - 'alfresco/alfresco-share') : + 'quay.io/alfresco/alfresco-share') : (this.props.ags ? 'quay.io/alfresco/alfresco-governance-share-enterprise': - 'alfresco/alfresco-share' + 'quay.io/alfresco/alfresco-share' ) ); @@ -297,21 +295,22 @@ module.exports = class extends Generator { // Add resources for SSL configuration if (this.props.httpMode == 'https') { + var subfolder = this.props.acsVersion == 'latest' ? '7.x' : '6.x' this.fs.copy( - this.templatePath('keystores/alfresco'), + this.templatePath('keystores/' + subfolder + '/alfresco'), this.destinationPath('keystores/alfresco') ) this.fs.copy( - this.templatePath('keystores/solr'), + this.templatePath('keystores/' + subfolder + '/solr'), this.destinationPath('keystores/solr') ) this.fs.copy( - this.templatePath('keystores/client'), + this.templatePath('keystores/' + subfolder + '/client'), this.destinationPath('keystores/client') ) if (this.props.zeppelin == true) { this.fs.copy( - this.templatePath('keystores/zeppelin'), + this.templatePath('keystores/' + subfolder + '/zeppelin'), this.destinationPath('keystores/zeppelin') ) } diff --git a/e2e-test/generator-alfresco-docker-compose/generators/app/templates/images/search/Dockerfile b/e2e-test/generator-alfresco-docker-compose/generators/app/templates/images/search/Dockerfile index 0ff7861ba..6c13bf1ed 100755 --- a/e2e-test/generator-alfresco-docker-compose/generators/app/templates/images/search/Dockerfile +++ b/e2e-test/generator-alfresco-docker-compose/generators/app/templates/images/search/Dockerfile @@ -32,16 +32,29 @@ ENV TRUSTSTORE_TYPE $TRUSTSTORE_TYPE ARG KEYSTORE_TYPE ENV KEYSTORE_TYPE $KEYSTORE_TYPE +# Passwords using Env Vars +ARG PASSWORDS_AS_ENV_VARS +ENV PASSWORDS_AS_ENV_VARS $PASSWORDS_AS_ENV_VARS + # Set SSL properties RUN if [ "$ALFRESCO_COMMS" == "https" ] ; then \ - sed -i '/^bash.*/i \ - sed -i "'"s/alfresco.encryption.ssl.keystore.location=.*/alfresco.encryption.ssl.keystore.location=\\\/opt\\\/<%=searchPath%>\\\/keystore\\\/ssl.repo.client.keystore/g"'" ${DIST_DIR}/solrhome/templates/rerank/conf/solrcore.properties && \ - sed -i "'"s/alfresco.encryption.ssl.keystore.passwordFileLocation=.*/alfresco.encryption.ssl.keystore.passwordFileLocation=\\\/opt\\\/<%=searchPath%>\\\/keystore\\\/ssl-keystore-passwords.properties/g"'" ${DIST_DIR}/solrhome/templates/rerank/conf/solrcore.properties && \ - sed -i "'"s/alfresco.encryption.ssl.keystore.type=.*/alfresco.encryption.ssl.keystore.type=${KEYSTORE_TYPE}/g"'" ${DIST_DIR}/solrhome/templates/rerank/conf/solrcore.properties && \ - sed -i "'"s/alfresco.encryption.ssl.truststore.location=.*/alfresco.encryption.ssl.truststore.location=\\\/opt\\\/<%=searchPath%>\\\/keystore\\\/ssl.repo.client.truststore/g"'" ${DIST_DIR}/solrhome/templates/rerank/conf/solrcore.properties && \ - sed -i "'"s/alfresco.encryption.ssl.truststore.passwordFileLocation=.*/alfresco.encryption.ssl.truststore.passwordFileLocation=\\\/opt\\\/<%=searchPath%>\\\/keystore\\\/ssl-truststore-passwords.properties/g"'" ${DIST_DIR}/solrhome/templates/rerank/conf/solrcore.properties && \ - sed -i "'"s/alfresco.encryption.ssl.truststore.type=.*/alfresco.encryption.ssl.truststore.type=${TRUSTSTORE_TYPE}/g"'" ${DIST_DIR}/solrhome/templates/rerank/conf/solrcore.properties' \ - ${DIST_DIR}/solr/bin/search_config_setup.sh; \ + if [ "$PASSWORDS_AS_ENV_VARS" == "true" ]; then \ + sed -i '/^bash.*/i \ + sed -i "'"s/alfresco.encryption.ssl.keystore.location=.*/alfresco.encryption.ssl.keystore.location=\\\/opt\\\/<%=searchPath%>\\\/keystore\\\/ssl-repo-client.keystore/g"'" ${DIST_DIR}/solrhome/templates/rerank/conf/solrcore.properties && \ + sed -i "'"s/alfresco.encryption.ssl.keystore.type=.*/alfresco.encryption.ssl.keystore.type=${KEYSTORE_TYPE}/g"'" ${DIST_DIR}/solrhome/templates/rerank/conf/solrcore.properties && \ + sed -i "'"s/alfresco.encryption.ssl.truststore.location=.*/alfresco.encryption.ssl.truststore.location=\\\/opt\\\/<%=searchPath%>\\\/keystore\\\/ssl-repo-client.truststore/g"'" ${DIST_DIR}/solrhome/templates/rerank/conf/solrcore.properties && \ + sed -i "'"s/alfresco.encryption.ssl.truststore.type=.*/alfresco.encryption.ssl.truststore.type=${TRUSTSTORE_TYPE}/g"'" ${DIST_DIR}/solrhome/templates/rerank/conf/solrcore.properties' \ + ${DIST_DIR}/solr/bin/search_config_setup.sh; \ + else \ + sed -i '/^bash.*/i \ + sed -i "'"s/alfresco.encryption.ssl.keystore.location=.*/alfresco.encryption.ssl.keystore.location=\\\/opt\\\/<%=searchPath%>\\\/keystore\\\/ssl.repo.client.keystore/g"'" ${DIST_DIR}/solrhome/templates/rerank/conf/solrcore.properties && \ + sed -i "'"s/alfresco.encryption.ssl.keystore.passwordFileLocation=.*/alfresco.encryption.ssl.keystore.passwordFileLocation=\\\/opt\\\/<%=searchPath%>\\\/keystore\\\/ssl-keystore-passwords.properties/g"'" ${DIST_DIR}/solrhome/templates/rerank/conf/solrcore.properties && \ + sed -i "'"s/alfresco.encryption.ssl.keystore.type=.*/alfresco.encryption.ssl.keystore.type=${KEYSTORE_TYPE}/g"'" ${DIST_DIR}/solrhome/templates/rerank/conf/solrcore.properties && \ + sed -i "'"s/alfresco.encryption.ssl.truststore.location=.*/alfresco.encryption.ssl.truststore.location=\\\/opt\\\/<%=searchPath%>\\\/keystore\\\/ssl.repo.client.truststore/g"'" ${DIST_DIR}/solrhome/templates/rerank/conf/solrcore.properties && \ + sed -i "'"s/alfresco.encryption.ssl.truststore.passwordFileLocation=.*/alfresco.encryption.ssl.truststore.passwordFileLocation=\\\/opt\\\/<%=searchPath%>\\\/keystore\\\/ssl-truststore-passwords.properties/g"'" ${DIST_DIR}/solrhome/templates/rerank/conf/solrcore.properties && \ + sed -i "'"s/alfresco.encryption.ssl.truststore.type=.*/alfresco.encryption.ssl.truststore.type=${TRUSTSTORE_TYPE}/g"'" ${DIST_DIR}/solrhome/templates/rerank/conf/solrcore.properties' \ + ${DIST_DIR}/solr/bin/search_config_setup.sh; \ + fi \ fi # REPLICATION diff --git a/e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/alfresco/keystore b/e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/6.x/alfresco/keystore similarity index 100% rename from e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/alfresco/keystore rename to e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/6.x/alfresco/keystore diff --git a/e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/alfresco/keystore-passwords.properties b/e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/6.x/alfresco/keystore-passwords.properties similarity index 100% rename from e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/alfresco/keystore-passwords.properties rename to e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/6.x/alfresco/keystore-passwords.properties diff --git a/e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/alfresco/ssl-keystore-passwords.properties b/e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/6.x/alfresco/ssl-keystore-passwords.properties similarity index 100% rename from e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/alfresco/ssl-keystore-passwords.properties rename to e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/6.x/alfresco/ssl-keystore-passwords.properties diff --git a/e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/alfresco/ssl-truststore-passwords.properties b/e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/6.x/alfresco/ssl-truststore-passwords.properties similarity index 100% rename from e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/alfresco/ssl-truststore-passwords.properties rename to e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/6.x/alfresco/ssl-truststore-passwords.properties diff --git a/e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/alfresco/ssl.keystore b/e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/6.x/alfresco/ssl.keystore similarity index 100% rename from e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/alfresco/ssl.keystore rename to e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/6.x/alfresco/ssl.keystore diff --git a/e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/alfresco/ssl.truststore b/e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/6.x/alfresco/ssl.truststore similarity index 100% rename from e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/alfresco/ssl.truststore rename to e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/6.x/alfresco/ssl.truststore diff --git a/e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/client/browser.p12 b/e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/6.x/client/browser.p12 similarity index 100% rename from e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/client/browser.p12 rename to e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/6.x/client/browser.p12 diff --git a/e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/solr/ssl-keystore-passwords.properties b/e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/6.x/solr/ssl-keystore-passwords.properties similarity index 100% rename from e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/solr/ssl-keystore-passwords.properties rename to e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/6.x/solr/ssl-keystore-passwords.properties diff --git a/e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/solr/ssl-truststore-passwords.properties b/e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/6.x/solr/ssl-truststore-passwords.properties similarity index 100% rename from e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/solr/ssl-truststore-passwords.properties rename to e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/6.x/solr/ssl-truststore-passwords.properties diff --git a/e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/solr/ssl.repo.client.keystore b/e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/6.x/solr/ssl.repo.client.keystore similarity index 100% rename from e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/solr/ssl.repo.client.keystore rename to e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/6.x/solr/ssl.repo.client.keystore diff --git a/e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/solr/ssl.repo.client.truststore b/e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/6.x/solr/ssl.repo.client.truststore similarity index 100% rename from e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/solr/ssl.repo.client.truststore rename to e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/6.x/solr/ssl.repo.client.truststore diff --git a/e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/zeppelin/ssl.repo.client.keystore b/e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/6.x/zeppelin/ssl.repo.client.keystore similarity index 100% rename from e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/zeppelin/ssl.repo.client.keystore rename to e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/6.x/zeppelin/ssl.repo.client.keystore diff --git a/e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/zeppelin/ssl.repo.client.truststore b/e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/6.x/zeppelin/ssl.repo.client.truststore similarity index 100% rename from e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/zeppelin/ssl.repo.client.truststore rename to e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/6.x/zeppelin/ssl.repo.client.truststore diff --git a/e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/7.x/alfresco/keystore b/e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/7.x/alfresco/keystore new file mode 100644 index 0000000000000000000000000000000000000000..439bc1f861272ed97542fa9242f42ffd04e16988 GIT binary patch literal 327 zcmXqLVsvI=WHxC0%f_kI=F#?@myw-uLF0Rt#y19yFAW->p$Ok#X}oICc*&sgJR3Ju zH4hgf>w+qS3S92M!f1Tik zH9Q_)YD(SXXZ-Z$HgqwNhr5we#8601fFYM5m7#8E2#ZiiW^Ss2 zp{a$bg_(hwfuV_^p@E%&q5&rxt2Q4qlN2ihi%7tUi}~TpGSzd_qXZSXHa(ri<~xB! f#N|lal#HE$%&VpJjhV!L%WjK|cV%K`Y+C>T8B%IO literal 0 HcmV?d00001 diff --git a/e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/7.x/alfresco/ssl.keystore b/e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/7.x/alfresco/ssl.keystore new file mode 100644 index 0000000000000000000000000000000000000000..c0cff56f8b1b130dd2ce09fdfbf9175e17aedae3 GIT binary patch literal 3288 zcmeH}do+~$8pr3Icdll}Z5ZW#t7NbAPJ>Kp8>vtdhQu!0#K>)CG(&Ec88U~YLTqtd z3Po-0lx&v>gR&LbF4cAy*>0OfWvYWSlU=9PY3)DGT4(jo{^PgS`#kUSuJwC=&+q&B zKCP{-tuPo2{&C6ic!3)jfj(R&&x^CcivfcHN8KL$VhuwcFfxItlF~^i8a^gj4!Iy4 zfWwszYnREr+s#`T#k{T5Fr-}A>fq)C5l~n|p^b2X001~742mM-AS?=WL7<3m>+K%` z3?A;XhfIVNq;Jr8nY|1@ZzkK9$?zs?K^oGe44!29=U%kw9G~D&2A8RA7wm0BmWO1e z1vos~g&n}=gtEzM&}wN0#4B2b@Pauk?T=qtSwfmBgiR1+44IKPQH(c{-Bk$2(vb3J z`1Lm^$ocfl03-|)B*Gv;JX|1vO@>%jj_+@-npB|{k;_4ub;DVMvgm8r2L-R9=JtlD zFN$+)>qRM7en_7kq5R?Hq>$f~B%fY-`Gnb+m#%lXR=s#4!H}eQFzyE)0p&iESNf{* z;en9$g%?DmPSr92Td3?#C3RQaD<~O1T-Mk?khPO-Q=HyY7jqtaEe9Y8;2;14@;o7r z_f64|MUW6mSj<3u!=3)AoAkRDnJK5~Vk6f+NK4WH4L!svh{XSL0bW`_L?9s$0G8i( zP(x1ggg{8HlJL_5kX#y2^5^`R-^g7#~ZnGaWmGazA5TSa<49a3&lxf0l+X)NpP7# zoE0rJdrL%cMIwO9kU9$c4GKU2hHxSP$ssT?V$W`Uk^j$d5G2qL(QrQ8Db`y*apH)- z#cb(dZ%|b~+LFgg%E#9xx2cl(sojZuP>g)6>q1LRw@d0w{_ah$(dnDzI{S>bF0nc} z>A1?+B8xGH{rC5u#Vz-8Ot+N-3UnWvE#!Q+G+cyE?LXGI&ZI;QxJ~^;j(>9Sj=Jpd z<^1H;P5yPK2QYEC4q)_o=5)tZvctup$0A*&{IH~U>JjCz|28-(k6J(-p#nrWN@92{3Pd zbMlnnrGc?W5%C6w_FymjEO})7`L=y;f7W*g4$l=OHpOjrWzhpvj+4%3)}Lu{5tQHY zCxR9&O>Qm+e4fN~nfExI5MQrTs3+dF+O^>HG^jsyWNB;E*{*R#BZTK9Rkk1tslw&sG+Gr7XuB^B@zvVU-%n6Kew-5M5*r4H9`>2!zCH&3o6{62r%gX_n)|^!zF;K( zgJ!aQ-b-=Xgwew5%_7G9W3Ywz< zE))&0-&?E%uW$$^@l>p*&;}5Sw*!C_uP7Qs`A>>~SWD917sO=K>2})IOg59t2xfX~ z)0y03Os=*SlN;>kn_Inmg($CV}_JdW*%Y{xhCr&LCp&C1!a)n_Lrf5 zNpLD(Ao%K6EBecWigb$j=t0O_h$xNMAn=e33L{}V5{U)@NJ*MhKoB5YG&Ydq#R&A} z@Pa?nj?}Jp&cJ$qwD<5q9UZOSg}$`kYUB0%s)W!mb!H8&`9cX5Wp^N~r~+;CPsig5 zLLz2v=O*{4OGndPTPpgD@rX+&eu&&WME({DoMWbfPpna&(y#YsO+^t6Vr1b4p_j< z#K^=XlBBSW|F>81_op1ccFAfPPq7Jz3^L$l8#k_1$?-y_Ry6a3t>BaBQgjp4L|L5ggAsS^Vc5UbWr5j%+Z995|m(Pdq zywF?kgUu5-SIuV>Yh+?(WMEvJU=Rn66j?qNF%}U{j{Ap>Y`ynpL+~E^*^|41f9QqZ zHIN5JfUGi$gn?KCb_F~j1;Q+>2F#3%|G_aR%MWC+umF=%8*%^u6Ad%a=M6`{uVCic ztKo3v*0X|}&t`q@VlPh?%ROqn|L8Q`kjvTO!XGNKFL7ATJ-H%owt+^u+|QnRk+s=gH3oi(>rkHzOTL+F3T z-fwrzR{3#v-YS}a?TJx)*Zwl-0A~sQ;^G|LqSOLlcF4(0%_{+BiNeSeCr-h$ z1T!!@Fh2%jA4ry95@1C0%fP2E)C^z*%mBuq3}6P%07g*;m9R{SC5Muc8MO>FAem7S zJu?OY1F|@?B)_Oq!5JK!2KbWx<@U`^uX&p;On|L>xprUR3I_I72@!97|9z4dSiJqsJ~PQm)^&v+4V32CMQHnduDxD3 z*?DDa^ZLh^-}%^0YB;0p8+pX9({EL3#_Sn9GRse|stD*7Pg#3s3;)luO{{;Lu9`$b zl716okwFt<9=HetCSN8424I=Rnb79J*!IJTkr9-9*&Gav5kkByYGNw>1*v(#!9EJ^ zsd=eIi6yBi3c;yGWvN99&Z$KunQ58HKry5oWmb2Bzdr2P`RmK8%!CE2HYW%t#~L*5 zgJh}3Z3c~-v1h7uzN3ox1h!t~-pU zzZ^E(cR}Mt*tBAXllGStXK@AXf870bfg;<^|IL+bITwyFKAaf%%=HCFdsd3^dBc;& z|NMOg*UVg@Hp9?x5A%wPN|VJl?!Eo>JiEVj^y4-`lX-d3K4E+xle=G>%USV9kR|)i cHGauea=oD&UI|J%%dUKLv}lRl!@X5e07h+FmjD0& literal 0 HcmV?d00001 diff --git a/e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/7.x/client/browser.p12 b/e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/7.x/client/browser.p12 new file mode 100644 index 0000000000000000000000000000000000000000..a58b6a736510e39f2e2fed678d4d4dd9bcd77bba GIT binary patch literal 3033 zcmY+DXE+;-7RM7KwP&o>9<_>uh^kd;OI4^*BQ4TtHAbl|ErQn8L=`cMsx5>!#%slF zwY1b;L3vSIYuw)Z-21-I{cxW1Jip&L=hOK^u&kqWKzaz4H4((D5Mvs1zz#eIBw|@3 z7_h7%zi}W0OE2|*iXO~>rRVyM*?t!ji1oi)Y%D-JB9^Wef~Bj3$b(q^hkqZxzyP+H z^e|{s5~xW)oAWYVr%zX@Kw#+T0UcHhSh~omni?1jC(P9^!+YLvO8yp1HpXVpjvYC= zxm3d0#}G=jxElu2W3vZm6tDSbjlEF==ql`xy62aCr>{iQ4!rl*j>qm*1Zlj^R()S| zVanuz$Z#Z!aMgy99*sJ?Zk$L|j(+~No*z{Fa537JoNCXwwev|D>ot<~Bv!fiX8vqW z^XZG%fIm?jm7zRh9)A@qKu^vEOgp z&DUr0qTw88U}{XCULWO$AaiJ%gP@4*0aF_I>qLZAv^ZJy2Dl9GSLvdPD2M>QrELqz ziVV3BCP?)+%1>_Ix-ky7PMU^(i(JVrR$8+%U)||H8q8O*Gu|!o%wY&}VPbR#K3Hkx z#q@g&I9Y|3D`o(C+>zRi{Fff)YXQ+`#Ox@NqS|q2a z-iWCWqxrt&Iw-vd=Z#+(@b9-loN)24U(MA*>MBhO0hzkStM(JjhE^rTQVH=)FL4?6pR74rkGbw}p)`-(|kOI^IBJ%f(GlQXA zq8tD>fG@xY5D4%C+y(eTB>vHSP%s0}&4*rh#i8nOC|nh$p$dV*G$B|f#NR{bKtwE) z&Tphi2L$|Xs{a_^|78R8?`-tcU35hnu3qGoJ#TS;#Le`uiLd=X*@(oRJCqXI7=)g( z&5gZupb-rgcS15uT;v&Iqklv_dLk9j+O2qs5}8331>(~GQihH2dQG>6vspO2WWSf` zkv6~ql7)JZ6V!XTX6DS;-AW=Qcrh(M9Ru#^CThhv*XZ&b&Da_-3$hE?_XQ$1&S_YN zeZ4|$>Zfs;er@j>tg`NS_U-<*bLr!T4Tz72i1bNvlG3VyHj1mv?UZHC#v~CJ^>%iGXwZb3p`reiJn%s7x^ZS z1jUVw)TFxm2qMMs#@_YZ*Lfx{HBPF>^lw$@c1_|0w{U`G$M3XlL@j50q_nf;rIS8a z+a!AOjsHcxfb*F0^Zgd%Sj_c{xO3yZ-5Z%G@yz&{D!HGjm_nvUA~Ce*ATId#+NAF@ zv^1uN5`3EXZ~Ox6dAME1Evf(7lCdrr?Dl_I6$=Y3eRwk)P$}lBrQA``X^p>g{&h zSPFN6b6>4z-RlE>U%h>)j;9MM^A*&vvZgoFK^JXyn~$(fR(syrW`&ykg`=$xR7xq>m=cuK;}pju@8`(+Pik*_Cm+)~8P&T^f+x77X_LiTw4o zx)kq6HcA{F!U;CxPrx@zD$)~L`^o!&nnj@%+pSTD{9=d9*I8^v zUcINSS(0N1Z6qHTNNGl-@4@rU**a$l64DKty%6z+?G964I8(Bu49#OH_meF14_@ZZ znb0zxg4OdyWoWSNs<$5zG#(`O|>1DIDYd|RvZYSQ4aL@OgS7R9MU`p zaO!$DIv_O`^ELCQhBfZYGdwH*$dneb78Ew5C$YLG=t@0CeR8D>&ULm9a}tC%8?ii)go-)aj&`mB?r7h-lO1mk;PGC(F$_&F4*T6<10Y?=!Na?^@E%<=O*oS z8vBT4?*)zaMG_Oorxr{mlljob`ng{W)Sd=4HSWAPAna~msR1a8UC+MMVV`w}ko2*~ z#ISACV&kg(Zq2}>FNfN<$#_@)iG}}4G ztWbn0H=6>;sL6v?u8&jdLimfB1J*fV=S<>Jv`@JT(OOu4NP5CE{|Yg^uxTy3AHd`C z(1ziC<06X^ui-nm$G14uLCw=2j}z;cEeelQ?~p!&ofxS?fySmH zAv}TXg`;NCp8NgTPL)UCD$(Ko#i)WptGOy~8~(OBVOcpyg5D!c+kh-WW;#nZZXx-d zlo>kcM8~IB5AfRx;Rnn#1 zZ`U@$Zo1pEfY|SA?e=BPk10nPLFnA=m7e8vJuH17argYMx>^6i6_*K_D-9WrhaHbG zL8F^hPN?=*qV59-{|MI?xoMQDYWL%OhZo07f)W$3Y_J3EZB=hd!Y295$BNN>Sg2Ml zLmyS%wVS8Qrxqx!iwtPV1PsC~(ybfK zP-{yva=9qXaG;B2vDB(|59TxQF0Z@RkAt;XJzV(Q@FW(WBj9>3QB~J8>l&;YzI!6v zO0&#vnKFG2%dX@M9X7$f`@p!y7^0YXRJ3GjNngDAQ?{1+oThV>bduzQ8vjDlYa#|w z{oN4r0dYK3HiL22oJ;4-lHyep`9UhrGRibDK=|<1N z<&N&??7HP7rj_D@DsX3OErSw|A=8~PfS-HdOJ(&(5?i4P_coVj P9}%1{uhG#1V~GC(R->oB literal 0 HcmV?d00001 diff --git a/e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/7.x/solr/ssl-repo-client.keystore b/e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/7.x/solr/ssl-repo-client.keystore new file mode 100644 index 0000000000000000000000000000000000000000..f1474dd04f3cbe8fed7fe9e6f68daf658e75e181 GIT binary patch literal 3298 zcmeH}do+~$8pr3I8^g>nrbfhZOWPs#J4P;vjoBp`F2t(2z;g{ba zH~-@^0}v1xHyHwQ&qKLf$V{kd{d1>|h8byta^m+ev9HFj{E8K+K(&`FpINjDmD^m+ zx48cxy)-jx;WyG8%??-GmL{2XyD(aRil$7DRJgzTB1xSf<(81i#vonii*7I1cR7dt zv@(fDC^c+hP-Qz^4G6!~HJ8*p_o{7e!C)!a9;sQ&f5rY2Rgn)6xKJ1X0g609kL{+& zW8nn&4wUHA`z?2#yu4xG_)SiFqW!AY@ljAg&;WTA_%4tT^Z5d_umBH7fG_~q+U}sd zgy0Dd7hI+Ij}Jg_X<&jsk7->ggQ1($Onx4k3?7`!ew`>1P9p@g8aB6H(sbzaKaO9B z@P8y~aHXTnBL`HYU*8)ZAEDxIQnS`lFgzO z?hjCXF4f?7B~A-YnSnTN0m;9Q=vG*ZpC4jntIDpG52v{{RlNZd1d;$C5^Oe8lf?)Y z$bsq02;>NIAujE0V*)Ld3Up8i${|r2NB|C~L-7D80T*3&pVRLQ)Xjs!U>FlP4;o_( zfWzTGsMwJ;$;H-a%l&nx2XfsElbmrEBNA#{u?2%!=0B$PR#98D-5tb{+`=H^bI!&E znL}FAeRra&v!gWX59LKu>aYiTtIhJ%L!1&a%!B*vUf|FAMi$Mb+O9A!+B9_Kcww(K z@UoeSmNsMGCLiQwWV)z?cTAeK4)*fHY)?n+nm^8j{%{iYMtLNriGN|CBuM?x!oyhU`_!Ep?NLTU1{kB<~rP z^ot;m`dN*6dncJ05Rc$D?Q1vkI@}ya<|7WaJlv-}@$=Hm8De`J;@f9phljN9N` z@BM?1VeNSfvwm(%Ut3HPD_0ARJWH|}(&@DmS6zqpnI^AQP*wZF>c;yLzY6O~U%zqt z3IFX>_0fbVf08r>lA5aUb=%473Es)8=BjQD-qvzA^gCgjR);*BmmR{y-BYP8T?gyz zM=j1@R(V>C44Bzxs6bCNYD}J8rBCIh8I8ZfE68rtP~7hB7+FpnP|Hl&TB45bT$3Jb zt9S6;gSE^A(ERBE&nA_aIvmbD1%n+wAdy02_RmS$=``CvdpSCfrWH0{5UVDM1zIHiPRB4n^ZgwfFq%0p6W|pFNIUsB6iAKN*({WPjLUZ1fi(b-l zu0{K`l9R_8?|2ig?j|>wQ66rS{XM{80(iqF1dH+x;80266Xkt*8vZw~wFS5)3308v zjcaXJF!H^He8x}y2kTG=)!tc$(x=v8Cy+hcmlMQ_R3vYUAN(^=_no_CHiO|k%BVCX z4fHmq_#s_v4R%*XC+##!I(zDAR{QVknTrk8y$=Sf__k#bkqS@eS*y2;3X(;4U-ETj zA`#a^UV+NF#?BhnucHS-(#hpZ|YjwMOc zt43~1tcBe~Z1VbbT;7422e3dM5V&o)0@y-9h`?b{p7BS3_*F9i5W*H|0+Rld!oc?u z_Sgh70`2W7isp^*ZoJk#2Kfyo`LU9>*8^z zZ+YTr?jV={&H%WzAoteiE#NBmY%!Pn#cvqz^J9x}rug7UhyoBVj8)-iPz)(5Ks*8= z0t3Ju!Xyrk0mXSJW)O|a^a)~fK9P{ny!PMIsMLxL8TL6q79ZbQMqPjG+mvx8*_w zuJUr(-Yg=q5mxq-ayp@=X?VFu#K!Q%L_AI>FYpA#3HyqcG~LZC+rYv7H-3?j`D9f#yiwCM`2_QILf6({|3{#rV#)D literal 0 HcmV?d00001 diff --git a/e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/7.x/solr/ssl-repo-client.truststore b/e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/7.x/solr/ssl-repo-client.truststore new file mode 100644 index 0000000000000000000000000000000000000000..c09e6891a0e5e89969690e53b91828140b6e4e16 GIT binary patch literal 2883 zcmX?i?%X*B1_mY|W(L#z#l<Q3<2haEeAeR-9cuwd2Z1mWaZgT{TpV3bv6 zY20SexVb@uljHuOBU|si*$}+Pe)i<9;2(P7cNaFcENN^U@D$c0lBBSW|F>81_op1c zcFAfPPq7Jz3<4)oS$;;w|DZr(U0@&#@{cMDj{z4Ohc+7{D=RxQqk#xWP>_Y!fV+t^ zCqFqcCnLYO1UXRvvk)^dHJBdIUZdAscss+&nfuZIubk^{WbU21_3y4bjHkaGHrsbW z<3-rCVuq9UmlbDm1?_*_{dIvN+s^;Zm25c|jxau)82HTf1xI^Uit%~Flg9u2eFfLd zT%k6@&~Oj)ii=8<#WwD}{q;P%zjgHEHbIkldC@*$d>@m$U!2QX@kfv)`_DCgNknmw z2+fqb$%&BS;AkqmIAD5h(8L5R5|{!Y#R2w!8QcYgA+Ufj0TmF&-~z%hYQWPM-eTf) zspGDfVJCOL7Pnk!xPzHX{n6Y9eC(?@Pb~df|H-#h=I@XBo`()kUN(KkhbPAGl7j@7 z9GfUGW6Ppii|5HoDV6eve!S||5)+E)m|o1w7WIDdman_cM3i3q{!Exvarb{-&K06j zmSWd-?q9m`Wzx2zM|k;s_|6Nx^*-12k1V+Twzc5T-R6lWt*aZ)1n&7Cx5vbg zVe07vZlO}MuI8$~6xy9Nw^fhD=QTsHP^(5z%15difhoU;x>cjHfg&)Dfz`AW z?y3=QV!zwGb=6l&=8Y(r!_AZ!qSNZe9#k*#h zOIxhF9VhO&ZrX0oACDMR_^%rm?N+l+F3eiwBYe=RT&``gbH?W@ehe*#msKonm^p97 zzVBH-961gJ&uVONi1tzWP~Ojc?(ChbGK^-&zJ+|rb1FT2nT~a$ulmf^jBSeE(^5-! z7{5QmDO6;tIa(*eo0XtC(e*vUoi%=OX>5g0_qwq!40G0By<~THd4){sKJJrKHd?az zL{zNY$?h>Hutu;=nB$YHQB?iH!WrSGciuIOT`~7jz_idbjTQg?c}?a|JJ0B$b?oc| z1|D|P(uP;T{dNCpci7rlIJ`CV^vs)aM6kI$H#_g%fxITZ3+I#1@n~LQS^4{?#e~B$ UK6_e{>&hArKW8|-kn?aG0D2FY!vFvP literal 0 HcmV?d00001 diff --git a/e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/7.x/zeppelin/ssl.repo.client.keystore b/e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/7.x/zeppelin/ssl.repo.client.keystore new file mode 100644 index 0000000000000000000000000000000000000000..b0dc4ef08f20f8e319befa4f5c61f3599c135152 GIT binary patch literal 3265 zcmX?i?%X*B1_mZL=1$B>D@rX+&euy$WME*-o*|~4%)lCpt}rciG~ zVFN*s3_A~(Z(?RjYFfhHvk{{Irtt#G+IMpOO?kLjeOmkOFQV z_RzfSy!`S!Lums^kO(u6kaKBqNq(*ZT&J_6ft)z6p{0SDk-33|v4yc&lsK<3h-(Dp z4!i>!=Oc$1BP#=QV=se2V<%H%Bf~c907es!s9w(ED5hD6m0FYjA7-%jzSU)dzBzY`yP;d)~*FL}r8IN1|DK6@?nyqiD1EM5Ek{%Of4 zDN_v7_lqn}e#Xkhkrnau?d6%z1LX|grA&U=WmBQFhf9ZtqovGKsPtX->y=w>xtzZG zkcpX*fpKwyK^!cW=6*U;24zU2eMdLfa#|VIRJo(h8gJdV4IBQNoz|#J~+Hud}FghHkZ)3 z#*Ze2(^kLD6F={|C0X$CIT`D&w=--{h6wwf=9rsu)xSIF@lNgMq0`>@N;cT5wU?)C zYCkFzw{U^yLiWEqYyuMRTYf*fG+?LQ|78y~(^F0uCe7Gz=;y3uaue6<*~HjRN=bf` zeMot)(HpKSyUs-b^8q6Re{pe+UQucRFc0KprskDE@`2_)Uh*%(Bm?*~;bK%GGVYXOm`s z7YaUeVe)$}OVO?KL!Xvx+W&sNhSA(pwceLs9+ToXobok3N&mds+_@jNd_TOrVV6VT zdFji?Y7V!44hfKn(KA}VXr*g(WPa#uj*Im@k{@5@wcXFU{9R2xKqDq#!u~TB@u^&w z*4|A>@6Yp|?cZ#_OR&tlxh~~ymwhZl?A|$Fu18*b(a`OD%c*OF#arWjW%W;INlsPC zw7IkYaQQ*0SNED0M=#S;I3&A2B41`JYmMH<>7QlW?z#Vbw>n9`r!}v0A@jot(#yWR zShvjZfJ8=*-0DWP>C)RmbndB?N*q+lxXX}WxvcP?NQ2(AYr@rf+m~L^O$nZBskwyB zf4#tyck@b)H{NM4n>;0=>&1FChPQW{AJ$ofZGE)b_Xkt{MD>3iX3_WGa_;%GssC`? zcFl-wvEt3o7PEgj+;D+?;yi!ZnJ%Sbu0JxR1=j5?KE*S^r$6Jvd$uP_`{sZ2(ddp% z6YZb!PS?O8vF6t52cHy+vi$bET)9x(@5#K(rT>rJov~zZU~z6{IPcQ-;yN~M-~Fpw zmxpmgO$b-^-4wfR-Qw*&-XB~-i-HWQPHZTS>WH%4dZtBmoBc)~{r($G1%Y!!IhgJg zRfi-<{BK@7mot7?sb#i&#y9TxqI1KE~rpqdGfVa8~4n5?){m=?oz;>Wn~*b z{f-Sdc*k)nyUgYcw`k*}OPq!2Wt-NSgNt)msm%;5v6*iGu_L6^W)fgT%XtG|8e$gL zMxf%_1Xf&E!m1D~^$8hOhn9f`r0P&auR4N&*}gclB)_Oq!5N(S4e%Azt1le7%2E8X z>xO})zpma3%P`?vnZMQO08iEpE&3B#V!3&u8d}JZ~q5)_xMWGi@kSNWxehA z@1fbOx&zPMSH7GXE6K3ocr#nN#W~aEkb=62G2NhvF&SLp0t;Iv0|sCX%bC#T!Pxf0 ziIEXh*s?ho7$byuS=7W-{0ma^f`fe&+*9*XixNvxQxt+zi^@`q6r59wN;1e>|sjhi7AK;vqI#ueDBfQ5}MOBx#oJcTtOwN*e3 z33xTcy1+mf#8+kEG2mk30G8{ltnAE;1|lFqK^9&E?k3Kh{N%)(jQrvfw0r}qu`;f) zwA_DJ&ETVN>X6!y@!8eset5m={+if#%9UTw=jy0Tvu$Qj=UM-bW68-$MxSPXH(LI1 z{9ja4(np^KNnFsxgabe5dpRw}vR2`Wm0mgp< zIM{vWx*uby;@kV?V&2=8_EVOhcqevivFe;zqm45fv>x&0`O(JNKsC;;JZjwkwWCsp R^IlGP`#I&4U%Td=1_1Dfk68c! literal 0 HcmV?d00001 diff --git a/e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/7.x/zeppelin/ssl.repo.client.truststore b/e2e-test/generator-alfresco-docker-compose/generators/app/templates/keystores/7.x/zeppelin/ssl.repo.client.truststore new file mode 100644 index 0000000000000000000000000000000000000000..2feb2d8270a13046c97c5fade08f3b3eaad25749 GIT binary patch literal 2850 zcmX?i?%X*B1_mY|W(Lz7#l<;#MX3e(3=E9fGsM(87+53pObsl7@>dO-m@fjc)dFTF zMkXdE0R{tJHcqWJkGAi;jEvl@3eG0uz?^*hMkAY zH!(9MH7_GIF~v~cKo%s(#UtjJlU9^koSd%^oS#-wo>-Ks;8T*KXDDF62U5Vz!ycNK zotIypXDDqT2@+xE5ppgqF3HbTfa`R2G>{YLHMBG^Gcq@@Ft#u@ixTHG0&z{CT!SK* zo3J>L47Vs7C_>yKh3=LhUsBt#LURRxVVWi-Jppv860)8!YoV%48XYJOlb39 zZ2RHF$jHLN%*5tkV2lvrWlCa8J!kElMm&O;HF=Eh>+I2lf#^ILZ);btv3Ny_pwGX4jJ4C?{| zVUQM;Zk+06mv!J=&kxG=iO?*lmz)U6{AzpPnV;!_K@-y* zAa;dhex%SC{AGYKs0=WIl>q}@NZ~6Gwpj--ns`L@au!E1ZIk?eZU3@t-+#q93XUdi zC;CHQMB_-@5sBiZ}hrCTabh_}~lI8-sbtJ7&kpp6Kz}YoX`e{PAV!+VA&I zOFl`NVwk>PWO4E{RxXaLh^KEa&wL&zXZS8<^2;up3Z*?KbjOyTm3dq{JiUyWWmShWURa1&agchBJ6vbV{Xb-|L&m2JGGyO zPJ81k*#mp#->PdQM6Jt9mCHYPEA?3YBZ@8}PIv0V+2cX(dFF7YOHLnDc4>Zc)`G6Ui2bgaFu_LS! zWJL4JfR|JRE5S+FDD?tTkaB$+S3s+w_bd{s{W!V$%Q}cp!qeRbF z&E?-2GX0$N4Y?ju^_zN^4=hzrH>vuzdc(TwYBj6+x6gffZvDp7i(a!I+}3>kpZU7O zbJl&W>t<0h^Oah|6h3jz>x*0Zqg)xy;@*^F)BW&z)%`WG@02UQp3l`$nP%I}pw6@Y9mkTBlZ-yi{%*AV;l^1_*79Bd zW;d^Nd45?YHT$T*x)aalggR8G`ZTxRV=@o=7vsW~yFX*)>8Uz0O9G7l1aPqX%ymD; uQpLCT&BeU8EA6K&Kk-iN)?(E;vvh9q&lF|Oip`nJIy=km^B=8aOZ5TmUxfGo literal 0 HcmV?d00001 diff --git a/e2e-test/generator-alfresco-docker-compose/generators/app/templates/latest/.env b/e2e-test/generator-alfresco-docker-compose/generators/app/templates/latest/.env new file mode 100755 index 000000000..405062f6a --- /dev/null +++ b/e2e-test/generator-alfresco-docker-compose/generators/app/templates/latest/.env @@ -0,0 +1,14 @@ +ALFRESCO_TAG=latest +ALFRESCO_CE_TAG=latest +SHARE_TAG=latest +POSTGRES_TAG=11.4 +TRANSFORM_ROUTER_TAG=1.2.0 +TRANSFORM_CORE_AIO_TAG=2.2.1 +SHARED_FILE_STORE_TAG=0.7.0 +ACTIVE_MQ_TAG=5.15.8 +DIGITAL_WORKSPACE_TAG=1.5.0 +ACS_NGINX_TAG=3.0.1 +ACS_COMMUNITY_NGINX_TAG=1.0.0 +SEARCH_TAG=latest +ZEPPELIN_TAG=latest +ACA_TAG=master-latest \ No newline at end of file diff --git a/e2e-test/generator-alfresco-docker-compose/generators/app/templates/latest/docker-compose-ce.yml b/e2e-test/generator-alfresco-docker-compose/generators/app/templates/latest/docker-compose-ce.yml new file mode 100755 index 000000000..fb3ab2c01 --- /dev/null +++ b/e2e-test/generator-alfresco-docker-compose/generators/app/templates/latest/docker-compose-ce.yml @@ -0,0 +1,206 @@ +# Using version 2 as 3 does not support resource constraint options (cpu_*, mem_* limits) for non swarm mode in Compose +version: "2" + +services: + + alfresco: + build: + context: ./alfresco + args: + ALFRESCO_TAG: ${ALFRESCO_CE_TAG} + SOLR_COMMS: <%=secureComms%> <% if (httpMode == 'https') { %> + TRUSTSTORE_TYPE: JCEKS + TRUSTSTORE_PASS: kT9X6oe68t + KEYSTORE_TYPE: JCEKS + KEYSTORE_PASS: kT9X6oe68t <% } %> + COMPRESS_CONTENT: "<%=gzip%>" + mem_limit: 1800m + environment: + JAVA_OPTS : " + -Ddb.driver=org.postgresql.Driver + -Ddb.username=alfresco + -Ddb.password=alfresco + -Ddb.url=jdbc:postgresql://postgres:5432/alfresco + -Dsolr.host=<%=searchSolrHost%> + -Dsolr.port.ssl=8983 + -Dsolr.secureComms=<%=secureComms%> + -Dsolr.baseUrl=/solr + -Dindex.subsystem.name=solr6 + -Dshare.host=localhost + -Dalfresco.port=8080 + -Daos.baseUrlOverwrite=http://localhost:8080/alfresco/aos + -Dmessaging.broker.url=\"failover:(nio://activemq:61616)?timeout=3000&jms.useCompression=true\" + -Ddeployment.method=DOCKER_COMPOSE + + -DlocalTransform.core-aio.url=http://transform-core-aio:8090/ + -Dalfresco-pdf-renderer.url=http://transform-core-aio:8090/ + -Djodconverter.url=http://transform-core-aio:8090/ + -Dimg.url=http://transform-core-aio:8090/ + -Dtika.url=http://transform-core-aio:8090/ + -Dtransform.misc.url=http://transform-core-aio:8090/ + + -Dcsrf.filter.enabled=false + -Dalfresco.restApi.basicAuthScheme=true + -Xms1500m -Xmx1500m + " <% if (httpMode == 'https') { %> + ports: + - 8443:8443 + volumes: + - ./keystores/alfresco:/usr/local/tomcat/alf_data/keystore <% } %> + + solr6: + build: + context: ./search + args: + SEARCH_TAG: ${SEARCH_TAG} + SOLR_HOSTNAME: solr6 + ALFRESCO_HOSTNAME: alfresco + ALFRESCO_COMMS: <%=secureComms%> <% if (httpMode == 'https') { %> + TRUSTSTORE_TYPE: JCEKS + KEYSTORE_TYPE: JCEKS <% } %> <% if (replication) { %> + ENABLE_MASTER: "true" + ENABLE_SLAVE: "false" <% } %> + COMPRESS_CONTENT: "<%=gzip%>" + mem_limit: 1200m + environment: + #Solr needs to know how to register itself with Alfresco + SOLR_ALFRESCO_HOST: "alfresco" + SOLR_ALFRESCO_PORT: "<%=alfrescoPort%>" + ALFRESCO_SECURE_COMMS: <%=secureComms%> + #Alfresco needs to know how to call solr + SOLR_SOLR_HOST: "solr6" + SOLR_SOLR_PORT: "8983" + #Create the default alfresco and archive cores + SOLR_CREATE_ALFRESCO_DEFAULTS: "alfresco,archive" + SOLR_JAVA_MEM: "-Xms1g -Xmx1g" <% if (httpMode == 'https') { %> + SOLR_SSL_TRUST_STORE: "/opt/<%=searchPath%>/keystore/ssl.repo.client.truststore" + SOLR_SSL_TRUST_STORE_PASSWORD: "kT9X6oe68t" + SOLR_SSL_TRUST_STORE_TYPE: "JCEKS" + SOLR_SSL_KEY_STORE: "/opt/<%=searchPath%>/keystore/ssl.repo.client.keystore" + SOLR_SSL_KEY_STORE_PASSWORD: "kT9X6oe68t" + SOLR_SSL_KEY_STORE_TYPE: "JCEKS" + SOLR_SSL_NEED_CLIENT_AUTH: "true" + SOLR_OPTS: " + -Dsolr.ssl.checkPeerName=false + -Dsolr.allow.unsafe.resourceloading=true + " <% } %> + ports: + - 8083:8983 <% if (httpMode == 'https') { %> + volumes: + - ./keystores/solr:/opt/<%=searchPath%>/keystore <% } %> + + <% if (replication) { %> + solr6secondary: + build: + context: ./search + args: + SEARCH_TAG: ${SEARCH_TAG} + SOLR_HOSTNAME: solr6secondary + ALFRESCO_HOSTNAME: alfresco + ALFRESCO_COMMS: <%=secureComms%> <% if (httpMode == 'https') { %> + TRUSTSTORE_TYPE: JCEKS + KEYSTORE_TYPE: JCEKS <% } %> <% if (replication) { %> + ENABLE_MASTER: <% if (replication == 'master-master') { %>"true"<% } else { %>"false"<% } %> + ENABLE_SLAVE: <% if (replication == 'master-master') { %>"false"<% } else { %>"true"<% } %> + MASTER_HOST: solr6 <% } %> + COMPRESS_CONTENT: "<%=gzip%>" + mem_limit: 1200m + environment: + #Solr needs to know how to register itself with Alfresco + SOLR_ALFRESCO_HOST: "alfresco" + SOLR_ALFRESCO_PORT: "<%=alfrescoPort%>" + ALFRESCO_SECURE_COMMS: <%=secureComms%> + #Alfresco needs to know how to call solr + SOLR_SOLR_HOST: "solr6secondary" + SOLR_SOLR_PORT: "8983" + #Create the default alfresco and archive cores + SOLR_CREATE_ALFRESCO_DEFAULTS: "alfresco,archive" + SOLR_JAVA_MEM: "-Xms1g -Xmx1g" <% if (httpMode == 'https') { %> + SOLR_SSL_TRUST_STORE: "/opt/<%=searchPath%>/keystore/ssl.repo.client.truststore" + SOLR_SSL_TRUST_STORE_PASSWORD: "kT9X6oe68t" + SOLR_SSL_TRUST_STORE_TYPE: "JCEKS" + SOLR_SSL_KEY_STORE: "/opt/<%=searchPath%>/keystore/ssl.repo.client.keystore" + SOLR_SSL_KEY_STORE_PASSWORD: "kT9X6oe68t" + SOLR_SSL_KEY_STORE_TYPE: "JCEKS" + SOLR_SSL_NEED_CLIENT_AUTH: "true" + SOLR_OPTS: " + -Dsolr.ssl.checkPeerName=false + -Dsolr.allow.unsafe.resourceloading=true + " <% } %> + ports: + - 8084:8983 <% if (httpMode == 'https') { %> + volumes: + - ./keystores/solr:/opt/<%=searchPath%>/keystore <% } %> + <% } %> + + transform-core-aio: + image: alfresco/alfresco-transform-core-aio:${TRANSFORM_CORE_AIO_TAG} + mem_limit: 1536m + environment: + JAVA_OPTS: " -Xms256m -Xmx1536m" + ports: + - 8090:8090 + + share: + build: + context: ./share + args: + SHARE_TAG: ${SHARE_TAG} + mem_limit: 1g + environment: + REPO_HOST: "alfresco" + REPO_PORT: "8080" + JAVA_OPTS: " + -Xms500m + -Xmx500m + -Dalfresco.host=localhost + -Dalfresco.port=8080 + -Dalfresco.context=alfresco + -Dalfresco.protocol=http + " + + postgres: + image: postgres:${POSTGRES_TAG} + mem_limit: 512m + environment: + - POSTGRES_PASSWORD=alfresco + - POSTGRES_USER=alfresco + - POSTGRES_DB=alfresco + command: postgres -c max_connections=300 -c log_min_messages=LOG + ports: + - 5432:5432 + + activemq: + image: alfresco/alfresco-activemq:${ACTIVE_MQ_TAG} + mem_limit: 1g + ports: + - 8161:8161 # Web Console + - 5672:5672 # AMQP + - 61616:61616 # OpenWire + - 61613:61613 # STOMP + + content-app: + image: alfresco/alfresco-content-app:${ACA_TAG} + mem_limit: 256m + depends_on: + - alfresco + + # HTTP proxy to provide HTTP Default port access to services + # SOLR API and SOLR Web Console are protected to avoid unauthenticated access + proxy: + image: nginx:stable-alpine + mem_limit: 128m + depends_on: + - alfresco + volumes: + - ./config/nginx.conf:/etc/nginx/nginx.conf + - ./config/nginx.htpasswd:/etc/nginx/conf.d/nginx.htpasswd <% if (httpWebMode == 'https') { %> + - ./config/cert/localhost.cer:/etc/nginx/localhost.cer + - ./config/cert/localhost.key:/etc/nginx/localhost.key <% } %> + ports: + - <%=port%>:<%=port%> + links: + - alfresco + - share + - solr6 + - content-app \ No newline at end of file diff --git a/e2e-test/generator-alfresco-docker-compose/generators/app/templates/latest/docker-compose-ee.yml b/e2e-test/generator-alfresco-docker-compose/generators/app/templates/latest/docker-compose-ee.yml new file mode 100755 index 000000000..5b3ebb509 --- /dev/null +++ b/e2e-test/generator-alfresco-docker-compose/generators/app/templates/latest/docker-compose-ee.yml @@ -0,0 +1,338 @@ +# Using version 2 as 3 does not support resource constraint options (cpu_*, mem_* limits) for non swarm mode in Compose +version: "2" + +services: + alfresco: + build: + context: ./alfresco + args: + ALFRESCO_TAG: ${ALFRESCO_TAG} + SOLR_COMMS: <%=secureComms%> <% if (httpMode == 'https') { %> + TRUSTSTORE_TYPE: JCEKS + TRUSTSTORE_PASS: kT9X6oe68t + KEYSTORE_TYPE: JCEKS + KEYSTORE_PASS: kT9X6oe68t <% } %> + COMPRESS_CONTENT: "<%=gzip%>" + mem_limit: 1800m + environment: + JAVA_OPTS: " + -Ddb.driver=org.postgresql.Driver + -Ddb.username=alfresco + -Ddb.password=alfresco + -Ddb.url=jdbc:postgresql://postgres:5432/alfresco + -Dsolr.host=<%=searchSolrHost%> + -Dsolr.port.ssl=8983 + -Dsolr.secureComms=<%=secureComms%> + -Dsolr.baseUrl=/solr <% if (sharding == 'true') { %> + -Dsolr.useDynamicShardRegistration=true <% } %> + -Dindex.subsystem.name=solr6 + + -Daos.baseUrlOverwrite=http://localhost:8080/alfresco/aos + -Dmessaging.broker.url=\"failover:(nio://activemq:61616)?timeout=3000&jms.useCompression=true\" + -Ddeployment.method=DOCKER_COMPOSE + + -Dmetadata-keystore.password=mp6yc0UD9e + -Dmetadata-keystore.aliases=metadata + -Dmetadata-keystore.metadata.password=mp6yc0UD9e + -Dmetadata-keystore.metadata.algorithm=AES <% if (httpMode == 'https') { %> + -Dencryption.keystore.type=pkcs12 + -Dencryption.cipherAlgorithm=AES/CBC/PKCS5Padding + -Dencryption.keyAlgorithm=AES <% } %> + + -Dtransform.service.enabled=true + -Dtransform.service.url=http://transform-router:8095 + -Dsfs.url=http://shared-file-store:8099/ + -DlocalTransform.core-aio.url=http://transform-core-aio:8090/ + -Dalfresco-pdf-renderer.url=http://transform-core-aio:8090/ + -Djodconverter.url=http://transform-core-aio:8090/ + -Dimg.url=http://transform-core-aio:8090/ + -Dtika.url=http://transform-core-aio:8090/ + -Dtransform.misc.url=http://transform-core-aio:8090/ + + -Dcsrf.filter.enabled=false + -Dalfresco.restApi.basicAuthScheme=true + -Xms1500m -Xmx1500m + <% if (httpMode == 'https') { %> + -Dssl-keystore.password=kT9X6oe68t + -Dssl-keystore.aliases=ssl-alfresco-ca,ssl-repo + -Dssl-keystore.ssl-alfresco-ca.password=kT9X6oe68t + -Dssl-keystore.ssl-repo.password=kT9X6oe68t + -Dssl-truststore.password=kT9X6oe68t + -Dssl-truststore.aliases=alfresco-ca,ssl-repo-client + -Dssl-truststore.alfresco-ca.password=kT9X6oe68t + -Dssl-truststore.ssl-repo-client.password=kT9X6oe68t <% } %> + " <% if (httpMode == 'https') { %> + ports: + - 8443:8443 + volumes: + - ./keystores/alfresco:/usr/local/tomcat/alf_data/keystore <% } %> + + solr6: + build: + context: ./search + args: + SEARCH_TAG: ${SEARCH_TAG} + SOLR_HOSTNAME: solr6 + ALFRESCO_HOSTNAME: alfresco + ALFRESCO_COMMS: <%=secureComms%> <% if (httpMode == 'https') { %> + TRUSTSTORE_TYPE: JCEKS + KEYSTORE_TYPE: JCEKS + PASSWORDS_AS_ENV_VARS: "true" <% } %> <% if (replication) { %> + ENABLE_MASTER: "true" + ENABLE_SLAVE: "false" <% } %> <% if (sharding == 'true') { %> + ENABLE_SHARDING: "true" + NUM_SHARDS: "2" + SHARD_ID: "0" <% if (shardingMethod == 'DB_ID_RANGE') { %> + SHARD_RANGE: "0-800" <% } %> <% if (shardingMethod == 'DATE') { %> + SHARD_KEY: "cm:created" + SHARD_DATE_GROUPING: "2" <% } %> <% if (shardingMethod == 'PROPERTY') { %> + SHARD_KEY: "shard:shardId" <% } %> <% if (shardingMethod == 'EXPLICIT_ID' || shardingMethod == 'EXPLICIT_ID_FALLBACK_LRIS') { %> + SHARD_KEY: "shard:shardId" <% } %> + SHARDING_METHOD: <%=shardingMethod%> <% } %> + COMPRESS_CONTENT: "<%=gzip%>" + mem_limit: 1200m + environment: + #Solr needs to know how to register itself with Alfresco + SOLR_ALFRESCO_HOST: "alfresco" + SOLR_ALFRESCO_PORT: "<%=alfrescoPort%>" + ALFRESCO_SECURE_COMMS: <%=secureComms%> + #Alfresco needs to know how to call solr + SOLR_SOLR_HOST: "solr6" + SOLR_SOLR_PORT: "8983" + #Create the default alfresco and archive cores + SOLR_CREATE_ALFRESCO_DEFAULTS: "alfresco,archive" + SOLR_JAVA_MEM: "-Xms1g -Xmx1g" <% if (httpMode == 'https') { %> + SOLR_SSL_TRUST_STORE: "/opt/<%=searchPath%>/keystore/ssl-repo-client.truststore" + SOLR_SSL_TRUST_STORE_PASSWORD: "kT9X6oe68t" + SOLR_SSL_TRUST_STORE_TYPE: "JCEKS" + SOLR_SSL_KEY_STORE: "/opt/<%=searchPath%>/keystore/ssl-repo-client.keystore" + SOLR_SSL_KEY_STORE_PASSWORD: "kT9X6oe68t" + SOLR_SSL_KEY_STORE_TYPE: "JCEKS" + SOLR_SSL_NEED_CLIENT_AUTH: "true" <% if (sharding == 'true') { %> + SOLR_SSL_CLIENT_KEY_STORE: "/opt/<%=searchPath%>/keystore/ssl-repo-client.keystore" + SOLR_SSL_CLIENT_KEY_STORE_PASSWORD: "kT9X6oe68t" + SOLR_SSL_CLIENT_KEY_STORE_TYPE: "JCEKS" + SOLR_SSL_CLIENT_TRUST_STORE: "/opt/<%=searchPath%>/keystore/ssl-repo-client.keystore" + SOLR_SSL_CLIENT_TRUST_STORE_PASSWORD: "kT9X6oe68t" + SOLR_SSL_CLIENT_TRUST_STORE_TYPE: "JCEKS" <% } %> + SOLR_OPTS: " + -Dsolr.ssl.checkPeerName=false + -Dsolr.allow.unsafe.resourceloading=true + -Dssl-keystore.password=kT9X6oe68t + -Dssl-keystore.aliases=ssl-alfresco-ca,ssl-repo-client + -Dssl-keystore.ssl-alfresco-ca.password=kT9X6oe68t + -Dssl-keystore.ssl-repo-client.password=kT9X6oe68t + -Dssl-truststore.password=kT9X6oe68t + -Dssl-truststore.aliases=ssl-alfresco-ca,ssl-repo,ssl-repo-client + -Dssl-truststore.ssl-alfresco-ca.password=kT9X6oe68t + -Dssl-truststore.ssl-repo.password=kT9X6oe68t + -Dssl-truststore.ssl-repo-client.password=kT9X6oe68t + " <% } %> + ports: + - 8083:8983 <% if (httpMode == 'https') { %> + volumes: + - ./keystores/solr:/opt/<%=searchPath%>/keystore <% } %> + + <% if (sharding == 'true' || replication) { %> + solr6secondary: + build: + context: ./search + args: + SEARCH_TAG: ${SEARCH_TAG} + SOLR_HOSTNAME: solr6secondary + ALFRESCO_HOSTNAME: alfresco + ALFRESCO_COMMS: <%=secureComms%> <% if (httpMode == 'https') { %> + TRUSTSTORE_TYPE: JCEKS + KEYSTORE_TYPE: JCEKS + PASSWORDS_AS_ENV_VARS: "true" <% } %> <% if (replication) { %> + ENABLE_MASTER: <% if (replication == 'master-master') { %>"true"<% } else { %>"false"<% } %> + ENABLE_SLAVE: <% if (replication == 'master-master') { %>"false"<% } else { %>"true"<% } %> + MASTER_HOST: solr6 <% } %> <% if (sharding == 'true') { %> + ENABLE_SHARDING: "true" + NUM_SHARDS: "2" + SHARD_ID: "1" <% if (shardingMethod == 'DB_ID_RANGE') { %> + SHARD_RANGE: "801-40000" <% } %> <% if (shardingMethod == 'DATE') { %> + SHARD_KEY: "cm:created" + SHARD_DATE_GROUPING: "2" <% } %> <% if (shardingMethod == 'PROPERTY') { %> + SHARD_KEY: "shard:shardId" <% } %> <% if (shardingMethod == 'EXPLICIT_ID' || shardingMethod == 'EXPLICIT_ID_FALLBACK_LRIS') { %> + SHARD_KEY: "shard:shardId" <% } %> + SHARDING_METHOD: <%=shardingMethod%> <% } %> + COMPRESS_CONTENT: "<%=gzip%>" + mem_limit: 1200m + environment: + #Solr needs to know how to register itself with Alfresco + SOLR_ALFRESCO_HOST: "alfresco" + SOLR_ALFRESCO_PORT: "<%=alfrescoPort%>" + ALFRESCO_SECURE_COMMS: <%=secureComms%> + #Alfresco needs to know how to call solr + SOLR_SOLR_HOST: "solr6secondary" + SOLR_SOLR_PORT: "8983" + #Create the default alfresco and archive cores + SOLR_CREATE_ALFRESCO_DEFAULTS: "alfresco,archive" + SOLR_JAVA_MEM: "-Xms1g -Xmx1g" <% if (httpMode == 'https') { %> + SOLR_SSL_TRUST_STORE: "/opt/<%=searchPath%>/keystore/ssl-repo-client.truststore" + SOLR_SSL_TRUST_STORE_PASSWORD: "kT9X6oe68t" + SOLR_SSL_TRUST_STORE_TYPE: "JCEKS" + SOLR_SSL_KEY_STORE: "/opt/<%=searchPath%>/keystore/ssl-repo-client.keystore" + SOLR_SSL_KEY_STORE_PASSWORD: "kT9X6oe68t" + SOLR_SSL_KEY_STORE_TYPE: "JCEKS" + SOLR_SSL_NEED_CLIENT_AUTH: "true" <% if (sharding == 'true') { %> + SOLR_SSL_CLIENT_KEY_STORE: "/opt/<%=searchPath%>/keystore/ssl-repo-client.keystore" + SOLR_SSL_CLIENT_KEY_STORE_PASSWORD: "kT9X6oe68t" + SOLR_SSL_CLIENT_KEY_STORE_TYPE: "JCEKS" + SOLR_SSL_CLIENT_TRUST_STORE: "/opt/<%=searchPath%>/keystore/ssl-repo-client.keystore" + SOLR_SSL_CLIENT_TRUST_STORE_PASSWORD: "kT9X6oe68t" + SOLR_SSL_CLIENT_TRUST_STORE_TYPE: "JCEKS" <% } %> + SOLR_OPTS: " + -Dsolr.ssl.checkPeerName=false + -Dsolr.allow.unsafe.resourceloading=true + -Dssl-keystore.password=kT9X6oe68t + -Dssl-keystore.aliases=ssl-alfresco-ca,ssl-repo-client + -Dssl-keystore.ssl-alfresco-ca.password=kT9X6oe68t + -Dssl-keystore.ssl-repo-client.password=kT9X6oe68t + -Dssl-truststore.password=kT9X6oe68t + -Dssl-truststore.aliases=ssl-alfresco-ca,ssl-repo,ssl-repo-client + -Dssl-truststore.ssl-alfresco-ca.password=kT9X6oe68t + -Dssl-truststore.ssl-repo.password=kT9X6oe68t + -Dssl-truststore.ssl-repo-client.password=kT9X6oe68t + " <% } %> + ports: + - 8084:8983 <% if (httpMode == 'https') { %> + volumes: + - ./keystores/solr:/opt/<%=searchPath%>/keystore <% } %> + <% } %> + + <% if (zeppelin == 'true') { %> + zeppelin: + build: + context: ./zeppelin + args: + ZEPPELIN_TAG: ${ZEPPELIN_TAG} + ALFRESCO_COMMS: <%=secureComms%> + mem_limit: 768m + environment: <% if (httpMode == 'https') { %> + REPO_PROTOCOL: "https" <% } %> + REPO_HOST: "alfresco" + REPO_PORT: "<%=alfrescoPort%>" <% if (httpMode == 'https') { %> + JAVA_OPTS: " + -Dalfresco.enable.ssl=true + -Dsolr.ssl.checkPeerName=false + -Djavax.net.ssl.keyStore=/zeppelin/keystore/ssl.repo.client.keystore + -Djavax.net.ssl.keyStorePassword=kT9X6oe68t + -Djavax.net.ssl.keyStoreType=JCEKS + -Djavax.net.ssl.trustStore=/zeppelin/keystore/ssl.repo.client.truststore + -Djavax.net.ssl.trustStorePassword=kT9X6oe68t + -Djavax.net.ssl.trustStoreType=JCEKS + " <% } %> + ports: + - 9090:9090 <% if (httpMode == 'https') { %> + volumes: + - ./keystores/zeppelin:/zeppelin/keystore <% } %> + <% } %> + + transform-router: + mem_limit: 512m + image: quay.io/alfresco/alfresco-transform-router:${TRANSFORM_ROUTER_TAG} + environment: + JAVA_OPTS: " -Xms256m -Xmx512m" + ACTIVEMQ_URL: "nio://activemq:61616" + CORE_AIO_URL : "http://transform-core-aio:8090" + FILE_STORE_URL: "http://shared-file-store:8099/alfresco/api/-default-/private/sfs/versions/1/file" + ports: + - 8095:8095 + links: + - activemq + + transform-core-aio: + image: alfresco/alfresco-transform-core-aio:${TRANSFORM_CORE_AIO_TAG} + mem_limit: 1536m + environment: + JAVA_OPTS: " -Xms256m -Xmx1536m" + ACTIVEMQ_URL: "nio://activemq:61616" + FILE_STORE_URL: "http://shared-file-store:8099/alfresco/api/-default-/private/sfs/versions/1/file" + ports: + - 8090:8090 + links: + - activemq + + shared-file-store: + image: alfresco/alfresco-shared-file-store:${SHARED_FILE_STORE_TAG} + mem_limit: 512m + environment: + JAVA_OPTS: " -Xms256m -Xmx512m" + scheduler.content.age.millis: 86400000 + scheduler.cleanup.interval: 86400000 + ports: + - 8099:8099 + volumes: + - shared-file-store-volume:/tmp/Alfresco/sfs + + share: + build: + context: ./share + args: + SHARE_TAG: ${SHARE_TAG} + mem_limit: 1g + environment: + REPO_HOST: "alfresco" + REPO_PORT: "8080" + JAVA_OPTS: " + -Xms500m + -Xmx500m + -Dalfresco.host=localhost + -Dalfresco.port=8080 + -Dalfresco.context=alfresco + -Dalfresco.protocol=http + " + + postgres: + image: postgres:${POSTGRES_TAG} + mem_limit: 512m + environment: + - POSTGRES_PASSWORD=alfresco + - POSTGRES_USER=alfresco + - POSTGRES_DB=alfresco + command: postgres -c max_connections=300 -c log_min_messages=LOG + ports: + - 5432:5432 + + activemq: + image: alfresco/alfresco-activemq:${ACTIVE_MQ_TAG} + mem_limit: 1g + ports: + - 8161:8161 # Web Console + - 5672:5672 # AMQP + - 61616:61616 # OpenWire + - 61613:61613 # STOMP + + content-app: + image: quay.io/alfresco/alfresco-digital-workspace:${DIGITAL_WORKSPACE_TAG} + mem_limit: 128m + environment: + BASEPATH: ./ + + # HTTP proxy to provide HTTP Default port access to services + # SOLR API and SOLR Web Console are protected to avoid unauthenticated access + proxy: + image: nginx:stable-alpine + mem_limit: 128m + depends_on: + - alfresco + volumes: + - ./config/nginx.conf:/etc/nginx/nginx.conf + - ./config/nginx.htpasswd:/etc/nginx/conf.d/nginx.htpasswd <% if (httpWebMode == 'https') { %> + - ./config/cert/localhost.cer:/etc/nginx/localhost.cer + - ./config/cert/localhost.key:/etc/nginx/localhost.key <% } %> + ports: + - <%=port%>:<%=port%> + links: + - alfresco + - share + - solr6 + - content-app + +volumes: + shared-file-store-volume: + driver_opts: + type: tmpfs + device: tmpfs From b530ef731cb095a245d025bef46020b2a5690a7f Mon Sep 17 00:00:00 2001 From: Elia Porciani Date: Wed, 3 Jun 2020 12:29:08 +0100 Subject: [PATCH 3/5] [SEARCH-2278] Modified DistributedAlfrescoSolrTrackerRace in order to test an hole of more than an hour between transactions --- .../solr/AlfrescoCoreAdminHandler.java | 4 +- .../alfresco/solr/HandlerReportHelper.java | 4 +- .../org/alfresco/solr/tracker/AclTracker.java | 226 ++++++----- .../alfresco/solr/tracker/CascadeTracker.java | 4 +- .../alfresco/solr/tracker/ContentTracker.java | 2 +- .../solr/tracker/MetadataTracker.java | 355 ++++++++---------- .../org/alfresco/solr/AlfrescoSolrUtils.java | 27 +- .../org/alfresco/solr/SolrITInitializer.java | 12 +- .../DistributedAlfrescoSolrTrackerRaceIT.java | 56 +-- ...rescoTrackerWithDelayedTransactionsIT.java | 169 +++++++++ .../tracker/DistributedCascadeTrackerIT.java | 6 +- 11 files changed, 506 insertions(+), 359 deletions(-) create mode 100644 search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedAlfrescoTrackerWithDelayedTransactionsIT.java diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoCoreAdminHandler.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoCoreAdminHandler.java index 846b364ca..18ccb6d6f 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoCoreAdminHandler.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoCoreAdminHandler.java @@ -1621,7 +1621,7 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler { // Gets Metadata health and fixes any problems MetadataTracker metadataTracker = trackerRegistry.getTrackerForCore(coreName, MetadataTracker.class); - IndexHealthReport indexHealthReport = metadataTracker.checkIndex(null, null, null, null); + IndexHealthReport indexHealthReport = metadataTracker.checkIndex(null, null, null); IOpenBitSet toReindex = indexHealthReport.getTxInIndexButNotInDb(); toReindex.or(indexHealthReport.getDuplicatedTxInIndex()); toReindex.or(indexHealthReport.getMissingTxFromIndex()); @@ -1636,7 +1636,7 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler // Gets the Acl health and fixes any problems AclTracker aclTracker = trackerRegistry.getTrackerForCore(coreName, AclTracker.class); - indexHealthReport = aclTracker.checkIndex(null, null, null, null); + indexHealthReport = aclTracker.checkIndex(null, null, null); toReindex = indexHealthReport.getAclTxInIndexButNotInDb(); toReindex.or(indexHealthReport.getDuplicatedAclTxInIndex()); toReindex.or(indexHealthReport.getMissingAclTxFromIndex()); diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/HandlerReportHelper.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/HandlerReportHelper.java index a6738ba79..43d402de5 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/HandlerReportHelper.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/HandlerReportHelper.java @@ -157,7 +157,7 @@ class HandlerReportHelper { // ACL AclTracker aclTracker = trackerRegistry.getTrackerForCore(coreName, AclTracker.class); - IndexHealthReport aclReport = aclTracker.checkIndex(toTx, toAclTx, fromTime, toTime); + IndexHealthReport aclReport = aclTracker.checkIndex(toAclTx, fromTime, toTime); NamedList ihr = new SimpleOrderedMap<>(); ihr.add("DB acl transaction count", aclReport.getDbAclTransactionCount()); ihr.add("Count of duplicated acl transactions in the index", aclReport.getDuplicatedAclTxInIndex() @@ -187,7 +187,7 @@ class HandlerReportHelper // Metadata MetadataTracker metadataTracker = trackerRegistry.getTrackerForCore(coreName, MetadataTracker.class); - IndexHealthReport metaReport = metadataTracker.checkIndex(toTx, toAclTx, fromTime, toTime); + IndexHealthReport metaReport = metadataTracker.checkIndex(toTx, fromTime, toTime); ihr.add("DB transaction count", metaReport.getDbTransactionCount()); ihr.add("Count of duplicated transactions in the index", metaReport.getDuplicatedTxInIndex() .cardinality()); diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/AclTracker.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/AclTracker.java index e039893b6..bb40da9e0 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/AclTracker.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/AclTracker.java @@ -56,8 +56,6 @@ import org.alfresco.solr.client.Acl; import org.alfresco.solr.client.AclChangeSet; import org.alfresco.solr.client.AclChangeSets; import org.alfresco.solr.client.AclReaders; -import org.alfresco.solr.client.GetNodesParameters; -import org.alfresco.solr.client.Node; import org.alfresco.solr.client.SOLRAPIClient; import org.json.JSONException; import org.slf4j.Logger; @@ -73,21 +71,30 @@ public class AclTracker extends AbstractTracker { protected final static Logger LOGGER = LoggerFactory.getLogger(AclTracker.class); + private static final int ACL_CHANGE_SETS_FOUND_QUEUE_SIZE = 100; private static final int DEFAULT_CHANGE_SET_ACLS_BATCH_SIZE = 2000; private static final int DEFAULT_ACL_BATCH_SIZE = 100; private static final int DEFAULT_ACL_TRACKER_MAX_PARALLELISM = 32; + private static final long DEFAULT_ACL_TRACKER_TIMESTEP = TIME_STEP_1_HR_IN_MS; + + private static final long INITIAL_MAX_ACL_CHANGE_SET_ID = 2000L; + private static final int MAX_NUMBER_OF_ACL_CHANGE_SETS = 2000; + + private static final long MAX_TIME_STEP = TIME_STEP_32_DAYS_IN_MS; private int aclTrackerParallelism; - private int changeSetAclsBatchSize = DEFAULT_CHANGE_SET_ACLS_BATCH_SIZE; - private int aclBatchSize = DEFAULT_ACL_BATCH_SIZE; + private int changeSetAclsBatchSize; + private int aclBatchSize; + private long timeStep; + private int maxNumberOfAclChangeSets; - private ConcurrentLinkedQueue aclChangeSetsToReindex = new ConcurrentLinkedQueue(); - private ConcurrentLinkedQueue aclChangeSetsToIndex = new ConcurrentLinkedQueue(); - private ConcurrentLinkedQueue aclChangeSetsToPurge = new ConcurrentLinkedQueue(); - private ConcurrentLinkedQueue aclsToReindex = new ConcurrentLinkedQueue(); - private ConcurrentLinkedQueue aclsToIndex = new ConcurrentLinkedQueue(); - private ConcurrentLinkedQueue aclsToPurge = new ConcurrentLinkedQueue(); + private ConcurrentLinkedQueue aclChangeSetsToReindex = new ConcurrentLinkedQueue<>(); + private ConcurrentLinkedQueue aclChangeSetsToIndex = new ConcurrentLinkedQueue<>(); + private ConcurrentLinkedQueue aclChangeSetsToPurge = new ConcurrentLinkedQueue<>(); + private ConcurrentLinkedQueue aclsToReindex = new ConcurrentLinkedQueue<>(); + private ConcurrentLinkedQueue aclsToIndex = new ConcurrentLinkedQueue<>(); + private ConcurrentLinkedQueue aclsToPurge = new ConcurrentLinkedQueue<>(); private DocRouter docRouter; private ForkJoinPool forkJoinPool; @@ -121,14 +128,21 @@ public class AclTracker extends AbstractTracker super(p, client, coreName, informationServer, Tracker.Type.ACL); changeSetAclsBatchSize = Integer.parseInt(p.getProperty("alfresco.changeSetAclsBatchSize", String.valueOf(DEFAULT_CHANGE_SET_ACLS_BATCH_SIZE))); - aclBatchSize = Integer.parseInt(p.getProperty("alfresco.aclBatchSize", String.valueOf(DEFAULT_ACL_BATCH_SIZE))); + aclBatchSize = Integer.parseInt(p.getProperty("alfresco.aclBatchSize", + String.valueOf(DEFAULT_ACL_BATCH_SIZE))); shardMethod = p.getProperty("shard.method", SHARD_METHOD_DBID); docRouter = DocRouterFactory.getRouter(p, ShardMethodEnum.getShardMethod(shardMethod)); - aclTrackerParallelism = Integer.parseInt(p.getProperty("alfresco.aclTrackerMaxParallelism", + aclTrackerParallelism = Integer.parseInt(p.getProperty("alfresco.acl.tracker.maxParallelism", String.valueOf(DEFAULT_ACL_TRACKER_MAX_PARALLELISM))); forkJoinPool = new ForkJoinPool(aclTrackerParallelism); + timeStep = Long.parseLong(p.getProperty("alfresco.acl.tracker.timestep", + String.valueOf(DEFAULT_ACL_TRACKER_TIMESTEP))); + + maxNumberOfAclChangeSets = Integer.parseInt(p.getProperty("alfresco.acl.tracker.maxNumberOfAclChangeSets", + String.valueOf(MAX_NUMBER_OF_ACL_CHANGE_SETS))); + RUN_LOCK_BY_CORE.put(coreName, new Semaphore(1, true)); WRITE_LOCK_BY_CORE.put(coreName, new Semaphore(1, true)); } @@ -168,8 +182,10 @@ public class AclTracker extends AbstractTracker Long aclChangeSetId = aclChangeSetsToIndex.poll(); if (aclChangeSetId != null) { - AclChangeSets aclChangeSets = client.getAclChangeSets(null, aclChangeSetId, null, aclChangeSetId+1, 1); - if ((aclChangeSets.getAclChangeSets().size() > 0) && aclChangeSetId.equals(aclChangeSets.getAclChangeSets().get(0).getId())) + AclChangeSets aclChangeSets = client.getAclChangeSets(null, aclChangeSetId, + null, aclChangeSetId+1, 1); + if ((aclChangeSets.getAclChangeSets().size() > 0) && + aclChangeSetId.equals(aclChangeSets.getAclChangeSets().get(0).getId())) { AclChangeSet changeSet = aclChangeSets.getAclChangeSets().get(0); List acls = client.getAcls(Collections.singletonList(changeSet), null, Integer.MAX_VALUE); @@ -184,9 +200,8 @@ public class AclTracker extends AbstractTracker } else { - LOGGER.info( - "[CORE {}] - INDEX ACTION - AclChangeSetId {} was not found in database, it has NOT been reindexed", - coreName, aclChangeSetId); + LOGGER.info("[CORE {}] - INDEX ACTION - AclChangeSetId {} was not found in database, " + + "it has NOT been reindexed", coreName, aclChangeSetId); } } checkShutdown(); @@ -227,11 +242,14 @@ public class AclTracker extends AbstractTracker { this.infoSrv.deleteByAclChangeSetId(aclChangeSetId); - AclChangeSets aclChangeSets = client.getAclChangeSets(null, aclChangeSetId, null, aclChangeSetId+1, 1); - if ((aclChangeSets.getAclChangeSets().size() > 0) && aclChangeSetId.equals(aclChangeSets.getAclChangeSets().get(0).getId())) + AclChangeSets aclChangeSets = client.getAclChangeSets(null, aclChangeSetId, + null, aclChangeSetId+1, 1); + if ((aclChangeSets.getAclChangeSets().size() > 0) && + aclChangeSetId.equals(aclChangeSets.getAclChangeSets().get(0).getId())) { AclChangeSet changeSet = aclChangeSets.getAclChangeSets().get(0); - List acls = client.getAcls(Collections.singletonList(changeSet), null, Integer.MAX_VALUE); + List acls = client.getAcls(Collections.singletonList(changeSet), + null, Integer.MAX_VALUE); for (Acl acl : acls) { List readers = client.getAclReaders(Collections.singletonList(acl)); @@ -239,14 +257,14 @@ public class AclTracker extends AbstractTracker } this.infoSrv.indexAclTransaction(changeSet, true); - LOGGER.info("[CORE {}] - REINDEX ACTION - AclChangeSetId {} has been reindexed", coreName, aclChangeSetId); + LOGGER.info("[CORE {}] - REINDEX ACTION - AclChangeSetId {} has been reindexed", + coreName, aclChangeSetId); requiresCommit = true; } else { - LOGGER.info( - "[CORE {}] - REINDEX ACTION - AclChangeSetId {} was not found in database, it has NOT been reindexed", - coreName, aclChangeSetId); + LOGGER.info("[CORE {}] - REINDEX ACTION - AclChangeSetId {} was not found in database, " + + "it has NOT been reindexed", coreName, aclChangeSetId); } } checkShutdown(); @@ -295,7 +313,6 @@ public class AclTracker extends AbstractTracker } checkShutdown(); } - } protected void purgeAcls() throws IOException, JSONException @@ -376,7 +393,8 @@ public class AclTracker extends AbstractTracker state.setCheckedFirstAclTransactionTime(true); LOGGER.info("[CORE {}] - No acl transactions found - no verification required", coreName); - firstChangeSets = client.getAclChangeSets(null, 0L, null, 2000L, 1); + firstChangeSets = client.getAclChangeSets(null, 0L, + null, INITIAL_MAX_ACL_CHANGE_SET_ID, 1); if (!firstChangeSets.getAclChangeSets().isEmpty()) { AclChangeSet firstChangeSet = firstChangeSets.getAclChangeSets().get(0); @@ -388,19 +406,23 @@ public class AclTracker extends AbstractTracker if (!state.isCheckedFirstAclTransactionTime()) { - firstChangeSets = client.getAclChangeSets(null, 0L, null, 2000L, 1); + firstChangeSets = client.getAclChangeSets(null, 0L, + null, INITIAL_MAX_ACL_CHANGE_SET_ID, 1); if (!firstChangeSets.getAclChangeSets().isEmpty()) { AclChangeSet firstAclChangeSet= firstChangeSets.getAclChangeSets().get(0); long firstAclTxId = firstAclChangeSet.getId(); long firstAclTxCommitTime = firstAclChangeSet.getCommitTimeMs(); - int setSize = this.infoSrv.getAclTxDocsSize(""+firstAclTxId, ""+firstAclTxCommitTime); + int setSize = this.infoSrv.getAclTxDocsSize(Long.toString(firstAclTxId), + Long.toString(firstAclTxCommitTime)); if (setSize == 0) { LOGGER.error("[CORE {}] First acl transaction was not found with the correct timestamp.", coreName); - LOGGER.error("SOLR has successfully connected to your repository however the SOLR indexes and repository database do not match."); - LOGGER.error("If this is a new or rebuilt database your SOLR indexes also need to be re-built to match the database."); + LOGGER.error("SOLR has successfully connected to your repository " + + "however the SOLR indexes and repository database do not match."); + LOGGER.error("If this is a new or rebuilt database your SOLR indexes " + + "also need to be re-built to match the database."); LOGGER.error("You can also check your SOLR connection details in solrcore.properties."); throw new AlfrescoRuntimeException("Initial acl transaction not found with correct timestamp"); } @@ -421,7 +443,8 @@ public class AclTracker extends AbstractTracker { if (firstChangeSets == null) { - firstChangeSets = client.getAclChangeSets(null, 0L, null, 2000L, 1); + firstChangeSets = client.getAclChangeSets(null, 0L, + null, INITIAL_MAX_ACL_CHANGE_SET_ID, 1); } setLastChangeSetIdAndCommitTimeInTrackerState(firstChangeSets.getAclChangeSets(), state); @@ -432,19 +455,23 @@ public class AclTracker extends AbstractTracker AclChangeSet maxAclTxInIndex = this.infoSrv.getMaxAclChangeSetIdAndCommitTimeInIndex(); if (maxAclTxInIndex.getCommitTimeMs() > maxChangeSetCommitTimeInRepo) { - LOGGER.error("[CORE {}] Last acl transaction was found in index with timestamp later than that of repository.", coreName); + LOGGER.error("[CORE {}] Last acl transaction was found in index with " + + "timestamp later than that of repository.", coreName); LOGGER.error("Max Acl Tx In Index: " + maxAclTxInIndex.getId() + ", In Repo: " + maxChangeSetIdInRepo); LOGGER.error("Max Acl Tx Commit Time In Index: " + maxAclTxInIndex.getCommitTimeMs() + ", In Repo: " + maxChangeSetCommitTimeInRepo); - LOGGER.error("SOLR has successfully connected to your repository however the SOLR indexes and repository database do not match."); - LOGGER.error("If this is a new or rebuilt database your SOLR indexes also need to be re-built to match the database."); + LOGGER.error("SOLR has successfully connected to your repository" + + " however the SOLR indexes and repository database do not match."); + LOGGER.error("If this is a new or rebuilt database your SOLR indexes " + + "also need to be re-built to match the database."); LOGGER.error("You can also check your SOLR connection details in solrcore.properties."); throw new AlfrescoRuntimeException("Last acl transaction found in index with incorrect timestamp"); } else { state.setCheckedLastAclTransactionTime(true); - LOGGER.info("[CORE {}] - Verified last acl transaction timestamp in index less than or equal to that of repository.", coreName); + LOGGER.info("[CORE {}] - Verified last acl transaction timestamp in index less" + + " than or equal to that of repository.", coreName); } } } @@ -467,7 +494,9 @@ public class AclTracker extends AbstractTracker } } - protected AclChangeSets getSomeAclChangeSets(BoundedDeque changeSetsFound, Long fromCommitTime, long timeStep, int maxResults, long endTime) throws AuthenticationException, IOException, JSONException + protected AclChangeSets getSomeAclChangeSets(BoundedDeque changeSetsFound, Long fromCommitTime, + long timeStep, int maxResults, long endTime) + throws AuthenticationException, IOException, JSONException { long actualTimeStep = timeStep; @@ -477,15 +506,17 @@ public class AclTracker extends AbstractTracker Long startTime = fromCommitTime == null ? Long.valueOf(0L) : fromCommitTime; do { - aclChangeSets = client.getAclChangeSets(startTime, null, startTime + actualTimeStep, null, maxResults); + aclChangeSets = client.getAclChangeSets(startTime, null, + startTime + actualTimeStep, null, maxResults); startTime += actualTimeStep; actualTimeStep *= 2; - if(actualTimeStep > TIME_STEP_32_DAYS_IN_MS) + if(actualTimeStep > MAX_TIME_STEP) { - actualTimeStep = TIME_STEP_32_DAYS_IN_MS; + actualTimeStep = MAX_TIME_STEP; } } - while( ((aclChangeSets.getAclChangeSets().size() == 0) && (startTime < endTime)) || ((aclChangeSets.getAclChangeSets().size() > 0) && alreadyFoundChangeSets(changeSetsFound, aclChangeSets))); + while( ((aclChangeSets.getAclChangeSets().size() == 0) && (startTime < endTime)) || + ((aclChangeSets.getAclChangeSets().size() > 0) && alreadyFoundChangeSets(changeSetsFound, aclChangeSets))); return aclChangeSets; @@ -526,12 +557,13 @@ public class AclTracker extends AbstractTracker trackerStats.addAclTime(time); } - public IndexHealthReport checkIndex(Long toTx, Long toAclTx, Long fromTime, Long toTime) + public IndexHealthReport checkIndex(Long toAclTx, Long fromTime, Long toTime) throws AuthenticationException, IOException, JSONException { // DB ACL TX Count long firstChangeSetCommitTimex = 0; - AclChangeSets firstChangeSets = client.getAclChangeSets(null, 0L, null, 2000L, 1); + AclChangeSets firstChangeSets = client.getAclChangeSets(null, 0L, + null, INITIAL_MAX_ACL_CHANGE_SET_ID, 1); if(firstChangeSets.getAclChangeSets().size() > 0) { AclChangeSet firstChangeSet = firstChangeSets.getAclChangeSets().get(0); @@ -549,11 +581,11 @@ public class AclTracker extends AbstractTracker Long minAclTxId = null; long endTime = System.currentTimeMillis() + infoSrv.getHoleRetention(); AclChangeSets aclTransactions; - BoundedDeque changeSetsFound = new BoundedDeque<>(100); + BoundedDeque changeSetsFound = new BoundedDeque<>(ACL_CHANGE_SETS_FOUND_QUEUE_SIZE); DO: do { - aclTransactions = getSomeAclChangeSets(changeSetsFound, lastAclTxCommitTime, TIME_STEP_1_HR_IN_MS, 2000, - endTime); + aclTransactions = getSomeAclChangeSets(changeSetsFound, + lastAclTxCommitTime, timeStep, maxNumberOfAclChangeSets, endTime); for (AclChangeSet set : aclTransactions.getAclChangeSets()) { // include @@ -597,7 +629,8 @@ public class AclTracker extends AbstractTracker try { ArrayList answer = new ArrayList<>(); - AclChangeSets changeSet = client.getAclChangeSets(null, acltxid, null, acltxid+1, 1); + AclChangeSets changeSet = client.getAclChangeSets(null, acltxid, null, + acltxid+1, 1); List acls = client.getAcls(changeSet.getAclChangeSets(), null, Integer.MAX_VALUE); for (Acl acl : acls) { @@ -646,10 +679,9 @@ public class AclTracker extends AbstractTracker { long startElapsed = System.nanoTime(); - - boolean upToDate = false; + AclChangeSets aclChangeSets; - BoundedDeque changeSetsFound = new BoundedDeque(100); + BoundedDeque changeSetsFound = new BoundedDeque<>(ACL_CHANGE_SETS_FOUND_QUEUE_SIZE); long totalAclCount = 0; int aclCount; @@ -672,7 +704,7 @@ public class AclTracker extends AbstractTracker Long fromCommitTime = getChangeSetFromCommitTime(changeSetsFound, state.getLastChangeSetCommitTimeOnServer() == 0 ? state.getLastGoodChangeSetCommitTimeInIndex() : state.getLastChangeSetCommitTimeOnServer()); - aclChangeSets = getSomeAclChangeSets(changeSetsFound, fromCommitTime, TIME_STEP_1_HR_IN_MS, 2000, + aclChangeSets = getSomeAclChangeSets(changeSetsFound, fromCommitTime, timeStep, maxNumberOfAclChangeSets, state.getTimeToStopIndexing()); if (aclChangeSets.getAclChangeSets().size() > 0) @@ -690,53 +722,28 @@ public class AclTracker extends AbstractTracker LOGGER.info("{}-[CORE {}] No ACL change set found after lastTxCommitTime {}", Thread.currentThread().getId(), coreName, fromCommitTime); } - - // Ignore indexed ACL Change Sets - aclChangeSets = new AclChangeSets(aclChangeSets.getAclChangeSets().stream() - .filter(changeSet -> { - try - { - boolean isInIndex = (changeSet.getCommitTimeMs() <= state.getLastIndexedChangeSetCommitTime() && - infoSrv.aclChangeSetInIndex(changeSet.getId(), true)); - if (LOGGER.isTraceEnabled()) - { - LOGGER.trace("{}-[CORE {}] Skipping change Set Id {} as it was already indexed", - Thread.currentThread().getId(), coreName, changeSet.getId()); - } - return !isInIndex; - } - catch (IOException e) - { - LOGGER.warn( - "{}-[CORE {}] Error catched while checking if ACL Change Set {} was in index", - Thread.currentThread().getId(), coreName, changeSet.getId(), e); - return true; - } - }) - .collect(Collectors.toList())); - - // Make sure we do not go ahead of where we started - we will check the holes here // correctly next time - if (aclChangeSets.getAclChangeSets().stream().anyMatch(changeSet -> changeSet.getCommitTimeMs() > state.getTimeToStopIndexing())) + if (aclChangeSets.getAclChangeSets() + .stream() + .anyMatch(changeSet -> changeSet.getCommitTimeMs() > state.getTimeToStopIndexing())) { break; } final AtomicInteger counter = new AtomicInteger(); Collection> changeSetBatches = aclChangeSets.getAclChangeSets().stream() - .collect(Collectors.groupingBy(it -> counter.getAndAdd(it.getAclCount()) / changeSetAclsBatchSize)).values(); + .peek(changeSetsFound::add) + .filter(this::isAclChangeSetAlreadyIndexed) + .collect(Collectors.groupingBy(it -> counter.getAndAdd(it.getAclCount()) / changeSetAclsBatchSize)) + .values(); for (List changeSetBatch : changeSetBatches) { aclCount = indexBatchOfChangeSets(changeSetBatch); - for (AclChangeSet indexed : changeSetBatch) - { - changeSetsFound.add(indexed); - } // Update last committed transactions setLastChangeSetIdAndCommitTimeInTrackerState(changeSetBatch, state); indexAclChangeSetAfterWorker(changeSetBatch, state); @@ -757,28 +764,47 @@ public class AclTracker extends AbstractTracker } } - while ((aclChangeSets.getAclChangeSets().size() > 0) && (upToDate == false)); + while ((aclChangeSets.getAclChangeSets().size() > 0)); LOGGER.info("{}-[CORE {}] Tracked {} ACLs", Thread.currentThread().getId(), coreName, totalAclCount); } + private boolean isAclChangeSetAlreadyIndexed(AclChangeSet changeSet) + { + + try + { + boolean isInIndex = (changeSet.getCommitTimeMs() <= state.getLastIndexedChangeSetCommitTime() && + infoSrv.aclChangeSetInIndex(changeSet.getId(), true)); + if (LOGGER.isTraceEnabled()) + { + LOGGER.trace("{}-[CORE {}] Skipping change Set Id {} as it was already indexed", + Thread.currentThread().getId(), coreName, changeSet.getId()); + } + return !isInIndex; + } + catch (IOException e) + { + LOGGER.warn( + "{}-[CORE {}] Error catched while checking if ACL Change Set {} was in index", + Thread.currentThread().getId(), coreName, changeSet.getId(), e); + return true; + } + + } + private void setLastChangeSetIdAndCommitTimeInTrackerState(List aclChangeSets, TrackerState state) { if (!aclChangeSets.isEmpty()) { - Long maxChangeSetCommitTime = aclChangeSets.stream().max(Comparator.comparing(AclChangeSet::getCommitTimeMs)).get().getCommitTimeMs(); - if(maxChangeSetCommitTime != null) - { - state.setLastChangeSetCommitTimeOnServer(maxChangeSetCommitTime); - } - - Long maxChangeSetId = aclChangeSets.stream().max(Comparator.comparing(AclChangeSet::getId)).get().getId(); - if(maxChangeSetId != null) - { - state.setLastChangeSetIdOnServer(maxChangeSetId); - } + long maxChangeSetCommitTime = + aclChangeSets.stream().max(Comparator.comparing(AclChangeSet::getCommitTimeMs)).get().getCommitTimeMs(); + state.setLastChangeSetCommitTimeOnServer(maxChangeSetCommitTime); + + long maxChangeSetId = aclChangeSets.stream().max(Comparator.comparing(AclChangeSet::getId)).get().getId(); + state.setLastChangeSetIdOnServer(maxChangeSetId); } } @@ -804,15 +830,6 @@ public class AclTracker extends AbstractTracker } } - private int getAclCount(List changeSetBatch) - { - int count = 0; - for (AclChangeSet set : changeSetBatch) - { - count += set.getAclCount(); - } - return count; - } /** * Index ACLs from ACL Change Sets contained in changeSetBatch @@ -822,7 +839,8 @@ public class AclTracker extends AbstractTracker * @param changeSetBatch List of ACL Change Sets to be indexed * @return List of ACL Change Set indexed and Count of ACL indexed */ - private int indexBatchOfChangeSets(List changeSetBatch) throws AuthenticationException, IOException, JSONException, ExecutionException, InterruptedException { + private int indexBatchOfChangeSets(List changeSetBatch) + throws AuthenticationException, IOException, JSONException, ExecutionException, InterruptedException { // Exclude ACL Change Set with no ACLs inside List nonEmptyChangeSets = changeSetBatch.stream() .filter(set -> set.getAclCount() > 0) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/CascadeTracker.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/CascadeTracker.java index 7506cb2c7..96751218b 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/CascadeTracker.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/CascadeTracker.java @@ -92,10 +92,10 @@ public class CascadeTracker extends AbstractTracker implements Tracker { super(p, client, coreName, informationServer, Tracker.Type.CASCADE); - cascadeTrackerParallelism = Integer.parseInt(p.getProperty("alfresco.cascadeTrackerMaxParallelism", + cascadeTrackerParallelism = Integer.parseInt(p.getProperty("alfresco.cascade.tracker.maxParallelism", String.valueOf(DEFAULT_CASCADE_TRACKER_MAX_PARALLELISM))); - cascadeBatchSize = Integer.parseInt(p.getProperty("alfresco.cascadeNodeBatchSize", + cascadeBatchSize = Integer.parseInt(p.getProperty("alfresco.cascade.tracker.nodeBatchSize", String.valueOf(DEFAULT_CASCADE_NODE_BATCH_SIZE)));; forkJoinPool = new ForkJoinPool(cascadeTrackerParallelism); diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/ContentTracker.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/ContentTracker.java index ff2c66fd1..b260fa031 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/ContentTracker.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/ContentTracker.java @@ -83,7 +83,7 @@ public class ContentTracker extends AbstractTracker implements Tracker contentUpdateBatchSize = Integer.parseInt(p.getProperty("alfresco.contentUpdateBatchSize", String.valueOf(DEFAULT_CONTENT_UPDATE_BATCH_SIZE))); - contentTrackerParallelism = Integer.parseInt(p.getProperty("alfresco.contentTrackerMaxParallelism", + contentTrackerParallelism = Integer.parseInt(p.getProperty("alfresco.content.tracker.maxParallelism", String.valueOf(DEFAULT_CONTENT_TRACKER_MAX_PARALLELISM))); forkJoinPool = new ForkJoinPool(contentTrackerParallelism); diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java index 439b1c981..1067fdfe2 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/tracker/MetadataTracker.java @@ -49,7 +49,8 @@ import org.slf4j.LoggerFactory; import java.io.IOException; import java.util.ArrayList; -import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -59,6 +60,7 @@ import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.ExecutionException; import java.util.concurrent.ForkJoinPool; import java.util.concurrent.Semaphore; +import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; import static org.alfresco.repo.index.shard.ShardMethodEnum.DB_ID_RANGE; @@ -71,26 +73,31 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker { protected final static Logger LOGGER = LoggerFactory.getLogger(MetadataTracker.class); + private static final int METADATA_TRANSACTIONS_FOUND_QUEUE_SIZE = 100; private static final int DEFAULT_METADATA_TRACKER_MAX_PARALLELISM = 32; private static final int DEFAULT_TRANSACTION_DOCS_BATCH_SIZE = 2000; private static final int DEFAULT_MAX_NUMBER_OF_TRANSACTIONS = 2000; private static final int DEFAULT_NODE_BATCH_SIZE = 50; private static final String DEFAULT_INITIAL_TRANSACTION_RANGE = "0-2000"; + private static final long DEFAULT_METADATA_TRACKER_TIMESTEP = TIME_STEP_1_HR_IN_MS; + private static final long INITIAL_MAX_TXN_ID = 2000L; private int matadataTrackerParallelism; private int transactionDocsBatchSize; private int nodeBatchSize; private int maxNumberOfTransactions; + private long timeStep; - private ConcurrentLinkedQueue transactionsToReindex = new ConcurrentLinkedQueue<>(); - private ConcurrentLinkedQueue transactionsToIndex = new ConcurrentLinkedQueue<>(); - private ConcurrentLinkedQueue transactionsToPurge = new ConcurrentLinkedQueue<>(); - private ConcurrentLinkedQueue nodesToReindex = new ConcurrentLinkedQueue<>(); - private ConcurrentLinkedQueue nodesToIndex = new ConcurrentLinkedQueue<>(); - private ConcurrentLinkedQueue nodesToPurge = new ConcurrentLinkedQueue<>(); - private ConcurrentLinkedQueue queriesToReindex = new ConcurrentLinkedQueue<>(); + private final ConcurrentLinkedQueue transactionsToReindex = new ConcurrentLinkedQueue<>(); + private final ConcurrentLinkedQueue transactionsToIndex = new ConcurrentLinkedQueue<>(); + private final ConcurrentLinkedQueue transactionsToPurge = new ConcurrentLinkedQueue<>(); + private final ConcurrentLinkedQueue nodesToReindex = new ConcurrentLinkedQueue<>(); + private final ConcurrentLinkedQueue nodesToIndex = new ConcurrentLinkedQueue<>(); + private final ConcurrentLinkedQueue nodesToPurge = new ConcurrentLinkedQueue<>(); + private final ConcurrentLinkedQueue queriesToReindex = new ConcurrentLinkedQueue<>(); - private final boolean isRunningInProduction = !Boolean.parseBoolean(System.getProperty("alfresco.test", "false")); + private final boolean isRunningInProduction = + !Boolean.parseBoolean(System.getProperty("alfresco.test", "false")); private ForkJoinPool forkJoinPool; @@ -160,11 +167,18 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker String.valueOf(DEFAULT_TRANSACTION_DOCS_BATCH_SIZE))); nodeBatchSize = Integer.parseInt(p.getProperty("alfresco.nodeBatchSize", String.valueOf(DEFAULT_NODE_BATCH_SIZE))); - maxNumberOfTransactions = Integer.parseInt(p.getProperty("alfresco.maxNumberOfTransactions", String.valueOf(DEFAULT_MAX_NUMBER_OF_TRANSACTIONS))); - matadataTrackerParallelism = Integer.parseInt(p.getProperty("alfresco.metadataTrackerMaxParallelism", + maxNumberOfTransactions = Integer.parseInt(p.getProperty("alfresco.metadata.tracker.maxNumberOfTransactions", + String.valueOf(DEFAULT_MAX_NUMBER_OF_TRANSACTIONS))); + matadataTrackerParallelism = Integer.parseInt(p.getProperty("alfresco.metadata.tracker.maxParallelism", String.valueOf(DEFAULT_METADATA_TRACKER_MAX_PARALLELISM))); - String[] minTxninitialRangeString = p.getProperty("solr.initial.transaction.range", DEFAULT_INITIAL_TRANSACTION_RANGE).split("-"); + timeStep = Long.parseLong(p.getProperty("alfresco.metadata.tracker.timestep", + String.valueOf(DEFAULT_METADATA_TRACKER_TIMESTEP))); + + String[] minTxninitialRangeString = + p.getProperty("solr.initial.transaction.range", DEFAULT_INITIAL_TRANSACTION_RANGE) + .split("-"); + cascadeTrackerEnabled = informationServer.cascadeTrackingEnabled(); minTxnIdRange = new Pair<>(Long.valueOf(minTxninitialRangeString[0]), Long.valueOf(minTxninitialRangeString[1])); forkJoinPool = new ForkJoinPool(matadataTrackerParallelism); @@ -179,12 +193,13 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker // Try invoking getNextTxCommitTime service try { - client.getNextTxCommitTime(coreName, 0l); + client.getNextTxCommitTime(coreName, 0L); nextTxCommitTimeServiceAvailable = true; } catch (NoSuchMethodException e) { - LOGGER.warn("nextTxCommitTimeService is not available. Upgrade your ACS Repository version in order to use this feature: {} ", e.getMessage()); + LOGGER.warn("nextTxCommitTimeService is not available. " + + "Upgrade your ACS Repository version in order to use this feature: {} ", e.getMessage()); } catch (Exception e) { @@ -196,12 +211,13 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker { try { - client.getTxIntervalCommitTime(coreName, 0l, 0l); + client.getTxIntervalCommitTime(coreName, 0L, 0L); txIntervalCommitTimeServiceAvailable = true; } catch (NoSuchMethodException e) { - LOGGER.warn("txIntervalCommitTimeServiceAvailable is not available. Upgrade your ACS Repository version " + + LOGGER.warn("txIntervalCommitTimeServiceAvailable is not available. " + + "Upgrade your ACS Repository version " + "to use this feature with DB_ID_RANGE sharding: {} ", e.getMessage()); } catch (Exception e) @@ -219,8 +235,8 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker } @Override - protected void doTrack(String iterationId) throws AuthenticationException, IOException, JSONException, EncoderException - { + protected void doTrack(String iterationId) + throws AuthenticationException, IOException, JSONException { // MetadataTracker must wait until ModelTracker has run ModelTracker modelTracker = this.infoSrv.getAdminHandler().getTrackerRegistry().getModelTracker(); if (modelTracker != null && modelTracker.hasModels()) @@ -255,7 +271,7 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker queriesToReindex.size() > 0; } - private void trackRepository() throws IOException, AuthenticationException, JSONException, EncoderException + private void trackRepository() throws IOException, AuthenticationException, JSONException { checkShutdown(); @@ -303,7 +319,8 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker state.setCheckedFirstTransactionTime(true); LOGGER.info("No transactions found - no verification required"); - firstTransactions = client.getTransactions(null, minTxnIdRange.getFirst(), null, minTxnIdRange.getSecond(), 1); + firstTransactions = client.getTransactions(null, minTxnIdRange.getFirst(), + null, minTxnIdRange.getSecond(), 1); if (!firstTransactions.getTransactions().isEmpty()) { Transaction firstTransaction = firstTransactions.getTransactions().get(0); @@ -321,7 +338,7 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker // are skipped if they are not related with the range of the Shard. // Getting the minCommitTime for the Shard is enough in order to check // that the first transaction is present. - long minCommitTime = 0l; + long minCommitTime = 0L; if (docRouter instanceof DBIDRangeRouter && txIntervalCommitTimeServiceAvailable) { try @@ -333,28 +350,34 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker } catch (NoSuchMethodException e) { - LOGGER.warn("txIntervalCommitTimeServiceAvailable is not available. If you are using DB_ID_RANGE shard method, " - + "upgrade your ACS Repository version in order to use the skip transactions feature: {} ", e.getMessage()); + LOGGER.warn("txIntervalCommitTimeServiceAvailable is not available." + + " If you are using DB_ID_RANGE shard method, " + + "upgrade your ACS Repository version in order to use the skip transactions feature: {} ", + e.getMessage()); } } // When a Shard with DB_ID_RANGE method is empty, minCommitTime is -1. // No firstTransaction checking is required for this case. - if (minCommitTime != -1l) { + if (minCommitTime != -1L) { - firstTransactions = client.getTransactions(minCommitTime, 0L, null, 2000l, 1); + firstTransactions = client.getTransactions(minCommitTime, 0L, + null, INITIAL_MAX_TXN_ID, 1); if (!firstTransactions.getTransactions().isEmpty()) { Transaction firstTransaction = firstTransactions.getTransactions().get(0); long firstTxId = firstTransaction.getId(); long firstTransactionCommitTime = firstTransaction.getCommitTimeMs(); - int setSize = this.infoSrv.getTxDocsSize(""+firstTxId, ""+firstTransactionCommitTime); + int setSize = this.infoSrv.getTxDocsSize(Long.toString(firstTxId), + Long.toString(firstTransactionCommitTime)); if (setSize == 0) { LOGGER.error("First transaction was not found with the correct timestamp."); - LOGGER.error("SOLR has successfully connected to your repository however the SOLR indexes and repository database do not match."); - LOGGER.error("If this is a new or rebuilt database your SOLR indexes also need to be re-built to match the database."); + LOGGER.error("SOLR has successfully connected to your repository however the SOLR indexes" + + " and repository database do not match."); + LOGGER.error("If this is a new or rebuilt database your SOLR indexes also need to be " + + "re-built to match the database."); LOGGER.error("You can also check your SOLR connection details in solrcore.properties."); throw new AlfrescoRuntimeException("Initial transaction not found with correct timestamp"); } @@ -376,7 +399,8 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker { if (firstTransactions == null) { - firstTransactions = client.getTransactions(null, minTxnIdRange.getFirst(), null, minTxnIdRange.getSecond(), 1); + firstTransactions = client.getTransactions(null, minTxnIdRange.getFirst(), + null, minTxnIdRange.getSecond(), 1); } setLastTxCommitTimeAndTxIdInTrackerState(firstTransactions); @@ -391,8 +415,10 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker LOGGER.error("Max Tx In Index: " + maxTxInIndex.getId() + ", In Repo: " + maxTxnIdInRepo); LOGGER.error("Max Tx Commit Time In Index: " + maxTxInIndex.getCommitTimeMs() + ", In Repo: " + maxTxnCommitTimeInRepo); - LOGGER.error("SOLR has successfully connected to your repository however the SOLR indexes and repository database do not match."); - LOGGER.error("If this is a new or rebuilt database your SOLR indexes also need to be re-built to match the database."); + LOGGER.error("SOLR has successfully connected to your repository however the SOLR indexes" + + " and repository database do not match."); + LOGGER.error("If this is a new or rebuilt database your SOLR indexes also need to " + + "be re-built to match the database."); LOGGER.error("You can also check your SOLR connection details in solrcore.properties."); throw new AlfrescoRuntimeException("Last transaction found in index with incorrect timestamp"); } @@ -416,20 +442,22 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker Long transactionId = transactionsToIndex.poll(); if (transactionId != null) { - Transactions transactions = client.getTransactions(null, transactionId, null, transactionId+1, 1); - if ((transactions.getTransactions().size() > 0) && (transactionId.equals(transactions.getTransactions().get(0).getId()))) + Transactions transactions = client.getTransactions(null, transactionId, + null, transactionId + 1, 1); + if ((transactions.getTransactions().size() > 0) && + (transactionId.equals(transactions.getTransactions().get(0).getId()))) { Transaction info = transactions.getTransactions().get(0); GetNodesParameters gnp = new GetNodesParameters(); - ArrayList txs = new ArrayList(); + ArrayList txs = new ArrayList<>(); txs.add(info.getId()); gnp.setTransactionIds(txs); gnp.setStoreProtocol(storeRef.getProtocol()); gnp.setStoreIdentifier(storeRef.getIdentifier()); updateShardProperty(); - shardProperty.ifPresent(p -> gnp.setShardProperty(p)); + shardProperty.ifPresent(gnp::setShardProperty); gnp.setCoreName(coreName); @@ -455,27 +483,14 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker } else { - LOGGER.info("INDEX ACTION - Transaction {} was not found in database, it has NOT been reindexed", transactionId); - } - } - - if (docCount > batchCount) - { - if(this.infoSrv.getRegisteredSearcherCount() < getMaxLiveSearchers()) - { - checkShutdown(); - long endElapsed = System.nanoTime(); - trackerStats.addElapsedNodeTime(docCount, endElapsed-startElapsed); - startElapsed = endElapsed; - docCount = 0; - requiresCommit = false; + LOGGER.info("INDEX ACTION - Transaction {} was not found in database, it has NOT been reindexed", + transactionId); } } } - if (requiresCommit || (docCount > 0)) + if (requiresCommit) { checkShutdown(); - //this.infoSrv.commit(); long endElapsed = System.nanoTime(); trackerStats.addElapsedNodeTime(docCount, endElapsed - startElapsed); } @@ -520,13 +535,15 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker // make sure it is cleaned out so we do not miss deletes this.infoSrv.deleteByTransactionId(transactionId); - Transactions transactions = client.getTransactions(null, transactionId, null, transactionId+1, 1); - if ((transactions.getTransactions().size() > 0) && (transactionId.equals(transactions.getTransactions().get(0).getId()))) + Transactions transactions = client.getTransactions(null, transactionId, + null, transactionId+1, 1); + if ((transactions.getTransactions().size() > 0) && + (transactionId.equals(transactions.getTransactions().get(0).getId()))) { Transaction info = transactions.getTransactions().get(0); this.infoSrv.dirtyTransaction(info.getId()); GetNodesParameters gnp = new GetNodesParameters(); - ArrayList txs = new ArrayList(); + ArrayList txs = new ArrayList<>(); txs.add(info.getId()); gnp.setTransactionIds(txs); gnp.setStoreProtocol(storeRef.getProtocol()); @@ -550,7 +567,8 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker } else { - LOGGER.info("REINDEX ACTION - Transaction {} was not found in database, it has NOT been reindexed", transactionId); + LOGGER.info("REINDEX ACTION - Transaction {} was not found in database, it has NOT been reindexed", + transactionId); } } @@ -629,7 +647,7 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker } - private void purgeTransactions() throws IOException, AuthenticationException, JSONException + private void purgeTransactions() throws IOException, JSONException { boolean requiresCommit = false; while (transactionsToPurge.peek() != null) @@ -651,7 +669,7 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker } } - private void purgeNodes() throws IOException, AuthenticationException, JSONException + private void purgeNodes() throws IOException, JSONException { while (nodesToPurge.peek() != null) { @@ -715,22 +733,21 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker } protected Transactions getSomeTransactions(BoundedDeque txnsFound, Long fromCommitTime, long timeStep, - int maxResults, long endTime) throws AuthenticationException, IOException, JSONException, EncoderException, NoSuchMethodException + int maxResults, long endTime) + throws AuthenticationException, IOException, JSONException, EncoderException, NoSuchMethodException { - long actualTimeStep = timeStep; - ShardState shardstate = getShardState(); Transactions transactions; // step forward in time until we find something or hit the time bound // max id unbounded - Long startTime = fromCommitTime == null ? 0L : fromCommitTime; + long startTime = fromCommitTime == null ? 0L : fromCommitTime; if(startTime == 0) { return client.getTransactions(startTime, null, - startTime + actualTimeStep, + startTime + timeStep, null, maxResults, shardstate); @@ -738,8 +755,9 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker do { - transactions = client.getTransactions(startTime, null, startTime + actualTimeStep, null, maxResults, shardstate); - startTime += actualTimeStep; + transactions = client.getTransactions(startTime, null, startTime + timeStep, + null, maxResults, shardstate); + startTime += timeStep; // If no transactions are found, advance the time window to the next available transaction commit time if (nextTxCommitTimeServiceAvailable && transactions.getTransactions().size() == 0) @@ -749,7 +767,8 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker { LOGGER.info("{}-[CORE {}] Advancing transactions from {} to {}", Thread.currentThread().getId(), coreName, startTime, nextTxCommitTime); - transactions = client.getTransactions(nextTxCommitTime, null, nextTxCommitTime + actualTimeStep, null, maxResults, shardstate); + transactions = client.getTransactions(nextTxCommitTime, null, + nextTxCommitTime + timeStep, null, maxResults, shardstate); } } @@ -807,7 +826,7 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker fromCommitTime = shardMinCommitTime; } - Transactions transactions = getSomeTransactions(txnsFound, fromCommitTime, TIME_STEP_1_HR_IN_MS, maxNumberOfTransactions, + Transactions transactions = getSomeTransactions(txnsFound, fromCommitTime, timeStep, maxNumberOfTransactions, state.getTimeToStopIndexing()); @@ -819,7 +838,7 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker latestTransaction.setCommitTimeMs(transactions.getMaxTxnCommitTime()); latestTransaction.setId(transactions.getMaxTxnId()); transactions = new Transactions( - Arrays.asList(latestTransaction), + Collections.singletonList(latestTransaction), transactions.getMaxTxnCommitTime(), transactions.getMaxTxnId()); } @@ -827,81 +846,49 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker return transactions; } - /** - * Remove transactions already present in SOLR index - * - * @param transactions List of transactions to be indexed - * @return List of transactions not indexed in SOLR index - */ - private Transactions removeIndexedTransactions(Transactions transactions) + private boolean isTransactionIndexed(Transaction transaction) { - return new Transactions(transactions.getTransactions().stream() - .filter(transaction -> { - try - { - boolean isInIndex = (transaction.getCommitTimeMs() <= state.getLastIndexedTxCommitTime() && - infoSrv.txnInIndex(transaction.getId(), true)); - if (LOGGER.isTraceEnabled()) - { - LOGGER.trace("{}-[CORE {}] Skipping Transaction Id {} as it was already indexed", - Thread.currentThread().getId(), coreName, transaction.getId()); - } - return !isInIndex; - } - catch (IOException e) - { - LOGGER.warn( - "{}-[CORE {}] Error catched while checking if Transaction Id {} was in index", - Thread.currentThread().getId(), coreName, transaction.getId(), e); - return true; - } - }) - .collect(Collectors.toList())); - + try + { + boolean isInIndex = (transaction.getCommitTimeMs() <= state.getLastIndexedTxCommitTime() && + infoSrv.txnInIndex(transaction.getId(), true)); + if (LOGGER.isTraceEnabled()) + { + LOGGER.trace("{}-[CORE {}] Skipping Transaction Id {} as it was already indexed", + Thread.currentThread().getId(), coreName, transaction.getId()); + } + return !isInIndex; + } + catch (IOException e) + { + LOGGER.warn( + "{}-[CORE {}] Error catched while checking if Transaction Id {} was in index", + Thread.currentThread().getId(), coreName, transaction.getId(), e); + return true; + } } - //fixme remove - /** - * Keep only transactions previous to node transaction Id - * - * @param transactions List of transactions from Repository - * @param node Last Node indexed in the cycle - * @return Filtered list of transactions - */ - private Transactions filterTransactionsByNode(Transactions transactions, Node node) - { - return new Transactions(transactions.getTransactions().stream() - .filter(transaction -> { - return transaction.getId() < node.getTxnId(); - }) - .collect(Collectors.toList())); - } - /** * Indexing new transactions from repository in batches of "transactionDocsBatchSize" size. * * Additionally, the nodes inside a transaction batch are indexed in batches of "nodeBatchSize" size. * - * @throws AuthenticationException * @throws IOException * @throws JSONException - * @throws EncoderException */ - protected void trackTransactions() throws AuthenticationException, IOException, JSONException, EncoderException + protected void trackTransactions() throws IOException, JSONException { long startElapsed = System.nanoTime(); - boolean upToDate = false; Transactions transactions; - BoundedDeque txnsFound = new BoundedDeque(100); + BoundedDeque txnsFound = new BoundedDeque<>(METADATA_TRANSACTIONS_FOUND_QUEUE_SIZE); int totalUpdatedDocs = 0; LOGGER.info("{}-[CORE {}] Starting metadata tracker execution", Thread.currentThread().getId(), coreName); do { - try { /* @@ -918,7 +905,8 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker this.state = getTrackerState(); Long fromCommitTime = getTxFromCommitTime(txnsFound, - state.getLastIndexedTxCommitTime() == 0 ? state.getLastGoodTxCommitTimeInIndex() : state.getLastIndexedTxCommitTime()); + state.getLastIndexedTxCommitTime() == 0 ? state.getLastGoodTxCommitTimeInIndex() + : state.getLastIndexedTxCommitTime()); // Get transaction list to be indexed if (docRouter instanceof DBIDRangeRouter && txIntervalCommitTimeServiceAvailable) @@ -927,12 +915,10 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker } else { - transactions = getSomeTransactions(txnsFound, fromCommitTime, TIME_STEP_1_HR_IN_MS, maxNumberOfTransactions, + transactions = getSomeTransactions(txnsFound, fromCommitTime, timeStep, maxNumberOfTransactions, state.getTimeToStopIndexing()); } - // Remove transactions already indexed - transactions = removeIndexedTransactions(transactions); if (transactions.getTransactions().size() > 0) { @@ -949,48 +935,45 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker LOGGER.info("{}-[CORE {}] No transaction found after lastTxCommitTime {}", Thread.currentThread().getId(), coreName, - ((txnsFound.size() > 0) ? txnsFound.getLast().getCommitTimeMs() : state.getLastIndexedTxCommitTime())); + ((txnsFound.size() > 0) ? txnsFound.getLast().getCommitTimeMs() + : state.getLastIndexedTxCommitTime())); } - // Group the transactions in batches of transactionDocsBatchSize (or less) - List> txBatches = new ArrayList<>(); - List txBatch = new ArrayList<>(); - for (Transaction info : transactions.getTransactions()) { - - if (LOGGER.isTraceEnabled()) - { - LOGGER.trace("{}-[CORE {}] Tracking {} Transactions. Current Transaction Id to be indexed: {}", - Thread.currentThread().getId(), coreName, transactions.getTransactions().size(), info.getId()); - } - - // Make sure we do not go ahead of where we started - we will check the holes here - // correctly next time - if (info.getCommitTimeMs() > state.getTimeToStopIndexing()) { - upToDate = true; - break; - } - - txBatch.add(info); - if (getUpdateAndDeleteCount(txBatch) > transactionDocsBatchSize) { - txBatches.add(txBatch); - txBatch = new ArrayList<>(); - } - } - if (!txBatch.isEmpty()) + // Make sure we do not go ahead of where we started - we will check the holes here + // correctly next time + if (transactions.getTransactions() + .stream() + .anyMatch(transaction -> transaction.getCommitTimeMs() > state.getTimeToStopIndexing())) { - txBatches.add(txBatch); + break; } + long transaction_number = transactions.getTransactions().size(); + final AtomicInteger counter = new AtomicInteger(); + Collection> txBatches = transactions.getTransactions().stream() + .peek(txnsFound::add) + .filter(this::isTransactionIndexed) + .peek(transaction -> { + if (LOGGER.isTraceEnabled()) + { + LOGGER.trace("{}-[CORE {}] Tracking {} Transactions. Current Transaction Id to be indexed: {}", + Thread.currentThread().getId(), coreName, transaction_number, transaction.getId()); + } + }) + .collect(Collectors.groupingBy(transaction -> counter.getAndAdd( + (int) (transaction.getDeletes() + transaction.getUpdates())) / transactionDocsBatchSize)) + .values(); + // Index batches of transactions and the nodes updated or deleted within the transaction for (List batch : txBatches) { // Index nodes contained in the transactions - int docCount = indexBatchOfTransactions(batch, totalUpdatedDocs); + int docCount = indexBatchOfTransactions(batch); totalUpdatedDocs += docCount; // Add the transactions as found to avoid processing them again in the next iteration - batch.forEach(transaction -> txnsFound.add(transaction)); + batch.forEach(txnsFound::add); // Index the transactions indexTransactionsAfterWorker(batch); @@ -999,10 +982,7 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker startElapsed = endElapsed; } - - setLastTxCommitTimeAndTxIdInTrackerState(transactions); - } catch(Exception e) { @@ -1014,7 +994,7 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker } } - while ((transactions.getTransactions().size() > 0) && (upToDate == false)); + while ((transactions.getTransactions().size() > 0)); LOGGER.info("{}-[CORE {}] Tracked {} DOCs", Thread.currentThread().getId(), coreName, totalUpdatedDocs); } @@ -1062,20 +1042,6 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker txsIndexed.clear(); } - /** - * Return the number of updated and deleted nodes in a list of transactions - * @param txs List of transactions - * @return Number of updated and deleted nodes - */ - private long getUpdateAndDeleteCount(List txs) - { - long count = 0; - for (Transaction tx : txs) - { - count += (tx.getUpdates() + tx.getDeletes()); - } - return count; - } /** * Index a batch of transactions. @@ -1084,23 +1050,20 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker * the metadata of the nodes in smaller invocations to Repository * * @param txBatch Batch of transactions to be indexed - * @param indexedNodes Number of nodes indexed in this Tracker execution - * * @return Number of nodes indexed and last node indexed * * @throws AuthenticationException * @throws IOException * @throws JSONException */ - private int indexBatchOfTransactions(List txBatch, int indexedNodes) throws AuthenticationException, IOException, JSONException, ExecutionException, InterruptedException { + private int indexBatchOfTransactions(List txBatch) + throws AuthenticationException, IOException, JSONException, ExecutionException, InterruptedException { // Skip transactions without modifications (updates, deletes) - ArrayList nonEmptyTxs = new ArrayList<>(txBatch.size()); ArrayList txIds = new ArrayList<>(); for (Transaction tx : txBatch) { if (tx.getUpdates() > 0 || tx.getDeletes() > 0) { - nonEmptyTxs.add(tx); txIds.add(tx.getId()); } } @@ -1118,22 +1081,18 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker if (LOGGER.isDebugEnabled()) { - LOGGER.debug("{}-[CORE {}] Found {} Nodes to be indexed from Transactions: {}", Thread.currentThread().getId(), - coreName, nodes.size(), txIds); + LOGGER.debug("{}-[CORE {}] Found {} Nodes to be indexed from Transactions: {}", + Thread.currentThread().getId(), coreName, nodes.size(), txIds); } // Group the nodes in batches of nodeBatchSize (or less) List> nodeBatches = Lists.partition(nodes, nodeBatchSize); - Integer processedNodes = forkJoinPool.submit(() -> + return forkJoinPool.submit(() -> nodeBatches.parallelStream().map(batch -> { new NodeIndexWorker(batch, infoSrv).run(); return batch.size(); }).reduce(0, Integer::sum)).get(); - - - - return processedNodes; } @@ -1260,7 +1219,7 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker try { GetNodesParameters gnp = new GetNodesParameters(); - ArrayList txs = new ArrayList(); + ArrayList txs = new ArrayList<>(); txs.add(txid); gnp.setTransactionIds(txs); gnp.setStoreProtocol(storeRef.getProtocol()); @@ -1268,26 +1227,19 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker gnp.setCoreName(coreName); return client.getNodes(gnp, Integer.MAX_VALUE); } - catch (IOException e) - { - throw new AlfrescoRuntimeException("Failed to get nodes", e); - } - catch (JSONException e) - { - throw new AlfrescoRuntimeException("Failed to get nodes", e); - } - catch (AuthenticationException e) + catch (IOException | AuthenticationException | JSONException e) { throw new AlfrescoRuntimeException("Failed to get nodes", e); } } - public IndexHealthReport checkIndex(Long toTx, Long toAclTx, Long fromTime, Long toTime) + public IndexHealthReport checkIndex(Long toTx, Long fromTime, Long toTime) throws IOException, AuthenticationException, JSONException, EncoderException, NoSuchMethodException { // DB TX Count long firstTransactionCommitTime = 0; - Transactions firstTransactions = client.getTransactions(null, 0L, null, 2000l, 1); + Transactions firstTransactions = client.getTransactions(null, 0L, + null, INITIAL_MAX_TXN_ID, 1); if(firstTransactions.getTransactions().size() > 0) { Transaction firstTransaction = firstTransactions.getTransactions().get(0); @@ -1295,7 +1247,7 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker } IOpenBitSet txIdsInDb = infoSrv.getOpenBitSetInstance(); - Long lastTxCommitTime = Long.valueOf(firstTransactionCommitTime); + long lastTxCommitTime = firstTransactionCommitTime; if (fromTime != null) { lastTxCommitTime = fromTime; @@ -1304,24 +1256,24 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker Long minTxId = null; Transactions transactions; - BoundedDeque txnsFound = new BoundedDeque(100); + BoundedDeque txnsFound = new BoundedDeque<>(METADATA_TRANSACTIONS_FOUND_QUEUE_SIZE); long endTime = System.currentTimeMillis() + infoSrv.getHoleRetention(); DO: do { - transactions = getSomeTransactions(txnsFound, lastTxCommitTime, TIME_STEP_1_HR_IN_MS, maxNumberOfTransactions, endTime); + transactions = getSomeTransactions(txnsFound, lastTxCommitTime, timeStep, maxNumberOfTransactions, endTime); for (Transaction info : transactions.getTransactions()) { // include if (toTime != null) { - if (info.getCommitTimeMs() > toTime.longValue()) + if (info.getCommitTimeMs() > toTime) { break DO; } } if (toTx != null) { - if (info.getId() > toTx.longValue()) + if (info.getId() > toTx) { break DO; } @@ -1382,9 +1334,6 @@ public class MetadataTracker extends CoreStatePublisher implements Tracker infoSrv.clearProcessedTransactions(); } - /** - * @param query - */ public void addQueryToReindex(String query) { this.queriesToReindex.offer(query); diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/AlfrescoSolrUtils.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/AlfrescoSolrUtils.java index 17d0166eb..c910878ab 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/AlfrescoSolrUtils.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/AlfrescoSolrUtils.java @@ -135,18 +135,17 @@ public class AlfrescoSolrUtils */ public static Transaction getTransaction(int deletes, int updates) { - long txnCommitTime = System.currentTimeMillis(); - Transaction transaction = new Transaction(); - transaction.setCommitTimeMs(txnCommitTime); - transaction.setId(generateId()); - transaction.setDeletes(deletes); - transaction.setUpdates(updates); - return transaction; + return getTransaction(deletes, updates, generateId()); } public static Transaction getTransaction(int deletes, int updates, long id) { - long txnCommitTime = System.currentTimeMillis(); + return getTransaction(deletes, updates, id, System.currentTimeMillis()); + } + + public static Transaction getTransaction(int deletes, int updates, long id, long timestamp) + { + long txnCommitTime = timestamp; Transaction transaction = new Transaction(); transaction.setCommitTimeMs(txnCommitTime); transaction.setId(id); @@ -309,6 +308,7 @@ public class AlfrescoSolrUtils Acl acl = new Acl(aclChangeSet.getId(), aclId); return acl; } + /** * Get an AclChangeSet * @param aclCount @@ -316,14 +316,17 @@ public class AlfrescoSolrUtils */ public static AclChangeSet getAclChangeSet(int aclCount) { - AclChangeSet aclChangeSet = new AclChangeSet(generateId(), System.currentTimeMillis(), aclCount); - return aclChangeSet; + return new AclChangeSet(generateId(), System.currentTimeMillis(), aclCount); } public static AclChangeSet getAclChangeSet(int aclCount, long id) { - AclChangeSet aclChangeSet = new AclChangeSet(id, System.currentTimeMillis(), aclCount); - return aclChangeSet; + return new AclChangeSet(id, System.currentTimeMillis(), aclCount); + } + + public static AclChangeSet getAclChangeSet(int aclCount, long id, long timestamp) + { + return new AclChangeSet(id, timestamp, aclCount); } private static AtomicLong id = new AtomicLong(System.currentTimeMillis()); diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/SolrITInitializer.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/SolrITInitializer.java index bb3b1fa52..24d744f00 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/SolrITInitializer.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/SolrITInitializer.java @@ -29,7 +29,6 @@ package org.alfresco.solr; import static org.alfresco.solr.AlfrescoSolrUtils.createCoreUsingTemplate; import com.carrotsearch.randomizedtesting.annotations.ThreadLeakScope; -import com.carrotsearch.randomizedtesting.annotations.ThreadLeakLingering; import org.alfresco.solr.basics.RandomSupplier; import org.alfresco.solr.client.SOLRAPIQueueClient; import org.apache.commons.io.FileUtils; @@ -98,7 +97,10 @@ import java.util.concurrent.atomic.AtomicInteger; public abstract class SolrITInitializer extends SolrTestCaseJ4 { private static final Logger LOGGER = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - + protected static final int DEFAULT_CONNECTION_TIMEOUT1 = DEFAULT_CONNECTION_TIMEOUT; + protected static final int CLIENT_SO_TIMEOUT = 90000; + protected final static int INDEX_TIMEOUT = 100000; + private static AtomicInteger nodeCnt; protected static boolean useExplicitNodeNames; @@ -115,8 +117,6 @@ public abstract class SolrITInitializer extends SolrTestCaseJ4 //Standalone Tests protected static SolrCore defaultCore; - protected static final int clientConnectionTimeout = DEFAULT_CONNECTION_TIMEOUT; - protected static final int clientSoTimeout = 90000; protected static final String id = "id"; @@ -501,8 +501,8 @@ public abstract class SolrITInitializer extends SolrTestCaseJ4 try { HttpSolrClient client = new HttpSolrClient(url); - client.setConnectionTimeout(clientConnectionTimeout); - client.setSoTimeout(clientSoTimeout); + client.setConnectionTimeout(DEFAULT_CONNECTION_TIMEOUT1); + client.setSoTimeout(CLIENT_SO_TIMEOUT); client.setDefaultMaxConnectionsPerHost(100); client.setMaxTotalConnections(100); return client; diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedAlfrescoSolrTrackerRaceIT.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedAlfrescoSolrTrackerRaceIT.java index 631cd7363..bc2cec9d2 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedAlfrescoSolrTrackerRaceIT.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedAlfrescoSolrTrackerRaceIT.java @@ -66,7 +66,7 @@ public class DistributedAlfrescoSolrTrackerRaceIT extends AbstractAlfrescoDistri @BeforeClass public static void initData() throws Throwable { - initSolrServers(2, "DistributedAlfrescoSolrTrackerRaceTest", null); + initSolrServers(2, "DistributedAlfrescoSolrTrackerRaceIT", null); } @AfterClass @@ -76,25 +76,23 @@ public class DistributedAlfrescoSolrTrackerRaceIT extends AbstractAlfrescoDistri } @Test - public void testTracker() throws Exception - { + public void testTracker() throws Exception { putHandleDefaults(); - AclChangeSet aclChangeSet = getAclChangeSet(1); + AclChangeSet aclChangeSet = getAclChangeSet(2, 1, System.currentTimeMillis() - + AbstractTracker.TIME_STEP_32_DAYS_IN_MS); Acl acl = getAcl(aclChangeSet); Acl acl2 = getAcl(aclChangeSet); - AclReaders aclReaders = getAclReaders(aclChangeSet, acl, singletonList("joel"), singletonList("phil"), null); - AclReaders aclReaders2 = getAclReaders(aclChangeSet, acl2, singletonList("jim"), singletonList("phil"), null); + AclReaders aclReaders = + getAclReaders(aclChangeSet, acl, singletonList("joel"), singletonList("phil"), null); + AclReaders aclReaders2 = + getAclReaders(aclChangeSet, acl2, singletonList("jim"), singletonList("phil"), null); // Transaction between [1-2000] is required, when greater value checking the core will fail - Transaction txn = getTransaction(0, 2, 1); - long txnCommitTimeMs = txn.getCommitTimeMs(); - - // Subtract from the commit time to go beyond hole retention - long backdatedCommitTimeMs = txnCommitTimeMs - 4600000; - txn.setCommitTimeMs(backdatedCommitTimeMs); + Transaction txn = getTransaction(0, 2, 1, + System.currentTimeMillis() - AbstractTracker.TIME_STEP_32_DAYS_IN_MS); //Next create two nodes to update for the transaction Node folderNode = getNode(txn, acl, Node.SolrApiNodeStatus.UPDATED); @@ -103,9 +101,12 @@ public class DistributedAlfrescoSolrTrackerRaceIT extends AbstractAlfrescoDistri // Next, create the node metadata for each node. // Note: the error node metadata will cause an exception. - NodeMetaData folderMetaData = getNodeMetaData(folderNode, txn, acl, "mike", null, false); - NodeMetaData fileMetaData = getNodeMetaData(fileNode, txn, acl, "mike", ancestors(folderMetaData.getNodeRef()), false); - NodeMetaData errorMetaData = getNodeMetaData(errorNode, txn, acl, "lisa", ancestors(folderMetaData.getNodeRef()), true); + NodeMetaData folderMetaData = + getNodeMetaData(folderNode, txn, acl, "mike", null, false); + NodeMetaData fileMetaData = getNodeMetaData(fileNode, txn, acl, "mike", + ancestors(folderMetaData.getNodeRef()), false); + NodeMetaData errorMetaData = getNodeMetaData(errorNode, txn, acl, "lisa", + ancestors(folderMetaData.getNodeRef()), true); // Index the transaction, nodes, and nodeMetaDatas. // Note that the content is automatically created by the test framework. @@ -113,25 +114,34 @@ public class DistributedAlfrescoSolrTrackerRaceIT extends AbstractAlfrescoDistri indexAclChangeSet(aclChangeSet, asList(acl, acl2), asList(aclReaders, aclReaders2)); BooleanQuery.Builder builder = new BooleanQuery.Builder(); - builder.add(new BooleanClause(new TermQuery(new Term(QueryConstants.FIELD_SOLR4_ID, "TRACKER!STATE!ACLTX")), BooleanClause.Occur.MUST)); - builder.add(new BooleanClause(LegacyNumericRangeQuery.newLongRange(QueryConstants.FIELD_S_ACLTXID, aclChangeSet.getId(), aclChangeSet.getId() + 1, true, false), BooleanClause.Occur.MUST)); + builder.add(new BooleanClause(new TermQuery(new Term(QueryConstants.FIELD_SOLR4_ID, "TRACKER!STATE!ACLTX")), + BooleanClause.Occur.MUST)); + + builder.add(new BooleanClause(LegacyNumericRangeQuery.newLongRange(QueryConstants.FIELD_S_ACLTXID, + aclChangeSet.getId(), aclChangeSet.getId() + 1, true, false), + BooleanClause.Occur.MUST)); + BooleanQuery waitForQuery = builder.build(); - waitForDocCountAllCores(waitForQuery, 1, 80000); + waitForDocCountAllCores(waitForQuery, 1, INDEX_TIMEOUT); // This ACL should have one record in each core with DBID sharding - waitForDocCountAllCores(new TermQuery(new Term(QueryConstants.FIELD_READER, "jim")), 1, 80000); + waitForDocCountAllCores(new TermQuery(new Term(QueryConstants.FIELD_READER, "jim")), 1, INDEX_TIMEOUT); // We should have 2 document in totals (1 folder and 1 file) - waitForDocCount(new TermQuery(new Term("content@s___t@{http://www.alfresco.org/model/content/1.0}content", "world")), 2, 100000); + waitForDocCount(new TermQuery(new Term("content@s___t@{http://www.alfresco.org/model/content/1.0}content", + "world")), 2, INDEX_TIMEOUT); // There should be 1 with the folder node identifier. - waitForDocCount(new TermQuery(new Term("content@s___t@{http://www.alfresco.org/model/content/1.0}content", Long.toString(folderNode.getId()))), 1, 80000); + waitForDocCount(new TermQuery(new Term("content@s___t@{http://www.alfresco.org/model/content/1.0}content", + Long.toString(folderNode.getId()))), 1, INDEX_TIMEOUT); // There should be 1 with the file node identifier. - waitForDocCount(new TermQuery(new Term("content@s___t@{http://www.alfresco.org/model/content/1.0}content", Long.toString(fileNode.getId()))), 1, 80000); + waitForDocCount(new TermQuery(new Term("content@s___t@{http://www.alfresco.org/model/content/1.0}content", + Long.toString(fileNode.getId()))), 1, INDEX_TIMEOUT); // and last but not least, the error node shouldn't be in the index. - waitForDocCount(new TermQuery(new Term("content@s___t@{http://www.alfresco.org/model/content/1.0}content", Long.toString(errorNode.getId()))), 0, 80000); + waitForDocCount(new TermQuery(new Term("content@s___t@{http://www.alfresco.org/model/content/1.0}content", + Long.toString(errorNode.getId()))), 0, INDEX_TIMEOUT); // This will run the same query on the control client and the cluster and compare the result. query(getDefaultTestClient(), diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedAlfrescoTrackerWithDelayedTransactionsIT.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedAlfrescoTrackerWithDelayedTransactionsIT.java new file mode 100644 index 000000000..60258626c --- /dev/null +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedAlfrescoTrackerWithDelayedTransactionsIT.java @@ -0,0 +1,169 @@ +/* + * #%L + * Alfresco Search Services + * %% + * Copyright (C) 2005 - 2020 Alfresco Software Limited + * %% + * This file is part of the Alfresco software. + * If the software was purchased under a paid Alfresco license, the terms of + * the paid license agreement will prevail. Otherwise, the software is + * provided under the following open source license terms: + * + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + * #L% + */ + +package org.alfresco.solr.tracker; + +import org.alfresco.repo.search.adaptor.lucene.QueryConstants; +import org.alfresco.solr.AbstractAlfrescoDistributedIT; +import org.alfresco.solr.client.Acl; +import org.alfresco.solr.client.AclChangeSet; +import org.alfresco.solr.client.AclReaders; +import org.alfresco.solr.client.Node; +import org.alfresco.solr.client.NodeMetaData; +import org.alfresco.solr.client.Transaction; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.BooleanClause; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.LegacyNumericRangeQuery; +import org.apache.lucene.search.TermQuery; +import org.apache.solr.SolrTestCaseJ4; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.util.List; + +import static java.util.Arrays.asList; +import static java.util.Collections.singletonList; +import static org.alfresco.solr.AlfrescoSolrUtils.ancestors; +import static org.alfresco.solr.AlfrescoSolrUtils.getAcl; +import static org.alfresco.solr.AlfrescoSolrUtils.getAclChangeSet; +import static org.alfresco.solr.AlfrescoSolrUtils.getAclReaders; +import static org.alfresco.solr.AlfrescoSolrUtils.getNode; +import static org.alfresco.solr.AlfrescoSolrUtils.getNodeMetaData; +import static org.alfresco.solr.AlfrescoSolrUtils.getTransaction; +import static org.alfresco.solr.AlfrescoSolrUtils.indexAclChangeSet; + +@SolrTestCaseJ4.SuppressSSL +public class DistributedAlfrescoTrackerWithDelayedTransactionsIT extends AbstractAlfrescoDistributedIT { + + private static NodeMetaData folderMetaData; + private static AclChangeSet aclChangeSet; + + @BeforeClass + public static void initData() throws Throwable + { + initializeDataBeforeServerCreation(); + initSolrServers(2, "DistributedAlfrescoTrackerWithDelayedTransactionsIT", + null); + } + + + @AfterClass + public static void destroyData() + { + dismissSolrServers(); + } + + private static void initializeDataBeforeServerCreation() + { + putHandleDefaults(); + + aclChangeSet = getAclChangeSet(2, 1, + System.currentTimeMillis() - AbstractTracker.TIME_STEP_32_DAYS_IN_MS); + + Acl acl = getAcl(aclChangeSet); + Acl acl2 = getAcl(aclChangeSet); + + AclReaders aclReaders = + getAclReaders(aclChangeSet, acl, singletonList("joel"), singletonList("phil"), null); + AclReaders aclReaders2 = + getAclReaders(aclChangeSet, acl2, singletonList("jim"), singletonList("phil"), null); + + // Transaction between [1-2000] is required, when greater value checking the core will fail + Transaction txn = getTransaction(0, 2, 1, + System.currentTimeMillis() - AbstractTracker.TIME_STEP_32_DAYS_IN_MS); + + //Next create two nodes to update for the transaction + Node folderNode = getNode(txn, acl, Node.SolrApiNodeStatus.UPDATED); + Node fileNode = getNode(txn, acl, Node.SolrApiNodeStatus.UPDATED); + + folderMetaData = getNodeMetaData(folderNode, txn, acl, "mike", null, false); + NodeMetaData fileMetaData = getNodeMetaData(fileNode, txn, acl, "mike", + ancestors(folderMetaData.getNodeRef()), false); + + indexAclChangeSet(aclChangeSet, asList(acl, acl2), asList(aclReaders, aclReaders2)); + indexTransaction(txn, List.of(folderNode, fileNode), List.of(folderMetaData, fileMetaData)); + } + + @Test + public void testTracker() throws Exception { + + BooleanQuery.Builder builder = new BooleanQuery.Builder(); + builder.add(new BooleanClause(new TermQuery(new Term(QueryConstants.FIELD_SOLR4_ID, "TRACKER!STATE!ACLTX")), + BooleanClause.Occur.MUST)); + builder.add(new BooleanClause(LegacyNumericRangeQuery.newLongRange(QueryConstants.FIELD_S_ACLTXID, + aclChangeSet.getId(), aclChangeSet.getId() + 1, true, false), + BooleanClause.Occur.MUST)); + + BooleanQuery waitForQuery = builder.build(); + waitForDocCountAllCores(waitForQuery, 1, INDEX_TIMEOUT); + + // This ACL should have one record in each core with DBID sharding + waitForDocCountAllCores(new TermQuery(new Term(QueryConstants.FIELD_READER, "jim")), + 1, INDEX_TIMEOUT); + + // We should have 2 document in totals (1 folder and 1 file) + waitForDocCount(new TermQuery(new Term("content@s___t@{http://www.alfresco.org/model/content/1.0}content", + "world")), 2, INDEX_TIMEOUT); + + + // This will run the same query on the control client and the cluster and compare the result. + query(getDefaultTestClient(), + true, + "{\"locales\":[\"en\"], \"templates\": [{\"name\":\"t1\", \"template\":\"%cm:content\"}]}", + params("q", "t1:world", "qt", "/afts", "shards.qt", "/afts", "start", "0", "rows", "6", "sort", "id asc")); + + + // Index new ACL after 32 days + AclChangeSet lateAclChangeSet = getAclChangeSet(1, 2); + Acl lateAcl = getAcl(lateAclChangeSet); + AclReaders lateAclReaders = + getAclReaders(lateAclChangeSet, lateAcl, singletonList("elia"), singletonList("phil"), null); + indexAclChangeSet(lateAclChangeSet, asList(lateAcl), asList(lateAclReaders)); + + // Index new transaction after 32 days + Transaction lateTransaction = getTransaction(0, 1); + Node lateNode = getNode(lateTransaction, lateAcl, Node.SolrApiNodeStatus.UPDATED); + NodeMetaData lateNodeMetaData = getNodeMetaData(lateNode, lateTransaction, lateAcl, "elia", + ancestors(folderMetaData.getNodeRef()), false); + + // Check new Acl has been indexed + builder = new BooleanQuery.Builder(); + builder.add(new BooleanClause(new TermQuery(new Term(QueryConstants.FIELD_SOLR4_ID, "TRACKER!STATE!ACLTX")), + BooleanClause.Occur.MUST)); + builder.add(new BooleanClause(LegacyNumericRangeQuery.newLongRange(QueryConstants.FIELD_S_ACLTXID, + lateAclChangeSet.getId(), lateAclChangeSet.getId() + 1, true, false), + BooleanClause.Occur.MUST)); + waitForQuery = builder.build(); + waitForDocCountAllCores(waitForQuery, 1, INDEX_TIMEOUT); + + // Check the new node has been indexed + indexTransaction(lateTransaction, List.of(lateNode), List.of(lateNodeMetaData)); + waitForDocCount(new TermQuery(new Term("content@s___t@{http://www.alfresco.org/model/content/1.0}content", + "world")), 3, INDEX_TIMEOUT); + } +} diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedCascadeTrackerIT.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedCascadeTrackerIT.java index 980053dfb..ad9ba2474 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedCascadeTrackerIT.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/tracker/DistributedCascadeTrackerIT.java @@ -77,8 +77,6 @@ public class DistributedCascadeTrackerIT extends AbstractAlfrescoDistributedIT private final String pathChild1 = "pathChild2"; - private final int timeout = 100000; - @Before public void initData() throws Throwable { @@ -139,7 +137,7 @@ public class DistributedCascadeTrackerIT extends AbstractAlfrescoDistributedIT */ indexParentFolderWithCascade(); - waitForDocCount(params("qt", "/afts", "q", "PATH:" + cascadingFirstChild), 1, timeout); + waitForDocCount(params("qt", "/afts", "q", "PATH:" + cascadingFirstChild), 1, INDEX_TIMEOUT); // Check if the path is updated for both the nodes assertShardCount(0, params("qt", "/afts", "q", "PATH:" + cascadingFirstChild), 1); @@ -184,7 +182,7 @@ public class DistributedCascadeTrackerIT extends AbstractAlfrescoDistributedIT /* * Get sure the nodes are indexed correctly in the shards */ - waitForDocCount(new TermQuery(new Term("content@s___t@{http://www.alfresco.org/model/content/1.0}content", "world")), 3, timeout); + waitForDocCount(new TermQuery(new Term("content@s___t@{http://www.alfresco.org/model/content/1.0}content", "world")), 3, INDEX_TIMEOUT); assertShardCount(0, new TermQuery(new Term("content@s___t@{http://www.alfresco.org/model/content/1.0}content", "world")), 2); assertShardCount(1, new TermQuery(new Term("content@s___t@{http://www.alfresco.org/model/content/1.0}content", "world")), 1); } From 87bcffb6deb2710d7700ae38f06d6375e2c9f713 Mon Sep 17 00:00:00 2001 From: Alessandro Benedetti Date: Fri, 5 Jun 2020 16:31:37 +0100 Subject: [PATCH 4/5] Feature/search 2228 --- .../alfresco/solr/AlfrescoSolrDataModel.java | 37 +- .../alfresco/solr/query/Solr4QueryParser.java | 410 +++++++++-------- .../AFTSDefaultTextQueryIT.java | 413 +++++++++++++++++- .../AbstractRequestHandlerIT.java | 13 + 4 files changed, 669 insertions(+), 204 deletions(-) diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoSolrDataModel.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoSolrDataModel.java index 9a676addd..99da978d0 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoSolrDataModel.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoSolrDataModel.java @@ -138,8 +138,9 @@ public class AlfrescoSolrDataModel implements QueryConstants public enum FieldUse { - FTS, // Term/Phrase/Range/Fuzzy/Prefix/Proximity/Wild - ID, // Exact/ExactRange - Comparison, In, Upper, Lower + FTS, // Term/Phrase/Range/Fuzzy/Prefix/Proximity + ID, // ExactRange - Comparison, In, Upper, Lower, Wildcard + EXACT, // Exact FACET, // Field, Range, Query MULTI_FACET, // Text fields will require cross language support to avoid tokenisation for facets STATS, // Stats @@ -701,6 +702,9 @@ public class AlfrescoSolrDataModel implements QueryConstants case ID: addIdentifierSearchFields(propertyDefinition, indexedField); break; + case EXACT: + addExactSearchFields(propertyDefinition, indexedField); + break; case MULTI_FACET: addMultiSearchFields(propertyDefinition, indexedField); break; @@ -808,9 +812,10 @@ public class AlfrescoSolrDataModel implements QueryConstants } /* - * Adds best identifier fields in order of preference + * If the untokenised version is available, go for it. + * If not it takes the tokenised versions available. */ - private void addIdentifierSearchFields(PropertyDefinition propertyDefinition, IndexedField indexedField) + private void addIdentifierSearchFields( PropertyDefinition propertyDefinition , IndexedField indexedField) { if ((propertyDefinition.getIndexTokenisationMode() == IndexTokenisationMode.FALSE) || (propertyDefinition.getIndexTokenisationMode() == IndexTokenisationMode.BOTH)) @@ -829,6 +834,30 @@ public class AlfrescoSolrDataModel implements QueryConstants } } + /* + * If only the untokenised version is available, go for it. + * If not it takes the tokenised cross locale version. + */ + private void addExactSearchFields(PropertyDefinition propertyDefinition, IndexedField indexedField) + { + if ((propertyDefinition.getIndexTokenisationMode() == IndexTokenisationMode.FALSE) + || !(propertyDefinition.getIndexTokenisationMode() == IndexTokenisationMode.BOTH)) + { + + indexedField.addField(getFieldForText(true, false, false, propertyDefinition), true, false); + indexedField.addField(getFieldForText(false, false, false, propertyDefinition), false, false); + } + else + { + if(crossLocaleSearchDataTypes.contains(propertyDefinition.getDataType().getName()) || crossLocaleSearchProperties.contains(propertyDefinition.getName())) + { + indexedField.addField(getFieldForText(false, true, false, propertyDefinition), false, false); + } else{ + throw new UnsupportedOperationException("Exact Term search is not supported unless you configure the field <"+propertyDefinition.getName()+"> for cross locale search"); + } + } + } + /* * Adds best identifier fields in order of preference */ diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/query/Solr4QueryParser.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/query/Solr4QueryParser.java index 13d13003c..02810e9be 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/query/Solr4QueryParser.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/query/Solr4QueryParser.java @@ -150,12 +150,17 @@ import java.util.concurrent.Future; import java.util.stream.Collectors; import java.util.stream.Stream; +import static org.apache.solr.search.QueryParsing.DEFTYPE; + /** * @author Andy * */ public class Solr4QueryParser extends QueryParser implements QueryConstants { + + private static final String CMIS_REQUEST_HANDLER = "cmis"; + public Solr4QueryParser(SolrQueryRequest req, Version matchVersion, String f, Analyzer a, FTSQueryParser.RerankPhase rerankPhase) { @@ -1480,13 +1485,18 @@ public class Solr4QueryParser extends QueryParser implements QueryConstants BooleanQuery.Builder query = new BooleanQuery.Builder(); for (String fieldName : text) { - Query part = getQuery.apply(fieldName); - if (part != null) - { - query.add(part, Occur.SHOULD); - } else - { - query.add(createNoMatchQuery(), Occur.SHOULD); + Query part; + try { + part = getQuery.apply(fieldName); + if (part != null) + { + query.add(part, Occur.SHOULD); + } else + { + query.add(createNoMatchQuery(), Occur.SHOULD); + } + } catch (UnsupportedOperationException e) { + logger.warn(e); } } return query.build(); @@ -4820,46 +4830,52 @@ public class Solr4QueryParser extends QueryParser implements QueryConstants AnalysisMode analysisMode, LuceneFunction luceneFunction, String expandedFieldName, List expandedLocales) throws ParseException { - BooleanQuery.Builder booleanQuery = new BooleanQuery.Builder(); - for (Locale locale : expandedLocales) + BooleanQuery.Builder booleanQueryBuilder = new BooleanQuery.Builder(); + IndexedField indexedField = AlfrescoSolrDataModel.getInstance().getQueryableFields(pDef.getName(), null, + FieldUse.FTS); + if (isExactTermSearch(analysisMode)) { - if (locale.toString().length() == 0) + for (FieldInstance field : indexedField.getFields()) { - IndexedField indexedField = AlfrescoSolrDataModel.getInstance().getQueryableFields(pDef.getName(), null, - FieldUse.FTS); + if (!field.isLocalised()) + { + Query subQuery = subQueryBuilder.getQuery(field.getField(), queryText, analysisMode, + luceneFunction); + if (subQuery != null) + { + booleanQueryBuilder.add(subQuery, Occur.SHOULD); + } + } + } + if (booleanQueryBuilder.build().clauses().size() == 0) + { + throw new UnsupportedOperationException("Exact Term search is not supported unless you configure the field <"+pDef.getName()+"> for cross locale search"); + } + } else + { + for (Locale locale : expandedLocales) + { + boolean localisedSearch = locale.toString().length() != 0; + if (localisedSearch) + { + StringBuilder builder = new StringBuilder(queryText.length() + 10); + builder.append("\u0000").append(locale.toString()).append("\u0000").append(queryText); + } for (FieldInstance field : indexedField.getFields()) { - if (!field.isLocalised()) + if ((!localisedSearch && !field.isLocalised()) || (localisedSearch && field.isLocalised())) { Query subQuery = subQueryBuilder.getQuery(field.getField(), queryText, analysisMode, luceneFunction); if (subQuery != null) { - booleanQuery.add(subQuery, Occur.SHOULD); - } - } - } - } else - { - StringBuilder builder = new StringBuilder(queryText.length() + 10); - builder.append("\u0000").append(locale.toString()).append("\u0000").append(queryText); - IndexedField indexedField = AlfrescoSolrDataModel.getInstance().getQueryableFields(pDef.getName(), null, - FieldUse.FTS); - for (FieldInstance field : indexedField.getFields()) - { - if (field.isLocalised()) - { - Query subQuery = subQueryBuilder.getQuery(field.getField(), builder.toString(), analysisMode, - luceneFunction); - if (subQuery != null) - { - booleanQuery.add(subQuery, Occur.SHOULD); + booleanQueryBuilder.add(subQuery, Occur.SHOULD); } } } } } - return getNonEmptyBooleanQuery(booleanQuery.build()); + return getNonEmptyBooleanQuery(booleanQueryBuilder.build()); } protected void addLocaleSpecificUntokenisedMLOrTextFunction(String expandedFieldName, PropertyDefinition pDef, @@ -4880,67 +4896,74 @@ public class Solr4QueryParser extends QueryParser implements QueryConstants } private FieldInstance getFieldInstance(String baseFieldName, PropertyDefinition pDef, Locale locale, - IndexTokenisationMode preferredIndexTokenisationMode) + IndexTokenisationMode preferredIndexTokenisationMode, AnalysisMode analysisMode) { if (pDef != null) { - switch (preferredIndexTokenisationMode) { - case BOTH: - throw new IllegalStateException("Preferred mode can not be BOTH"); - case FALSE: - if (locale.toString().length() == 0) - { - IndexedField indexedField = AlfrescoSolrDataModel.getInstance().getQueryableFields(pDef.getName(), - null, FieldUse.ID); - for (FieldInstance field : indexedField.getFields()) + case BOTH: + throw new IllegalStateException("Preferred mode can not be BOTH"); + case FALSE: + if (locale.toString().length() == 0) { - if (!field.isLocalised()) + IndexedField indexedField = AlfrescoSolrDataModel.getInstance().getQueryableFields(pDef.getName(), + null, FieldUse.ID); + for (FieldInstance field : indexedField.getFields()) { - return field; + if (!field.isLocalised()) + { + return field; + } + } + } else + { + IndexedField indexedField = AlfrescoSolrDataModel.getInstance().getQueryableFields(pDef.getName(), + null, FieldUse.ID); + for (FieldInstance field : indexedField.getFields()) + { + if (field.isLocalised()) + { + return field; + } } } - } else + break; + case TRUE: { - IndexedField indexedField = AlfrescoSolrDataModel.getInstance().getQueryableFields(pDef.getName(), - null, FieldUse.ID); - for (FieldInstance field : indexedField.getFields()) + IndexedField indexedField; + if (isExactTermSearch(analysisMode)) { - if (field.isLocalised()) + indexedField = AlfrescoSolrDataModel.getInstance().getQueryableFields(pDef.getName(), + null, FieldUse.EXACT); + } else + { + indexedField = AlfrescoSolrDataModel.getInstance().getQueryableFields(pDef.getName(), + null, FieldUse.FTS); + } + if (locale.toString().length() == 0) + { + for (FieldInstance field : indexedField.getFields()) { - return field; + if (!field.isLocalised()) + { + return field; + } + } + } else + { + for (FieldInstance field : indexedField.getFields()) + { + if (field.isLocalised()) + { + return field; + } } } + break; } - break; - case TRUE: - if (locale.toString().length() == 0) - { - IndexedField indexedField = AlfrescoSolrDataModel.getInstance().getQueryableFields(pDef.getName(), - null, FieldUse.FTS); - for (FieldInstance field : indexedField.getFields()) - { - if (!field.isLocalised()) - { - return field; - } - } - } else - { - IndexedField indexedField = AlfrescoSolrDataModel.getInstance().getQueryableFields(pDef.getName(), - null, FieldUse.FTS); - for (FieldInstance field : indexedField.getFields()) - { - if (field.isLocalised()) - { - return field; - } - } - } - break; } - return new FieldInstance("_dummy_", false, false); + return null; } @@ -4948,6 +4971,10 @@ public class Solr4QueryParser extends QueryParser implements QueryConstants } + private boolean isExactTermSearch(AnalysisMode analysisMode) { + return analysisMode==AnalysisMode.IDENTIFIER && !this.solrParams.get(DEFTYPE).equals(CMIS_REQUEST_HANDLER); + } + protected void addLocaleSpecificUntokenisedTextRangeFunction(String expandedFieldName, PropertyDefinition pDef, String lower, String upper, boolean includeLower, boolean includeUpper, LuceneFunction luceneFunction, Builder booleanQuery, Locale locale, IndexTokenisationMode tokenisationMode) throws ParseException @@ -4984,9 +5011,9 @@ public class Solr4QueryParser extends QueryParser implements QueryConstants } private void addMLTextOrTextAttributeQuery(String field, PropertyDefinition pDef, String queryText, - SubQuery subQueryBuilder, AnalysisMode analysisMode, LuceneFunction luceneFunction, - String expandedFieldName, IndexTokenisationMode tokenisationMode, Builder booleanQuery, Locale locale) - throws ParseException + SubQuery subQueryBuilder, AnalysisMode analysisMode, LuceneFunction luceneFunction, + String expandedFieldName, IndexTokenisationMode tokenisationMode, Builder booleanQuery, Locale locale) + throws ParseException { boolean lowercaseExpandedTerms = getLowercaseExpandedTerms(); @@ -4994,48 +5021,59 @@ public class Solr4QueryParser extends QueryParser implements QueryConstants { switch (tokenisationMode) { - case BOTH: - switch (analysisMode) - { - default: - case DEFAULT: - addLocaleSpecificMLOrTextAttribute(pDef, queryText, subQueryBuilder, analysisMode, luceneFunction, - booleanQuery, locale, expandedFieldName, tokenisationMode, IndexTokenisationMode.TRUE); - - if (ContentModel.PROP_NAME.equals(pDef.getName())) + case BOTH: + switch (analysisMode) { - setLowercaseExpandedTerms(false); - addLocaleSpecificMLOrTextAttribute(pDef, queryText, subQueryBuilder, analysisMode, luceneFunction, - booleanQuery, locale, expandedFieldName, tokenisationMode, IndexTokenisationMode.FALSE); - } + default: + case DEFAULT: + addLocaleSpecificMLOrTextAttribute(pDef, queryText, subQueryBuilder, analysisMode, luceneFunction, + booleanQuery, locale, expandedFieldName, tokenisationMode, IndexTokenisationMode.TRUE); + if (ContentModel.PROP_NAME.equals(pDef.getName())) + { + setLowercaseExpandedTerms(false); + addLocaleSpecificMLOrTextAttribute(pDef, queryText, subQueryBuilder, analysisMode, luceneFunction, + booleanQuery, locale, expandedFieldName, tokenisationMode, IndexTokenisationMode.FALSE); + } + + break; + case TOKENISE: + addLocaleSpecificMLOrTextAttribute(pDef, queryText, subQueryBuilder, analysisMode, luceneFunction, + booleanQuery, locale, expandedFieldName, tokenisationMode, IndexTokenisationMode.TRUE); + break; + case IDENTIFIER: + setLowercaseExpandedTerms(false); + if (isExactTermSearch(analysisMode)) + {//with exact search we favour tokenization, specifically cross locale tokenization + addLocaleSpecificMLOrTextAttribute(pDef, queryText, subQueryBuilder, analysisMode, luceneFunction, + booleanQuery, locale, expandedFieldName, tokenisationMode, IndexTokenisationMode.TRUE); + } else + { + addLocaleSpecificMLOrTextAttribute(pDef, queryText, subQueryBuilder, analysisMode, luceneFunction, + booleanQuery, locale, expandedFieldName, tokenisationMode, IndexTokenisationMode.FALSE); + } + break; + case FUZZY: + case PREFIX: + case WILD: + case LIKE: + setLowercaseExpandedTerms(false); + addLocaleSpecificMLOrTextAttribute(pDef, queryText, subQueryBuilder, analysisMode, luceneFunction, + booleanQuery, locale, expandedFieldName, tokenisationMode, IndexTokenisationMode.FALSE); + + break; + } break; - case TOKENISE: - addLocaleSpecificMLOrTextAttribute(pDef, queryText, subQueryBuilder, analysisMode, luceneFunction, - booleanQuery, locale, expandedFieldName, tokenisationMode, IndexTokenisationMode.TRUE); - break; - case IDENTIFIER: - case FUZZY: - case PREFIX: - case WILD: - case LIKE: + case FALSE: setLowercaseExpandedTerms(false); addLocaleSpecificMLOrTextAttribute(pDef, queryText, subQueryBuilder, analysisMode, luceneFunction, booleanQuery, locale, expandedFieldName, tokenisationMode, IndexTokenisationMode.FALSE); - break; - } - break; - case FALSE: - setLowercaseExpandedTerms(false); - addLocaleSpecificMLOrTextAttribute(pDef, queryText, subQueryBuilder, analysisMode, luceneFunction, - booleanQuery, locale, expandedFieldName, tokenisationMode, IndexTokenisationMode.FALSE); - break; - case TRUE: - default: - addLocaleSpecificMLOrTextAttribute(pDef, queryText, subQueryBuilder, analysisMode, luceneFunction, - booleanQuery, locale, expandedFieldName, tokenisationMode, IndexTokenisationMode.TRUE); - break; + case TRUE: + default: + addLocaleSpecificMLOrTextAttribute(pDef, queryText, subQueryBuilder, analysisMode, luceneFunction, + booleanQuery, locale, expandedFieldName, tokenisationMode, IndexTokenisationMode.TRUE); + break; } } finally { @@ -5055,22 +5093,26 @@ public class Solr4QueryParser extends QueryParser implements QueryConstants } private void addLocaleSpecificMLOrTextAttribute(PropertyDefinition pDef, String queryText, SubQuery subQueryBuilder, - AnalysisMode analysisMode, LuceneFunction luceneFunction, Builder booleanQuery, Locale locale, - String textFieldName, IndexTokenisationMode tokenisationMode, - IndexTokenisationMode preferredTokenisationMode) throws ParseException { + AnalysisMode analysisMode, LuceneFunction luceneFunction, Builder booleanQuery, Locale locale, + String textFieldName, IndexTokenisationMode tokenisationMode, + IndexTokenisationMode preferredTokenisationMode) throws ParseException + { - FieldInstance fieldInstance = getFieldInstance(textFieldName, pDef, locale, preferredTokenisationMode); - StringBuilder builder = new StringBuilder(queryText.length() + 10); - if (fieldInstance.isLocalised()) + FieldInstance fieldInstance = getFieldInstance(textFieldName, pDef, locale, preferredTokenisationMode, analysisMode); + if (fieldInstance != null) { - builder.append("\u0000").append(locale.toString()).append("\u0000"); - } - builder.append(queryText); - Query subQuery = subQueryBuilder.getQuery(fieldInstance.getField(), builder.toString(), analysisMode, - luceneFunction); - if (subQuery != null) - { - booleanQuery.add(subQuery, Occur.SHOULD); + StringBuilder builder = new StringBuilder(queryText.length() + 10); + if (fieldInstance.isLocalised()) + { + builder.append("\u0000").append(locale.toString()).append("\u0000"); + } + builder.append(queryText); + Query subQuery = subQueryBuilder.getQuery(fieldInstance.getField(), builder.toString(), analysisMode, + luceneFunction); + if (subQuery != null) + { + booleanQuery.add(subQuery, Occur.SHOULD); + } } } @@ -5115,76 +5157,80 @@ public class Solr4QueryParser extends QueryParser implements QueryConstants } private void addLocaleSpecificTextRange(String expandedFieldName, PropertyDefinition pDef, String part1, - String part2, boolean includeLower, boolean includeUpper, Builder booleanQuery, Locale locale, - AnalysisMode analysisMode, IndexTokenisationMode tokenisationMode, - IndexTokenisationMode preferredTokenisationMode) throws ParseException, IOException + String part2, boolean includeLower, boolean includeUpper, Builder booleanQuery, Locale locale, + AnalysisMode analysisMode, IndexTokenisationMode tokenisationMode, + IndexTokenisationMode preferredTokenisationMode) throws ParseException, IOException { - FieldInstance fieldInstance = getFieldInstance(expandedFieldName, pDef, locale, preferredTokenisationMode); - - String firstString = null; - if ((part1 != null) && !part1.equals("\u0000")) + FieldInstance fieldInstance = getFieldInstance(expandedFieldName, pDef, locale, preferredTokenisationMode, analysisMode); + if (fieldInstance != null) { - if (fieldInstance.isLocalised()) + String firstString = null; + if ((part1 != null) && !part1.equals("\u0000")) { - firstString = getFirstTokenForRange(getLocalePrefixedText(part1, locale), fieldInstance); - if (firstString == null) + if (fieldInstance.isLocalised()) { - firstString = "{" + locale.getLanguage() + "}"; + firstString = getFirstTokenForRange(getLocalePrefixedText(part1, locale), fieldInstance); + if (firstString == null) + { + firstString = "{" + locale.getLanguage() + "}"; + } + } else + { + firstString = getFirstTokenForRange(part1, fieldInstance); } } else { - firstString = getFirstTokenForRange(part1, fieldInstance); + if (fieldInstance.isLocalised()) + { + firstString = "{" + locale.getLanguage() + "}"; + } else + { + firstString = null; + } } - } else - { - if (fieldInstance.isLocalised()) + + String lastString = null; + if ((part2 != null) && !part2.equals("\uffff")) { - firstString = "{" + locale.getLanguage() + "}"; + if (fieldInstance.isLocalised()) + { + lastString = getFirstTokenForRange(getLocalePrefixedText(part2, locale), fieldInstance); + if (lastString == null) + { + lastString = "{" + locale.getLanguage() + "}\uffff"; + } + } else + { + lastString = getFirstTokenForRange(part2, fieldInstance); + } } else + { + if (fieldInstance.isLocalised()) + { + lastString = "{" + locale.getLanguage() + "}\uffff"; + } else + { + lastString = null; + } + } + + + // * is returned by stringField but it is not correct in case of range queries. + if (firstString != null && firstString.equals("*")) { firstString = null; } - } - String lastString = null; - if ((part2 != null) && !part2.equals("\uffff")) - { - if (fieldInstance.isLocalised()) - { - lastString = getFirstTokenForRange(getLocalePrefixedText(part2, locale), fieldInstance); - if (lastString == null) - { - lastString = "{" + locale.getLanguage() + "}\uffff"; - } - } else - { - lastString = getFirstTokenForRange(part2, fieldInstance); - } - } else - { - if (fieldInstance.isLocalised()) - { - lastString = "{" + locale.getLanguage() + "}\uffff"; - } else + if (lastString != null && lastString.equals("*")) { lastString = null; } + + TermRangeQuery query = new TermRangeQuery(fieldInstance.getField(), + firstString == null ? null : new BytesRef(firstString), + lastString == null ? null : new BytesRef(lastString), includeLower, includeUpper); + booleanQuery.add(query, Occur.SHOULD); } - - - // * is returned by stringField but it is not correct in case of range queries. - if (firstString != null && firstString.equals("*")){ - firstString = null; - } - - if (lastString != null && lastString.equals("*")){ - lastString = null; - } - - TermRangeQuery query = new TermRangeQuery(fieldInstance.getField(), - firstString == null ? null : new BytesRef(firstString), - lastString == null ? null : new BytesRef(lastString), includeLower, includeUpper); - booleanQuery.add(query, Occur.SHOULD); } private String getFirstTokenForRange(String string, FieldInstance field) throws IOException diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/afts/requestHandler/AFTSDefaultTextQueryIT.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/afts/requestHandler/AFTSDefaultTextQueryIT.java index 0994626ec..8bdd3ca17 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/afts/requestHandler/AFTSDefaultTextQueryIT.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/afts/requestHandler/AFTSDefaultTextQueryIT.java @@ -28,16 +28,19 @@ package org.alfresco.solr.query.afts.requestHandler; import org.alfresco.service.cmr.repository.NodeRef; import org.alfresco.service.namespace.QName; -import org.alfresco.solr.client.ContentPropertyValue; +import org.alfresco.solr.AlfrescoSolrDataModel; import org.alfresco.solr.client.MLTextPropertyValue; import org.alfresco.solr.client.PropertyValue; import org.alfresco.solr.client.StringPropertyValue; import org.alfresco.solr.dataload.TestDataProvider; import org.apache.solr.SolrTestCaseJ4; +import org.junit.After; import org.junit.BeforeClass; import org.junit.Test; +import java.lang.reflect.Field; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Locale; import java.util.Map; @@ -64,10 +67,17 @@ import static com.google.common.collect.ImmutableMap.of; @SolrTestCaseJ4.SuppressSSL public class AFTSDefaultTextQueryIT extends AbstractRequestHandlerIT { + // Following data is used for restoring alfrescoSolrDataModel after text execution + private HashSet oldCrossLocaleProperties; + private HashSet oldCrossLocaleTypes; + + static TestDataProvider dataProvider; + + @BeforeClass public static void beforeClass() throws Exception { - TestDataProvider dataProvider = new TestDataProvider(h); + dataProvider = new TestDataProvider(h); List> data = asList( of("name", "test1", @@ -93,7 +103,6 @@ public class AFTSDefaultTextQueryIT extends AbstractRequestHandlerIT TEST_ROOT_NODEREF = dataProvider.getRootNode(); - range(0, data.size()) .forEach(dbId -> { @@ -124,34 +133,402 @@ public class AFTSDefaultTextQueryIT extends AbstractRequestHandlerIT }); } + private void prepareExactSearchData() throws Exception { + TestDataProvider dataProvider = AFTSDefaultTextQueryIT.dataProvider; + + List> exactSearchData = asList( + of("name", "Running", + "description", "Running is a sport is a nice activity", + "content", "when you are running you are doing an amazing sport", + "title", "Running jumping", + "creator", "Alex"), + of("name", "Run", + "description", "you are supposed to run jump", + "content", "after many runs you are tired and if you jump it happens the same", + "title", "Run : a philosophy", + "creator", "Alex"), + of("name", "Poetry", + "description", "a document about poetry and jumpers", + "content", "poetry is unrelated to sport", + "title", "Running jumping twice jumpers", + "creator", "Alex"), + of("name", "Jump", + "description", "a document about jumps", + "content", "runnings jumpings", + "title", "Running", + "creator", "Alex"), + of("name", "Running jumping", + "description", "runners jumpers runs everywhere", + "content", "run is Good as jump", + "title", "Running the art of jumping", + "creator", "Alex")); + + range(0, exactSearchData.size()) + .forEach(dbId -> { + + Map record = exactSearchData.get(dbId); + + String name = record.get("name"); + String description = record.get("description"); + String content = record.get("content"); + String title = record.get("title"); + String creator = record.get("creator"); + + Map properties = new HashMap<>(); + properties.put(PROP_NAME, new StringPropertyValue(name)); + properties.put(PROP_DESCRIPTION, new MLTextPropertyValue(Map.of(Locale.getDefault(), description))); + properties.put(PROP_TITLE, new MLTextPropertyValue(Map.of(Locale.getDefault(), title))); + properties.put(PROP_CREATOR, new StringPropertyValue(creator)); + properties.put(PROP_RATING_SCHEME, new StringPropertyValue(title)); + + Map contents = Map.of(PROP_CONTENT, content); + + addNode(getCore(), + dataModel, 2, dbId+10, 1, + TYPE_CONTENT, null, properties, contents, + "the_owner_of_this_node_is" + name, + null, + new NodeRef[]{ TEST_ROOT_NODEREF }, + new String[]{ "/" + dataProvider.qName("a_qname_for_node_" + name) }, + dataProvider.newNodeRef(), true); + }); + } + + private void changeCrossLocaleEnabledFields() throws NoSuchFieldException, IllegalAccessException { + Field crossLocalePropertiesField = AlfrescoSolrDataModel + .getInstance() + .getClass() + .getDeclaredField("crossLocaleSearchProperties"); + + crossLocalePropertiesField.setAccessible(true); + + Field crossLocaleTypesField = AlfrescoSolrDataModel + .getInstance() + .getClass() + .getDeclaredField("crossLocaleSearchDataTypes"); + + crossLocaleTypesField.setAccessible(true); + + HashSet crossLocaleProperties = (HashSet) crossLocalePropertiesField.get(AlfrescoSolrDataModel.getInstance()); + HashSet crossLocaleTypes = (HashSet) crossLocaleTypesField.get(AlfrescoSolrDataModel.getInstance()); + + oldCrossLocaleTypes = (HashSet) crossLocaleTypes.clone(); + oldCrossLocaleProperties = (HashSet) crossLocaleProperties.clone(); + + crossLocaleTypes.clear(); + crossLocaleProperties.clear(); + crossLocaleProperties.add(PROP_TITLE); + } + + @After + public void restoreAlfrescoSolrDataModel() throws NoSuchFieldException, IllegalAccessException { + if (oldCrossLocaleTypes != null) { + Field crossLocaleTypesField = AlfrescoSolrDataModel + .getInstance() + .getClass() + .getDeclaredField("crossLocaleSearchDataTypes"); + + crossLocaleTypesField.setAccessible(true); + crossLocaleTypesField.set(AlfrescoSolrDataModel.getInstance(), oldCrossLocaleTypes); + } + if (oldCrossLocaleProperties != null) { + Field crossLocalePropertiesField = AlfrescoSolrDataModel + .getInstance() + .getClass() + .getDeclaredField("crossLocaleSearchProperties"); + crossLocalePropertiesField.setAccessible(true); + crossLocalePropertiesField.set(AlfrescoSolrDataModel.getInstance(), oldCrossLocaleProperties); + } + } + + /** + * Test exact search is working. + */ + @Test + public void exactSearch_singleTerm_shouldReturnResultsContainingExactTerm() throws Exception { + prepareExactSearchData(); + /* + * Out of the 5 'run corpus' documents + * 2 results are expected: + * + * - "name", "Run", + * "description", "you are supposed to run jump" + * + * - "name", "Running jumping", + * "content", "run is Good as jump", + */ + assertResponseCardinality("=run", 2); + + /* + * No result for runner, one record has runners, + * you can see the difference between exact search and not + */ + assertResponseCardinality("=runner", 0); + assertResponseCardinality("runner", 1); + + /* + * Out of the 5 'run corpus' documents + * 4 results are expected, only one doc is no fit for the query + * + * "name", "Run", + * "description", "you are supposed to run jump", + * "content", "after many runs you are tired and if you jump it happens the same", + * "title", "Run : a philosophy", + * "creator", "Alex" + */ + assertResponseCardinality("=running", 4); + } + + @Test + public void exactSearch_singleTermInFieldWithNoCrossLocaleEnabled_shouldReturnException() throws Exception { + /* + * Cross Locale is Disabled for: content, cm_description, name + * Cross Locale is Enabled for : cm_title + */ + changeCrossLocaleEnabledFields(); + prepareExactSearchData(); + + assertResponseCardinality("=cm_title:run", 1); + assertResponseException("=content:run", "Exact Term search is not supported unless you configure the field <{http://www.alfresco.org/model/content/1.0}content> for cross locale search"); + assertResponseException("=cm_description:run", "Exact Term search is not supported unless you configure the field <{http://www.alfresco.org/model/content/1.0}description> for cross locale search"); + assertResponseCardinality("=run", 1); + + restoreAlfrescoSolrDataModel(); + } + + @Test + public void exactSearch_singleTermInFieldWithOnlyUnTokenizedAnalysis_shouldReturnFullFieldValueMatch() throws Exception { + /** + * cm_ratingScheme is a copy field un-tokenized of Title, so it has the exact same content but not analysed. + * This means we produce just a token in the index, exactly as the full content. + * We can't expect any search to work except full exact value search + */ + prepareExactSearchData(); + /* + * Out of the 5 'run corpus' documents + * 0 results are expected: + * there is no result that have cm_ratingScheme:"running" + * + */ + assertResponseCardinality("=cm_ratingScheme:running", 0); + assertResponseCardinality("=cm_title:running", 4); + + /* + * Out of the 5 'run corpus' documents + * 1 result is expected: + * + * - "name", "Jump", + * ... + * "title", "Running" + * + */ + assertResponseCardinality("=cm_ratingScheme:Running", 1); + assertResponseCardinality("=cm_title:Running", 4); + + /* + * Out of the 5 'run corpus' documents + * 0 results are expected: + * there is no result that have exactly cm_ratingScheme:"Run" + * The closest we have is record Run (cm_ratingScheme:"Run : a philosophy") + * As you can see we don't have a full match, so it's not in the results. + * + */ + assertResponseCardinality("=cm_ratingScheme:Run", 0); + assertResponseCardinality("=cm_title:Run", 1); + + + } /** * Test exact search is working. */ @Test - public void defaultExactQueryTest() - { + public void exactSearch_multiTerm_shouldReturnResultsContainingExactTerm() throws Exception { + prepareExactSearchData(); + /* + * Out of the 5 'run corpus' documents + * 3 results are expected: + * + * - "name", "Run", + * "description", "you are supposed to run jump", + * + * - "name", "Running jumping", + * "description", "runners jumpers runs everywhere", + * "content", "run is Good as jump", + * + * - "name", "Jump", + * "description", "a document about jumps", + * + */ + assertResponseCardinality("=run =jump", 3); /* - * 3 results expected: - * record 1 ("test" in name, description and content) - * record 2 ("test" in name, and content) - * record 3 ("test" in description) + * No result for runner or jumper, one record has runners, + * and another record has jumpers + * + * - "name", "Poetry", + * "description", "a document about poetry and jumpers", + * - "name", "Running jumping", + * "description", "runners jumpers run everywhere", + * + * you can see the difference between exact search and not */ - assertResponseCardinality("test", 3); + assertResponseCardinality("=runner =jumper", 0); + assertResponseCardinality("runner jumper", 2); /* - * 3 results expected: - * record 2 ("Other" in title ) - * record 3 ("other" in content) - * record 4 ("other" in description) + * Out of the 5 'run corpus' documents + * 4 results are expected: + * Only one doc does't fit: + * - "name", "Run", + * "description", "you are supposed to run jump", + * "content", "after many runs you are tired and if you jump it happens the same", + * "title", "Run : a philosophy", + * "creator", "Alex"), + * + * */ - assertResponseCardinality("Other", 3); + assertResponseCardinality("=running =jumping", 4); + } + + + + @Test + public void exactSearch_multiTermInFieldWithNoCrossLocaleEnabled_shouldReturnException() throws Exception { + /* + * Cross Locale is Disabled for: content, cm_description, name + * Cross Locale is Enabled for : cm_title + */ + changeCrossLocaleEnabledFields(); + prepareExactSearchData(); + + assertResponseCardinality("=cm_title:run =cm_title:jump", 1); + assertResponseException("=content:run =content:jump", "Exact Term search is not supported unless you configure the field <{http://www.alfresco.org/model/content/1.0}content> for cross locale search"); + assertResponseException("=cm_description:run =cm_description:jump", "Exact Term search is not supported unless you configure the field <{http://www.alfresco.org/model/content/1.0}description> for cross locale search"); + assertResponseCardinality("=run =jump", 1); + + restoreAlfrescoSolrDataModel(); + } + + @Test + public void exactSearch_multiTermInFieldWithOnlyUnTokenizedAnalysis_shouldReturnFullFieldValueMatch() throws Exception { + /** + * cm_ratingScheme is a copy field un-tokenized of Title, so it has the exact same content but not analysed. + * This means we produce just a token in the index, exactly as the full content. + * We can't expect any search to work except full exact value search + */ + prepareExactSearchData(); + /* + * Out of the 5 'run corpus' documents + * 0 results are expected: + * there is no result that have cm_ratingScheme:"running" or "jumpers" + * + */ + assertResponseCardinality("=cm_ratingScheme:running =cm_ratingScheme:jumpers", 0); + assertResponseCardinality("=cm_title:running =cm_title:jumpers", 4); + } + + /** + * Test exact search is working. + */ + @Test + public void exactSearch_exactPhrase_shouldReturnResultsContainingExactPhrase() throws Exception { + prepareExactSearchData(); + /* + * Out of the 5 'run corpus' documents + * 1 results are expected: + * + * - "name", "Run", + * "description", "you are supposed to run jump", + * + */ + assertResponseCardinality("=\"run jump\"", 1); /* - * No results expected because creator should not be considered in default text search. + * No result for runner jumper, one record has runners jumpers, + * you can see the difference between exact search and not + * + * "name", "Running jumping", + * "description", "runners jumpers run everywhere", */ - assertResponseCardinality("Giovanni", 0); + assertResponseCardinality("=\"runner jumper\"", 0); + assertResponseCardinality("\"runner jumper\"", 1); + + /* + * Out of the 5 'run corpus' documents + * 3 results are expected: + * + * - "name", "Running", + * ... + * "title", "Running jumping", + * + * - "name", "Poetry", + * "title", "Running jumping twice jumpers" + * + * - "name", "Running jumping", + */ + assertResponseCardinality("=\"running jumping\"", 3); + assertResponseCardinality("\"running jumping\"", 5); + } + + + + @Test + public void exactSearch_phraseInFieldWithNoCrossLocaleEnabled_shouldReturnException() throws Exception { + /* + * Cross Locale is Disabled for: content, cm_description, name + * Cross Locale is Enabled for : cm_title + */ + changeCrossLocaleEnabledFields(); + prepareExactSearchData(); + + assertResponseCardinality("=cm_title:\"running jumping\"", 2); + assertResponseException("=content:\"running jumping\"", "Exact Term search is not supported unless you configure the field <{http://www.alfresco.org/model/content/1.0}content> for cross locale search"); + assertResponseException("=cm_description:\"running jumping\"", "Exact Term search is not supported unless you configure the field <{http://www.alfresco.org/model/content/1.0}description> for cross locale search"); + assertResponseCardinality("=\"running jumping\"", 2); + + restoreAlfrescoSolrDataModel(); + } + + @Test + public void exactSearch_phraseInFieldWithOnlyUnTokenizedAnalysis_shouldReturnFullFieldValueMatch() throws Exception { + /** + * cm_ratingScheme is a copy field un-tokenized of Title, so it has the exact same content but not analysed. + * This means we produce just a token in the index, exactly as the full content. + * We can't expect any search to work except full exact value search + */ + prepareExactSearchData(); + /* + * Out of the 5 'run corpus' documents + * 0 results are expected: + * the closest we got was this one, but it is uppercase + * - "name", "Running", + * "title", "Running jumping", + * + */ + assertResponseCardinality("=cm_ratingScheme:\"running jumping\"", 0); + assertResponseCardinality("=cm_title:\"running jumping\"", 2); + + /* + * Out of the 5 'run corpus' documents + * 1 results are expected: + * - "name", "Running", + * "title", "Running jumping", + * + */ + assertResponseCardinality("=cm_ratingScheme:\"Running jumping\"", 1); + assertResponseCardinality("=cm_title:\"Running jumping\"", 2); + + /* + * Out of the 5 'run corpus' documents + * 0 results are expected: + * the closest we got was this one, but it is uppercase + * - "name", "Poetry", + * "title", "Running jumping twice jumpers", + * + */ + assertResponseCardinality("=cm_ratingScheme:\"Running jumping twice\"", 0); + assertResponseCardinality("=cm_title:\"Running jumping twice\"", 1); } @@ -175,7 +552,7 @@ public class AFTSDefaultTextQueryIT extends AbstractRequestHandlerIT * record 3 ("other" in content) * record 4 ("other" in description) */ - assertResponseCardinality("other to otherz ", 3); + assertResponseCardinality("[other to otherz]", 3); } diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/afts/requestHandler/AbstractRequestHandlerIT.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/afts/requestHandler/AbstractRequestHandlerIT.java index 8b3cac72e..ac44c1b71 100644 --- a/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/afts/requestHandler/AbstractRequestHandlerIT.java +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/query/afts/requestHandler/AbstractRequestHandlerIT.java @@ -37,6 +37,9 @@ import java.util.stream.Stream; import static java.util.Arrays.asList; import static java.util.stream.IntStream.range; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.Assert.fail; public abstract class AbstractRequestHandlerIT extends AbstractAlfrescoSolrIT { @@ -52,6 +55,16 @@ public abstract class AbstractRequestHandlerIT extends AbstractAlfrescoSolrIT assertQ(areq(params("rows", "20", "qt", "/afts", "fq", "{!afts}AUTHORITY_FILTER_FROM_JSON", "q", query), json), "*[count(//doc)="+num+"]"); } + void assertResponseException(String query, String exceptionMessage) { + try { + h.query(areq(params("rows", "20", "qt", "/afts", "q", query), null)); + fail("No Exception Thrown"); + } catch (Exception e) { + assertThat(e.getMessage(), is(exceptionMessage)); + } + + } + void assertResponseCardinality(String query, int num) { assertQ(areq(params("rows", "20", "qt", "/afts", "q", query), null), "*[count(//doc)="+num+"]"); From b83bcb334bf8c0f353711a6e9226797047fef8f2 Mon Sep 17 00:00:00 2001 From: Andrea Gazzarini Date: Mon, 8 Jun 2020 09:05:01 +0100 Subject: [PATCH 5/5] [ SEARCH-2223 ] First implementation on FIX report (always goes through repo for asking nodes count) --- .../solr/admin/SolrE2eAdminTest.java | 57 +- .../solr/AlfrescoCoreAdminHandler.java | 352 ++++++++++-- .../alfresco/solr/SolrInformationServer.java | 2 +- .../templates/rerank/conf/solrcore.properties | 5 + .../solr/AlfrescoCoreAdminHandlerTest.java | 512 ++++++++++++++++++ 5 files changed, 846 insertions(+), 82 deletions(-) create mode 100644 search-services/alfresco-search/src/test/java/org/alfresco/solr/AlfrescoCoreAdminHandlerTest.java diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/solr/admin/SolrE2eAdminTest.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/solr/admin/SolrE2eAdminTest.java index b82f25b0e..2e59a6bc5 100644 --- a/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/solr/admin/SolrE2eAdminTest.java +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/searchServices/solr/admin/SolrE2eAdminTest.java @@ -29,6 +29,7 @@ package org.alfresco.test.search.functional.searchServices.solr.admin; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.Map; import org.alfresco.rest.core.RestResponse; import org.alfresco.search.TestGroup; @@ -603,56 +604,48 @@ public class SolrE2eAdminTest extends AbstractE2EFunctionalTest Assert.assertEquals(actionStatus, "scheduled"); } - /** - * FIX for every core. - * @throws Exception - */ - @Test(priority = 27, dependsOnMethods = "testPurge") - public void testFix() throws Exception - { - RestResponse response = restClient.withSolrAdminAPI().getAction("fix"); - - checkResponseStatusOk(response); - - DEFAULT_CORE_NAMES.forEach(core -> { - List txToReindex = response.getResponse().body().jsonPath().get("action." + core +".txToReindex"); - Assert.assertTrue(txToReindex.size() >= 0, "Expected a list of transactions (or empty list) to be reindexed,"); - List aclToReindex = response.getResponse().body().jsonPath().get("action." + core + ".aclChangeSetToReindex"); - Assert.assertTrue(aclToReindex.size() >= 0, "Expected a list of ACLs (or empty list) to be reindexed,"); - }); - - String actionStatus = response.getResponse().body().jsonPath().get("action.status"); - Assert.assertEquals(actionStatus, "scheduled"); - } - /** * FIX for specific core. - * @throws Exception + * The test checks the response structure in order to make sure the expected sections are present. + * + * We are not testing the content of each section because due to the underlying E2E infrastructure, we cannot know + * in advance the transactions that will be scheduled for reindexing. */ @Test(priority = 28) - public void testFixCore() throws Exception + public void testFixCore() { DEFAULT_CORE_NAMES.forEach(core -> { - try { RestResponse response = restClient.withParams("core=" + core).withSolrAdminAPI().getAction("fix"); checkResponseStatusOk(response); - List txToReindex = response.getResponse().body().jsonPath().get("action." + core +".txToReindex"); - Assert.assertTrue(txToReindex.size() >= 0, "Expected a list of transactions (or empty list) to be reindexed,"); - List aclToReindex = response.getResponse().body().jsonPath().get("action." + core + ".aclChangeSetToReindex"); - Assert.assertTrue(aclToReindex.size() >= 0, "Expected a list of ACLs (or empty list) to be reindexed,"); - + Map txInIndexNotInDb = response.getResponse().body().jsonPath().get("action." + core +".txToReindex.txInIndexNotInDb"); + Assert.assertNotNull(txInIndexNotInDb, "Expected a list of transactions (even empty) that are in index but not in the database to be reindexed,"); + + Map duplicatedTx = response.getResponse().body().jsonPath().get("action." + core +".txToReindex.duplicatedTxInIndex"); + Assert.assertNotNull(duplicatedTx, "Expected a list of duplicated transactions (even empty) to be reindexed,"); + + Map missingTx = response.getResponse().body().jsonPath().get("action." + core +".txToReindex.missingTxInIndex"); + Assert.assertNotNull(missingTx, "Expected a list of missing transactions (or empty list) to be reindexed,"); + + Map aclTxInIndexNotInDb = response.getResponse().body().jsonPath().get("action." + core + ".aclChangeSetToReindex.aclTxInIndexNotInDb"); + Assert.assertNotNull(aclTxInIndexNotInDb, "Expected a list of ACLs (or empty list) to be reindexed,"); + + Map duplicatedAclTxInIndex = response.getResponse().body().jsonPath().get("action." + core + ".aclChangeSetToReindex.duplicatedAclTxInIndex"); + Assert.assertNotNull(duplicatedAclTxInIndex, "Expected a list of ACLs (or empty list) to be reindexed,"); + + Map missingAclTxInIndex = response.getResponse().body().jsonPath().get("action." + core + ".aclChangeSetToReindex.missingAclTxInIndex"); + Assert.assertNotNull(missingAclTxInIndex, "Expected a list of ACLs (or empty list) to be reindexed,"); + String actionStatus = response.getResponse().body().jsonPath().get("action.status"); - Assert.assertEquals(actionStatus, "scheduled"); + Assert.assertEquals(actionStatus, "notScheduled"); } catch (Exception e) { throw new RuntimeException(e); } - }); } diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoCoreAdminHandler.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoCoreAdminHandler.java index 18ccb6d6f..0f19165e0 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoCoreAdminHandler.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/AlfrescoCoreAdminHandler.java @@ -54,6 +54,7 @@ import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.SimpleOrderedMap; import org.apache.solr.core.CoreContainer; import org.apache.solr.core.SolrCore; +import org.apache.solr.core.SolrResourceLoader; import org.apache.solr.handler.admin.CoreAdminHandler; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.response.SolrQueryResponse; @@ -76,16 +77,19 @@ import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Properties; -import java.util.Set; -import java.util.TreeSet; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Consumer; +import java.util.function.LongToIntFunction; import java.util.stream.Collectors; import static java.util.Arrays.asList; import static java.util.Arrays.stream; +import static java.util.Optional.of; import static java.util.Optional.ofNullable; +import static org.alfresco.repo.search.adaptor.lucene.QueryConstants.FIELD_INACLTXID; +import static org.alfresco.repo.search.adaptor.lucene.QueryConstants.FIELD_INTXID; import static org.alfresco.solr.HandlerOfResources.extractCustomProperties; import static org.alfresco.solr.HandlerOfResources.getSafeBoolean; import static org.alfresco.solr.HandlerOfResources.getSafeLong; @@ -99,6 +103,7 @@ import static org.alfresco.solr.HandlerReportHelper.buildAclTxReport; import static org.alfresco.solr.HandlerReportHelper.buildNodeReport; import static org.alfresco.solr.HandlerReportHelper.buildTrackerReport; import static org.alfresco.solr.HandlerReportHelper.buildTxReport; +import static org.alfresco.solr.utils.Utils.isNullOrEmpty; import static org.alfresco.solr.utils.Utils.notNullOrEmpty; /** @@ -160,22 +165,39 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler */ private static final String ACTION_STATUS_SUCCESS = "success"; private static final String ACTION_STATUS_ERROR = "error"; - private static final String ACTION_STATUS_SCHEDULED = "scheduled"; + static final String ACTION_STATUS_SCHEDULED = "scheduled"; + static final String ACTION_STATUS_NOT_SCHEDULED = "notScheduled"; + + static final String DRY_RUN_PARAMETER_NAME = "dryRun"; + static final String FROM_TX_COMMIT_TIME_PARAMETER_NAME = "fromTxCommitTime"; + static final String TO_TX_COMMIT_TIME_PARAMETER_NAME = "toTxCommitTime"; + static final String MAX_TRANSACTIONS_TO_SCHEDULE_PARAMETER_NAME = "maxScheduledTransactions"; + static final String MAX_TRANSACTIONS_TO_SCHEDULE_CONF_PROPERTY_NAME = "alfresco.admin.fix.maxScheduledTransactions"; + static final String TX_IN_INDEX_NOT_IN_DB = "txInIndexNotInDb"; + static final String DUPLICATED_TX_IN_INDEX = "duplicatedTxInIndex"; + static final String MISSING_TX_IN_INDEX = "missingTxInIndex"; + static final String ACL_TX_IN_INDEX_NOT_IN_DB = "aclTxInIndexNotInDb"; + static final String DUPLICATED_ACL_TX_IN_INDEX = "duplicatedAclTxInIndex"; + static final String MISSING_ACL_TX_IN_INDEX = "missingAclTxInIndex"; /** * JSON/XML labels for the Action response */ private static final String ACTION_LABEL = "action"; - private static final String ACTION_STATUS_LABEL = "status"; - private static final String ACTION_ERROR_MESSAGE_LABEL = "errorMessage"; + static final String ACTION_STATUS_LABEL = "status"; + + static final String ACTION_ERROR_MESSAGE_LABEL = "errorMessage"; + static final String UNKNOWN_CORE_MESSAGE = "Unknown core:"; + static final String UNPROCESSABLE_REQUEST_ON_SLAVE_NODES = "Requested action cannot be performed on slave nodes."; + private static final String ACTION_TX_TO_REINDEX = "txToReindex"; private static final String ACTION_ACL_CHANGE_SET_TO_REINDEX = "aclChangeSetToReindex"; private SolrTrackerScheduler scheduler; - private TrackerRegistry trackerRegistry; - private ConcurrentHashMap informationServers; + TrackerRegistry trackerRegistry; + ConcurrentHashMap informationServers; - private static List CORE_PARAMETER_NAMES = asList(CoreAdminParams.CORE, "coreName", "index"); + private final static List CORE_PARAMETER_NAMES = asList(CoreAdminParams.CORE, "coreName", "index"); public AlfrescoCoreAdminHandler() { @@ -938,8 +960,7 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler * * Synchronous execution * - * @param req Query Request without parameters - * - coreName, optional, the name of the core to be checked + * @param cname, optional, the name of the core to be checked * * @return Response including the action result: * - status: success, when the core has been created @@ -1366,7 +1387,6 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler * - fromAclTx, optional: from ACL transaction Id to filter report results * - toCalTx, optional: to ACL transaction Id to filter report results * - * @param Response including the action result: * - report.core: multiple Objects with the details of the report ("core" is the name of the Core) * * @throws JSONException @@ -1497,7 +1517,6 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler * * @param params Query Request with following parameters: * - core, optional: The name of the SOLR Core - * @param rsp Query Response including the action result: * - action.status: scheduled, as it will be executed by Trackers on the next maintenance operation * - core: list of Document Ids with error that are going to reindexed */ @@ -1594,66 +1613,140 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler * - action.status: scheduled, as it will be executed by Trackers on the next maintenance operation * - txToReindex: list of Transaction Ids that are going to be reindexed * - aclChangeSetToReindex: list of ACL Change Set Ids that are going to be reindexed - * @throws JSONException */ - private NamedList actionFIX(SolrParams params) throws JSONException + NamedList actionFIX(SolrParams params) throws JSONException { String requestedCoreName = coreName(params); + var wrapper = new Object() { - NamedList response = new SimpleOrderedMap<>();; + final NamedList response = new SimpleOrderedMap<>(); }; + if (isNullOrEmpty(requestedCoreName)) + { + return wrapper.response; + } + + if (!coreNames().contains(requestedCoreName)) + { + wrapper.response.add(ACTION_ERROR_MESSAGE_LABEL, UNKNOWN_CORE_MESSAGE + requestedCoreName); + return wrapper.response; + } + + if (!isMasterOrStandalone(requestedCoreName)) { + wrapper.response.add(ACTION_ERROR_MESSAGE_LABEL, UNPROCESSABLE_REQUEST_ON_SLAVE_NODES); + return wrapper.response; + } + + Long fromTxCommitTime = params.getLong(FROM_TX_COMMIT_TIME_PARAMETER_NAME); + Long toTxCommitTime = params.getLong(TO_TX_COMMIT_TIME_PARAMETER_NAME); + boolean dryRun = params.getBool(DRY_RUN_PARAMETER_NAME, true); + int maxTransactionsToSchedule = getMaxTransactionToSchedule(params); + + LOGGER.debug("FIX Admin request on core {}, parameters: " + + FROM_TX_COMMIT_TIME_PARAMETER_NAME + " = {}, " + + TO_TX_COMMIT_TIME_PARAMETER_NAME + " = {}, " + + DRY_RUN_PARAMETER_NAME + " = {}, " + + MAX_TRANSACTIONS_TO_SCHEDULE_PARAMETER_NAME + " = {}", + requestedCoreName, + ofNullable(fromTxCommitTime).map(Object::toString).orElse("N.A."), + ofNullable(toTxCommitTime).map(Object::toString).orElse("N.A."), + dryRun, + maxTransactionsToSchedule); + coreNames().stream() .filter(coreName -> requestedCoreName == null || coreName.equals(requestedCoreName)) .filter(this::isMasterOrStandalone) - .forEach(coreName -> { - wrapper.response.add(coreName, fixOnSpecificCore(coreName)); - }); + .forEach(coreName -> + wrapper.response.add( + coreName, + fixOnSpecificCore(coreName, fromTxCommitTime, toTxCommitTime, dryRun, maxTransactionsToSchedule))); + + if (wrapper.response.size() > 0) + { + wrapper.response.add(DRY_RUN_PARAMETER_NAME, dryRun); + + ofNullable(fromTxCommitTime).ifPresent(value -> wrapper.response.add(FROM_TX_COMMIT_TIME_PARAMETER_NAME, value)); + ofNullable(toTxCommitTime).ifPresent(value -> wrapper.response.add(TO_TX_COMMIT_TIME_PARAMETER_NAME, value)); + + wrapper.response.add(MAX_TRANSACTIONS_TO_SCHEDULE_PARAMETER_NAME, maxTransactionsToSchedule); + wrapper.response.add(ACTION_STATUS_LABEL, dryRun ? ACTION_STATUS_NOT_SCHEDULED : ACTION_STATUS_SCHEDULED); + } - wrapper.response.add(ACTION_STATUS_LABEL, ACTION_STATUS_SCHEDULED); return wrapper.response; } - private NamedList fixOnSpecificCore(String coreName) + /** + * Detects the transactions that need a FIX (i.e. reindexing) because the following reasons: + * + *
    + *
  • A transaction is in the index but not in repository
    • + *
  • A transaction is duplicated in the index
    • + *
  • A transaction is missing in the index
    • + *
+ * + * Depending on the dryRun parameter, other than collecting, this method could also schedule the transactions for + * reindexing. + * + * @param coreName the target core name. + * @param fromTxCommitTime the start commit time we consider for collecting transaction. + * @param toTxCommitTime the end commit time we consider for collecting transaction. + * @param dryRun a flag indicating if the collected transactions must be actually scheduled for reindexing. + * @param maxTransactionsToSchedule the maximum number of transactions to be scheduled for reindexing. + * @return a report about transactions that need to be fixed. + */ + NamedList fixOnSpecificCore(String coreName, Long fromTxCommitTime, Long toTxCommitTime, boolean dryRun, int maxTransactionsToSchedule) { try { - // Gets Metadata health and fixes any problems MetadataTracker metadataTracker = trackerRegistry.getTrackerForCore(coreName, MetadataTracker.class); - IndexHealthReport indexHealthReport = metadataTracker.checkIndex(null, null, null); - IOpenBitSet toReindex = indexHealthReport.getTxInIndexButNotInDb(); - toReindex.or(indexHealthReport.getDuplicatedTxInIndex()); - toReindex.or(indexHealthReport.getMissingTxFromIndex()); - long current = -1; - // Goes through problems in the index - Set txToReindex = new TreeSet<>(); - while ((current = toReindex.nextSetBit(current + 1)) != -1) - { - metadataTracker.addTransactionToReindex(current); - txToReindex.add(current); - } + final IndexHealthReport metadataTrackerIndexHealthReport = + metadataTracker.checkIndex(null, fromTxCommitTime, toTxCommitTime); + + LOGGER.debug("FIX Admin action built the MetadataTracker Index Health Report on core {}, parameters: " + + FROM_TX_COMMIT_TIME_PARAMETER_NAME + " = {}, " + + TO_TX_COMMIT_TIME_PARAMETER_NAME + " = {}, " + + DRY_RUN_PARAMETER_NAME + " = {}, " + + MAX_TRANSACTIONS_TO_SCHEDULE_PARAMETER_NAME + " = {}", + coreName, + ofNullable(fromTxCommitTime).map(Object::toString).orElse("N.A."), + ofNullable(toTxCommitTime).map(Object::toString).orElse("N.A."), + dryRun, + maxTransactionsToSchedule); - // Gets the Acl health and fixes any problems AclTracker aclTracker = trackerRegistry.getTrackerForCore(coreName, AclTracker.class); - indexHealthReport = aclTracker.checkIndex(null, null, null); - toReindex = indexHealthReport.getAclTxInIndexButNotInDb(); - toReindex.or(indexHealthReport.getDuplicatedAclTxInIndex()); - toReindex.or(indexHealthReport.getMissingAclTxFromIndex()); - current = -1; - // Goes through the problems in the index - Set aclChangeSetToReindex = new TreeSet<>(); - while ((current = toReindex.nextSetBit(current + 1)) != -1) - { - aclTracker.addAclChangeSetToReindex(current); - aclChangeSetToReindex.add(current); - } + final IndexHealthReport aclTrackerIndexHealthReport = + aclTracker.checkIndex(null, fromTxCommitTime, toTxCommitTime); + + LOGGER.debug("FIX Admin action built the AclTracker Index Health Report on core {}, parameters: " + + FROM_TX_COMMIT_TIME_PARAMETER_NAME + " = {}, " + + TO_TX_COMMIT_TIME_PARAMETER_NAME + " = {}, " + + DRY_RUN_PARAMETER_NAME + " = {}, " + + MAX_TRANSACTIONS_TO_SCHEDULE_PARAMETER_NAME + " = {}", + coreName, + ofNullable(fromTxCommitTime).map(Object::toString).orElse("N.A."), + ofNullable(toTxCommitTime).map(Object::toString).orElse("N.A."), + dryRun, + maxTransactionsToSchedule); NamedList response = new SimpleOrderedMap<>(); - response.add(ACTION_TX_TO_REINDEX, txToReindex); - response.add(ACTION_ACL_CHANGE_SET_TO_REINDEX, aclChangeSetToReindex); - return response; + response.add(ACTION_TX_TO_REINDEX, + txToReindex( + coreName, + metadataTracker, + metadataTrackerIndexHealthReport, + dryRun ? txid -> {} : metadataTracker::addTransactionToReindex, + maxTransactionsToSchedule)); + response.add(ACTION_ACL_CHANGE_SET_TO_REINDEX, + aclTxToReindex( + coreName, + aclTracker, + aclTrackerIndexHealthReport, + dryRun ? txid -> {} : aclTracker::addAclChangeSetToReindex, + maxTransactionsToSchedule)); + return response; } catch(Exception exception) { @@ -1661,6 +1754,153 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler } } + /** + * Detects the transactions that need a FIX (i.e. reindexing) because the following reasons: + * + *
    + *
  • A transaction is in the index but not in repository
    • + *
  • A transaction is duplicated in the index
    • + *
  • A transaction is missing in the index
    • + *
+ * + * Note: the method, as a side effect, could also schedule the detected transactions for reindexing. + * That is controlled by the scheduler input param (which is directly connected with the FIX tool "dryRun" parameter). + * + * @param coreName the target core name. + * @param tracker the {@link MetadataTracker} instance associated with the target core. + * @param report the index healt report produced by the tracker. + * @param scheduler the controller which manages the actual transaction scheduling. + * @param maxTransactionsToSchedule the maximum number of transactions to schedule for reindexing. + * @return a report which includes the transactions that need a reindexing. + * @see SEARCH-2233 + * @see SEARCH-2248 + */ + NamedList txToReindex( + String coreName, + MetadataTracker tracker, + final IndexHealthReport report, + Consumer scheduler, + int maxTransactionsToSchedule) + { + final AtomicInteger globalLimit = new AtomicInteger(maxTransactionsToSchedule); + + final LongToIntFunction retrieveTransactionRelatedNodesCountFromRepository = + txid -> notNullOrEmpty(tracker.getFullNodesForDbTransaction(txid)).size(); + + final LongToIntFunction retrieveTransactionRelatedNodesCountFromIndex = + txid -> of(getInformationServers().get(coreName)) + .map(SolrInformationServer.class::cast) + .map(server -> server.getDocListSize(FIELD_INTXID + ":" + txid)) + .orElse(0); + + NamedList txToReindex = new SimpleOrderedMap<>(); + txToReindex.add(TX_IN_INDEX_NOT_IN_DB, + manageTransactionsToBeFixed( + report.getTxInIndexButNotInDb(), + retrieveTransactionRelatedNodesCountFromIndex, + scheduler, + globalLimit)); + + txToReindex.add(DUPLICATED_TX_IN_INDEX, + manageTransactionsToBeFixed( + report.getDuplicatedTxInIndex(), + retrieveTransactionRelatedNodesCountFromIndex, + scheduler, + globalLimit)); + + txToReindex.add(MISSING_TX_IN_INDEX, + manageTransactionsToBeFixed( + report.getMissingTxFromIndex(), + retrieveTransactionRelatedNodesCountFromRepository, + scheduler, + globalLimit)); + return txToReindex; + } + + /** + * Detects the ACL transactions that need a FIX (i.e. reindexing) because the following reasons: + * + *
    + *
  • A transaction is in the index but not in repository
    • + *
  • A transaction is duplicated in the index
    • + *
  • A transaction is missing in the index
    • + *
+ * + * This method is almost the same as {@link #txToReindex(String, MetadataTracker, IndexHealthReport, Consumer, int)}. + * The main difference is the target tracker ({@link AclTracker} in this case, instead of {@link MetadataTracker}). + * + * Note: the method, as a side effect, could also schedule the detected transactions for reindexing. + * That is controlled by the scheduler input param (which is directly connected with the FIX tool "dryRun" parameter). + * + * @param coreName the target core name. + * @param tracker the {@link AclTracker} instance associated with the target core. + * @param report the index healt report produced by the tracker. + * @param scheduler the controller which manages the actual transaction scheduling. + * @return a report which includes the transactions that need a reindexing. + * @see SEARCH-2233 + * @see SEARCH-2248 + */ + NamedList aclTxToReindex( + String coreName, + AclTracker tracker, + final IndexHealthReport report, + Consumer scheduler, + int maxTransactionsToSchedule) + { + final AtomicInteger globalLimit = new AtomicInteger(maxTransactionsToSchedule); + + final LongToIntFunction retrieveAclTransactionRelatedNodesCountFromRepository = + txid -> notNullOrEmpty(tracker.getAclsForDbAclTransaction(txid)).size(); + + final LongToIntFunction retrieveAclTransactionRelatedNodesCountFromIndex = + txid -> of(getInformationServers().get(coreName)) + .map(SolrInformationServer.class::cast) + .map(server -> server.getDocListSize(FIELD_INACLTXID + ":" + txid)) + .orElse(0); + + NamedList aclTxToReindex = new SimpleOrderedMap<>(); + aclTxToReindex.add(ACL_TX_IN_INDEX_NOT_IN_DB, + manageTransactionsToBeFixed( + report.getAclTxInIndexButNotInDb(), + retrieveAclTransactionRelatedNodesCountFromIndex, + scheduler, + globalLimit)); + + aclTxToReindex.add(DUPLICATED_ACL_TX_IN_INDEX, + manageTransactionsToBeFixed( + report.getDuplicatedAclTxInIndex(), + retrieveAclTransactionRelatedNodesCountFromIndex, + scheduler, + globalLimit)); + + aclTxToReindex.add(MISSING_ACL_TX_IN_INDEX, + manageTransactionsToBeFixed( + report.getMissingAclTxFromIndex(), + retrieveAclTransactionRelatedNodesCountFromRepository, + scheduler, + globalLimit)); + + return aclTxToReindex; + } + + NamedList manageTransactionsToBeFixed( + IOpenBitSet transactions, + LongToIntFunction nodesCounter, + Consumer scheduler, + AtomicInteger limit) + { + final NamedList transactionsList = new SimpleOrderedMap<>(); + + long txid = -1; + while ((txid = transactions.nextSetBit(txid + 1)) != -1 && limit.decrementAndGet() >= 0) + { + transactionsList.add(String.valueOf(txid), nodesCounter.applyAsInt(txid)); + scheduler.accept(txid); + } + + return transactionsList; + } + /** * Get detailed report for a core or for every core including information * related with handlers and trackers. @@ -1830,4 +2070,18 @@ public class AlfrescoCoreAdminHandler extends CoreAdminHandler .findFirst() .orElse(null); } + + int getMaxTransactionToSchedule(SolrParams params) + { + String requestedCoreName = coreName(params); + return ofNullable(params.getInt(MAX_TRANSACTIONS_TO_SCHEDULE_PARAMETER_NAME)) + .orElseGet(() -> + ofNullable(coreContainer) + .map(container -> container.getCore(requestedCoreName)) + .map(SolrCore::getResourceLoader) + .map(SolrResourceLoader::getCoreProperties) + .map(conf -> conf.getProperty(MAX_TRANSACTIONS_TO_SCHEDULE_CONF_PROPERTY_NAME)) + .map(Integer::parseInt) + .orElse(Integer.MAX_VALUE)); // Last fallback if we don't have a request param and a value in configuration + } } \ No newline at end of file diff --git a/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java b/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java index fbed07f15..8e61a7962 100644 --- a/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java +++ b/search-services/alfresco-search/src/main/java/org/alfresco/solr/SolrInformationServer.java @@ -3632,7 +3632,7 @@ public class SolrInformationServer implements InformationServer return (NamedList) facetFields.get(field); } - private int getDocListSize(String query) + public int getDocListSize(String query) { try (SolrQueryRequest request = this.newSolrQueryRequest()) { diff --git a/search-services/alfresco-search/src/main/resources/solr/instance/templates/rerank/conf/solrcore.properties b/search-services/alfresco-search/src/main/resources/solr/instance/templates/rerank/conf/solrcore.properties index 8cebb969b..af5186f84 100644 --- a/search-services/alfresco-search/src/main/resources/solr/instance/templates/rerank/conf/solrcore.properties +++ b/search-services/alfresco-search/src/main/resources/solr/instance/templates/rerank/conf/solrcore.properties @@ -11,6 +11,11 @@ enable.alfresco.tracking=true #data.dir.store=workspace/SpacesStore #alfresco.stores=workspace://SpacesStore +# Max transaction to schedule for reindexing in the admin FIX tool. +# If the admin fix action is called with a "maxScheduledTransactions" request parameter +# then that is used (i.e. the value configured in this file is ignored). +alfresco.admin.fix.maxScheduledTransactions=500 + # # Properties loaded during alfresco tracking # diff --git a/search-services/alfresco-search/src/test/java/org/alfresco/solr/AlfrescoCoreAdminHandlerTest.java b/search-services/alfresco-search/src/test/java/org/alfresco/solr/AlfrescoCoreAdminHandlerTest.java new file mode 100644 index 000000000..3a8c7ef8f --- /dev/null +++ b/search-services/alfresco-search/src/test/java/org/alfresco/solr/AlfrescoCoreAdminHandlerTest.java @@ -0,0 +1,512 @@ +/* + * #%L + * Alfresco Search Services + * %% + * Copyright (C) 2005 - 2020 Alfresco Software Limited + * %% + * This file is part of the Alfresco software. + * If the software was purchased under a paid Alfresco license, the terms of + * the paid license agreement will prevail. Otherwise, the software is + * provided under the following open source license terms: + * + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + * #L% + */ + +package org.alfresco.solr; + +import org.alfresco.solr.adapters.IOpenBitSet; +import org.alfresco.solr.adapters.SolrOpenBitSetAdapter; +import org.alfresco.solr.tracker.AclTracker; +import org.alfresco.solr.tracker.IndexHealthReport; +import org.alfresco.solr.tracker.MetadataTracker; +import org.alfresco.solr.tracker.TrackerRegistry; +import org.apache.solr.common.params.ModifiableSolrParams; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.core.CoreContainer; +import org.apache.solr.core.SolrCore; +import org.apache.solr.core.SolrResourceLoader; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnitRunner; + +import java.util.Properties; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Consumer; + +import static java.util.Optional.of; +import static java.util.stream.IntStream.range; +import static org.alfresco.solr.AlfrescoCoreAdminHandler.ACL_TX_IN_INDEX_NOT_IN_DB; +import static org.alfresco.solr.AlfrescoCoreAdminHandler.ACTION_ERROR_MESSAGE_LABEL; +import static org.alfresco.solr.AlfrescoCoreAdminHandler.ACTION_STATUS_LABEL; +import static org.alfresco.solr.AlfrescoCoreAdminHandler.ACTION_STATUS_NOT_SCHEDULED; +import static org.alfresco.solr.AlfrescoCoreAdminHandler.ALFRESCO_CORE_NAME; +import static org.alfresco.solr.AlfrescoCoreAdminHandler.ARCHIVE_CORE_NAME; +import static org.alfresco.solr.AlfrescoCoreAdminHandler.DRY_RUN_PARAMETER_NAME; +import static org.alfresco.solr.AlfrescoCoreAdminHandler.DUPLICATED_ACL_TX_IN_INDEX; +import static org.alfresco.solr.AlfrescoCoreAdminHandler.DUPLICATED_TX_IN_INDEX; +import static org.alfresco.solr.AlfrescoCoreAdminHandler.FROM_TX_COMMIT_TIME_PARAMETER_NAME; +import static org.alfresco.solr.AlfrescoCoreAdminHandler.MAX_TRANSACTIONS_TO_SCHEDULE_PARAMETER_NAME; +import static org.alfresco.solr.AlfrescoCoreAdminHandler.MAX_TRANSACTIONS_TO_SCHEDULE_CONF_PROPERTY_NAME; +import static org.alfresco.solr.AlfrescoCoreAdminHandler.MISSING_ACL_TX_IN_INDEX; +import static org.alfresco.solr.AlfrescoCoreAdminHandler.MISSING_TX_IN_INDEX; +import static org.alfresco.solr.AlfrescoCoreAdminHandler.TO_TX_COMMIT_TIME_PARAMETER_NAME; +import static org.alfresco.solr.AlfrescoCoreAdminHandler.TX_IN_INDEX_NOT_IN_DB; +import static org.alfresco.solr.AlfrescoCoreAdminHandler.UNKNOWN_CORE_MESSAGE; +import static org.alfresco.solr.AlfrescoCoreAdminHandler.UNPROCESSABLE_REQUEST_ON_SLAVE_NODES; +import static org.apache.solr.common.params.CoreAdminParams.CORE; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +@RunWith(MockitoJUnitRunner.class) +public class AlfrescoCoreAdminHandlerTest +{ + private AlfrescoCoreAdminHandler admin; + + @Mock + TrackerRegistry registry; + + private ModifiableSolrParams params; + + @Before + public void setUp() + { + admin = new AlfrescoCoreAdminHandler(); + admin.trackerRegistry = registry; + when(registry.getCoreNames()).thenReturn(Set.of(ALFRESCO_CORE_NAME, ARCHIVE_CORE_NAME)); + + params = new ModifiableSolrParams(); + } + + @Test + public void noTargetCoreInParams() + { + assertEquals(0, params.size()); + + NamedList actionResponse = admin.actionFIX(params); + assertEquals(0, actionResponse.size()); + } + + @Test + public void unknownTargetCoreInParams() + { + String invalidCoreName = "thisIsAnInvalidOrAtLeastUnknownCoreName"; + params.set(CORE, invalidCoreName); + + NamedList actionResponse = admin.actionFIX(params); + assertEquals( 1, actionResponse.size()); + assertEquals(UNKNOWN_CORE_MESSAGE + invalidCoreName, actionResponse.get(ACTION_ERROR_MESSAGE_LABEL)); + } + + @Test + public void fixOnSlaveNodeHasNoEffect() + { + params.set(CORE, ALFRESCO_CORE_NAME); + + assertFalse(admin.isMasterOrStandalone(ALFRESCO_CORE_NAME)); + + NamedList actionResponse = admin.actionFIX(params); + assertEquals( 1, actionResponse.size()); + assertEquals(UNPROCESSABLE_REQUEST_ON_SLAVE_NODES, actionResponse.get(ACTION_ERROR_MESSAGE_LABEL)); + } + + @Test + public void maxTransactionScheduledParameterIsNotNull() + { + int expectedMaxTransactionToSchedule = 12876; + + params.set(CORE, ALFRESCO_CORE_NAME); + params.set(MAX_TRANSACTIONS_TO_SCHEDULE_PARAMETER_NAME, expectedMaxTransactionToSchedule); + + admin = new AlfrescoCoreAdminHandler(); + + assertEquals(expectedMaxTransactionToSchedule, admin.getMaxTransactionToSchedule(params)); + } + + @Test + public void maxTransactionScheduledIsNull_shouldBeGatheredFromCoreProperties() + { + params.set(CORE, ALFRESCO_CORE_NAME); + + int expectedMaxTransactionToSchedule = 17892; + Properties coreProperties = new Properties(); + coreProperties.setProperty( + MAX_TRANSACTIONS_TO_SCHEDULE_CONF_PROPERTY_NAME, + String.valueOf(expectedMaxTransactionToSchedule)); + + CoreContainer coreContainer = mock(CoreContainer.class); + SolrCore core = mock(SolrCore.class); + SolrResourceLoader resourceLoader = mock(SolrResourceLoader.class); + when(coreContainer.getCore(ALFRESCO_CORE_NAME)).thenReturn(core); + when(core.getResourceLoader()).thenReturn(resourceLoader); + when(resourceLoader.getCoreProperties()).thenReturn(coreProperties); + + admin = new AlfrescoCoreAdminHandler(coreContainer); + + assertEquals(expectedMaxTransactionToSchedule, admin.getMaxTransactionToSchedule(params)); + } + + @Test + public void maxTransactionScheduledParameterAndConfigurationIsNull_shouldGetTheHardCodedDefault() + { + params.set(CORE, ALFRESCO_CORE_NAME); + + Properties coreProperties = new Properties(); + + CoreContainer coreContainer = mock(CoreContainer.class); + SolrCore core = mock(SolrCore.class); + SolrResourceLoader resourceLoader = mock(SolrResourceLoader.class); + when(coreContainer.getCore(ALFRESCO_CORE_NAME)).thenReturn(core); + when(core.getResourceLoader()).thenReturn(resourceLoader); + when(resourceLoader.getCoreProperties()).thenReturn(coreProperties); + + admin = new AlfrescoCoreAdminHandler(coreContainer); + + assertEquals(Integer.MAX_VALUE, admin.getMaxTransactionToSchedule(params)); + } + + @Test + public void masterOrStandaloneNode_implicitDryRunParameterIsEchoed() + { + admin = new AlfrescoCoreAdminHandler() { + @Override + NamedList fixOnSpecificCore( + String coreName, + Long fromTxCommitTime, + Long toTxCommitTime, + boolean dryRun, + int maxTransactionsToSchedule) { + return new NamedList<>(); // dummy entry + } + + @Override + boolean isMasterOrStandalone(String coreName) + { + return true; + } + }; + + admin.trackerRegistry = registry; + + params.set(CORE, ALFRESCO_CORE_NAME); + + NamedList actionResponse = admin.actionFIX(params); + assertEquals(true, actionResponse.get(DRY_RUN_PARAMETER_NAME)); + assertEquals(ACTION_STATUS_NOT_SCHEDULED, actionResponse.get(ACTION_STATUS_LABEL)); + } + + @Test + public void masterOrStandaloneNode_explicitDryRunParameterIsEchoed() + { + assertThatExplicitParameterIsEchoed( + DRY_RUN_PARAMETER_NAME, + true); + + assertThatExplicitParameterIsEchoed( + DRY_RUN_PARAMETER_NAME, + false); + } + + @Test + public void masterOrStandaloneNode_explicitFromCommitTimeParameterIsEchoed() + { + assertThatExplicitParameterIsEchoed( + FROM_TX_COMMIT_TIME_PARAMETER_NAME, + System.currentTimeMillis()); + } + + @Test + public void masterOrStandaloneNode_explicitToCommitTimeParameterIsEchoed() + { + assertThatExplicitParameterIsEchoed( + TO_TX_COMMIT_TIME_PARAMETER_NAME, + System.currentTimeMillis()); + } + + @Test + public void masterOrStandaloneNode_explicitMaxTransactionsToScheduleParameterIsEchoed() + { + assertThatExplicitParameterIsEchoed( + MAX_TRANSACTIONS_TO_SCHEDULE_PARAMETER_NAME, + Integer.MAX_VALUE); + } + + @Test + public void manageTransactionsToBeFixed_shouldRespectTheInputGlobalLimit() + { + AtomicInteger limit = new AtomicInteger(4); + AtomicInteger transactionCount = new AtomicInteger(); + + IOpenBitSet transactions = new SolrOpenBitSetAdapter(); + range(1, 7).forEach(transactions::set); + + Consumer counter = tx -> transactionCount.incrementAndGet(); + admin.manageTransactionsToBeFixed(transactions, tx -> 0, counter, limit); + + assertEquals(-1, limit.get()); + assertEquals(4, transactionCount.get()); + } + + @Test + public void subsequentInvocationsToManageTransactionsToBeFixed_shouldRespectTheInputGlobalLimit() + { + // Limit is set to max 13 transactions + AtomicInteger limit = new AtomicInteger(13); + AtomicInteger transactionCount = new AtomicInteger(); + + // First transaction set contains 6 transactions + IOpenBitSet firstTransactionSet = new SolrOpenBitSetAdapter(); + range(1, 7).forEach(firstTransactionSet::set); + + Consumer counter = tx -> transactionCount.incrementAndGet(); + admin.manageTransactionsToBeFixed(firstTransactionSet, tx -> 0, counter, limit); + + assertEquals(7, limit.get()); + assertEquals(6, transactionCount.get()); + + // Second transaction set contains 9 transactions (more than the remaining transactions to process) + IOpenBitSet secondTransactionSet = new SolrOpenBitSetAdapter(); + range(10, 21).forEach(secondTransactionSet::set); + + admin.manageTransactionsToBeFixed(secondTransactionSet, tx -> 0, counter, limit); + + assertEquals("Global transaction limit should have been exceeded", -1, limit.get()); + assertEquals(13, transactionCount.get()); + + // Third transaction set contains 10 transactions, it should be completely ignored as we already exceeded the + // global limit above + IOpenBitSet thirdTransactionSet = new SolrOpenBitSetAdapter(); + range(31, 42).forEach(thirdTransactionSet::set); + + Consumer thisShoulndtBeInvoked = tx -> { throw new RuntimeException("We should never be here, as the global limit has been already exceeded."); }; + + admin.manageTransactionsToBeFixed(thirdTransactionSet, tx -> 0, thisShoulndtBeInvoked, limit); + + assertEquals( -2, limit.get()); + assertEquals(13, transactionCount.get()); + } + + @Test + public void noAclTransactionToReindex_shouldReturnAnEmptyResponse() + { + IndexHealthReport emptyReport = mock(IndexHealthReport.class); + when(emptyReport.getAclTxInIndexButNotInDb()).thenReturn(new SolrOpenBitSetAdapter()); + when(emptyReport.getDuplicatedAclTxInIndex()).thenReturn(new SolrOpenBitSetAdapter()); + when(emptyReport.getMissingAclTxFromIndex()).thenReturn(new SolrOpenBitSetAdapter()); + + NamedList subReport = admin.aclTxToReindex(ALFRESCO_CORE_NAME, mock(AclTracker.class), emptyReport, tx -> {}, Integer.MAX_VALUE); + + assertEquals( + Long.valueOf(0L), + of(subReport.get(ACL_TX_IN_INDEX_NOT_IN_DB)) + .map(NamedList.class::cast) + .map(NamedList::size) + .map(Number::longValue) + .orElseThrow(() -> new RuntimeException(ACL_TX_IN_INDEX_NOT_IN_DB + " section not found in response."))); + + assertEquals( + Long.valueOf(0L), + of(subReport.get(DUPLICATED_ACL_TX_IN_INDEX)) + .map(NamedList.class::cast) + .map(NamedList::size) + .map(Number::longValue) + .orElseThrow(() -> new RuntimeException(DUPLICATED_ACL_TX_IN_INDEX + " section not found in response."))); + + assertEquals( + Long.valueOf(0L), + of(subReport.get(MISSING_ACL_TX_IN_INDEX)) + .map(NamedList.class::cast) + .map(NamedList::size) + .map(Number::longValue) + .orElseThrow(() -> new RuntimeException(MISSING_ACL_TX_IN_INDEX + " section not found in response."))); + } + + @Test + public void noTransactionToReindex_shouldReturnAnEmptyResponse() + { + IndexHealthReport emptyReport = mock(IndexHealthReport.class); + when(emptyReport.getTxInIndexButNotInDb()).thenReturn(new SolrOpenBitSetAdapter()); + when(emptyReport.getDuplicatedTxInIndex()).thenReturn(new SolrOpenBitSetAdapter()); + when(emptyReport.getMissingTxFromIndex()).thenReturn(new SolrOpenBitSetAdapter()); + + NamedList subReport = admin.txToReindex(ALFRESCO_CORE_NAME, mock(MetadataTracker.class), emptyReport, tx -> {}, Integer.MAX_VALUE); + + assertEquals( + Long.valueOf(0L), + of(subReport.get(TX_IN_INDEX_NOT_IN_DB)) + .map(NamedList.class::cast) + .map(NamedList::size) + .map(Number::longValue) + .orElseThrow(() -> new RuntimeException(TX_IN_INDEX_NOT_IN_DB + " section not found in response."))); + + assertEquals( + Long.valueOf(0L), + of(subReport.get(DUPLICATED_TX_IN_INDEX)) + .map(NamedList.class::cast) + .map(NamedList::size) + .map(Number::longValue) + .orElseThrow(() -> new RuntimeException(DUPLICATED_TX_IN_INDEX + " section not found in response."))); + + assertEquals( + Long.valueOf(0L), + of(subReport.get(MISSING_TX_IN_INDEX)) + .map(NamedList.class::cast) + .map(NamedList::size) + .map(Number::longValue) + .orElseThrow(() -> new RuntimeException(MISSING_TX_IN_INDEX + " section not found in response."))); + } + + @Test + public void maxTransactionsGlobalLimitShouldBeAppliedInCascade() + { + SolrInformationServer server = mock(SolrInformationServer.class); + when(server.getDocListSize(anyString())).thenReturn(0); + + ConcurrentHashMap informationServers = new ConcurrentHashMap<>(); + informationServers.put(ALFRESCO_CORE_NAME, server); + admin.informationServers = informationServers; + + IOpenBitSet txInIndexButNotInDb = new SolrOpenBitSetAdapter(); + IOpenBitSet duplicatedTxInIndex = new SolrOpenBitSetAdapter(); + IOpenBitSet missingTxFromIndex = new SolrOpenBitSetAdapter(); + range(1, 10).forEach(txInIndexButNotInDb::set); + range(21, 32).forEach(duplicatedTxInIndex::set); + range(50, 61).forEach(missingTxFromIndex::set); + + int maxTransactionToSchedule = (int) (txInIndexButNotInDb.cardinality() + + duplicatedTxInIndex.cardinality() + + missingTxFromIndex.cardinality() - + 5); + + IndexHealthReport emptyReport = mock(IndexHealthReport.class); + when(emptyReport.getTxInIndexButNotInDb()).thenReturn(txInIndexButNotInDb); + when(emptyReport.getDuplicatedTxInIndex()).thenReturn(duplicatedTxInIndex); + when(emptyReport.getMissingTxFromIndex()).thenReturn(missingTxFromIndex); + + NamedList subReport = admin.txToReindex(ALFRESCO_CORE_NAME, mock(MetadataTracker.class), emptyReport, tx -> {}, maxTransactionToSchedule); + + assertEquals( + Long.valueOf(txInIndexButNotInDb.cardinality()), + of(subReport.get(TX_IN_INDEX_NOT_IN_DB)) + .map(NamedList.class::cast) + .map(NamedList::size) + .map(Number::longValue) + .orElseThrow(() -> new RuntimeException(TX_IN_INDEX_NOT_IN_DB + " section not found in response."))); + + assertEquals( + Long.valueOf(duplicatedTxInIndex.cardinality()), + of(subReport.get(DUPLICATED_TX_IN_INDEX)) + .map(NamedList.class::cast) + .map(NamedList::size) + .map(Number::longValue) + .orElseThrow(() -> new RuntimeException(DUPLICATED_TX_IN_INDEX + " section not found in response."))); + + assertEquals( + Long.valueOf(missingTxFromIndex.cardinality() - 5), + of(subReport.get(MISSING_TX_IN_INDEX)) + .map(NamedList.class::cast) + .map(NamedList::size) + .map(Number::longValue) + .orElseThrow(() -> new RuntimeException(MISSING_TX_IN_INDEX + " section not found in response."))); + } + + @Test + public void maxAclTransactionsGlobalLimitShouldBeAppliedInCascade() + { + SolrInformationServer server = mock(SolrInformationServer.class); + when(server.getDocListSize(anyString())).thenReturn(0); + + ConcurrentHashMap informationServers = new ConcurrentHashMap<>(); + informationServers.put(ALFRESCO_CORE_NAME, server); + admin.informationServers = informationServers; + + IOpenBitSet txInIndexButNotInDb = new SolrOpenBitSetAdapter(); + IOpenBitSet duplicatedTxInIndex = new SolrOpenBitSetAdapter(); + IOpenBitSet missingTxFromIndex = new SolrOpenBitSetAdapter(); + range(1, 10).forEach(txInIndexButNotInDb::set); + range(21, 32).forEach(duplicatedTxInIndex::set); + range(50, 61).forEach(missingTxFromIndex::set); + + int maxTransactionToSchedule = (int) (txInIndexButNotInDb.cardinality() + + duplicatedTxInIndex.cardinality() + + missingTxFromIndex.cardinality() - + 5); + + IndexHealthReport emptyReport = mock(IndexHealthReport.class); + when(emptyReport.getAclTxInIndexButNotInDb()).thenReturn(txInIndexButNotInDb); + when(emptyReport.getDuplicatedAclTxInIndex()).thenReturn(duplicatedTxInIndex); + when(emptyReport.getMissingAclTxFromIndex()).thenReturn(missingTxFromIndex); + + NamedList subReport = admin.aclTxToReindex(ALFRESCO_CORE_NAME, mock(AclTracker.class), emptyReport, tx -> {}, maxTransactionToSchedule); + + assertEquals( + Long.valueOf(txInIndexButNotInDb.cardinality()), + of(subReport.get(ACL_TX_IN_INDEX_NOT_IN_DB)) + .map(NamedList.class::cast) + .map(NamedList::size) + .map(Number::longValue) + .orElseThrow(() -> new RuntimeException(ACL_TX_IN_INDEX_NOT_IN_DB + " section not found in response."))); + + assertEquals( + Long.valueOf(duplicatedTxInIndex.cardinality()), + of(subReport.get(DUPLICATED_ACL_TX_IN_INDEX)) + .map(NamedList.class::cast) + .map(NamedList::size) + .map(Number::longValue) + .orElseThrow(() -> new RuntimeException(DUPLICATED_ACL_TX_IN_INDEX + " section not found in response."))); + + assertEquals( + Long.valueOf(missingTxFromIndex.cardinality() - 5), + of(subReport.get(MISSING_ACL_TX_IN_INDEX)) + .map(NamedList.class::cast) + .map(NamedList::size) + .map(Number::longValue) + .orElseThrow(() -> new RuntimeException(MISSING_ACL_TX_IN_INDEX + " section not found in response."))); + } + + private void assertThatExplicitParameterIsEchoed(String parameterName, T parameterValue) + { + admin = new AlfrescoCoreAdminHandler() { + @Override + NamedList fixOnSpecificCore( + String coreName, + Long fromTxCommitTime, + Long toTxCommitTime, + boolean dryRun, + int maxTransactionsToSchedule) { + return new NamedList<>(); // dummy entry + } + + @Override + boolean isMasterOrStandalone(String coreName) { + return true; + } + }; + + admin.trackerRegistry = registry; + + params.set(CORE, ALFRESCO_CORE_NAME); + params.set(parameterName, parameterValue.toString()); + + NamedList actionResponse = admin.actionFIX(params); + assertEquals(parameterValue, actionResponse.get(parameterName)); + } +} \ No newline at end of file