diff --git a/.travis.settings.xml b/.travis.settings.xml new file mode 100644 index 000000000..0ce1436a4 --- /dev/null +++ b/.travis.settings.xml @@ -0,0 +1,30 @@ + + + + alfresco-public-releases + ${env.MAVEN_USERNAME} + ${env.MAVEN_PASSWORD} + + + alfresco-public-snapshots + ${env.MAVEN_USERNAME} + ${env.MAVEN_PASSWORD} + + + alfresco-enterprise-releases + ${env.MAVEN_USERNAME} + ${env.MAVEN_PASSWORD} + + + alfresco-enterprise-snapshots + ${env.MAVEN_USERNAME} + ${env.MAVEN_PASSWORD} + + + + alfresco-hotfix + ${env.MAVEN_USERNAME} + ${env.MAVEN_PASSWORD} + + + diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 000000000..29fb6f269 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,218 @@ +os: linux +dist: xenial +language: java +jdk: + - openjdk11 + +services: + - docker + +cache: + directories: + - $HOME/.m2 +# the cache can grow constantly +before_cache: + - rm -rf $HOME/.m2/repository/org/alfresco/* + +branches: + only: + - master + - /release\/.*/ + +stages: + - name: Tests + if: commit_message !~ /\[skip tests\]/ + - Deploy + - Release + - Publish + +env: + global: + # Release version has to start with real version (2.0.0-....) for the docker image to build successfully. + - RELEASE_VERSION=2.0.0-A3 + - DEVELOPMENT_VERSION=2.0.0-SNAPSHOT + +# This should not be required on community build +before_install: + - "cp .travis.settings.xml $HOME/.m2/settings.xml" + +jobs: + include: + - stage: Tests + name: "Unit tests" + jdk: openjdk11 + install: + - travis_retry travis_wait 60 mvn --batch-mode -q clean package -DskipTests + script: + - travis_wait 60 mvn --batch-mode package -DfailIfNoTests=false -Dtests.timezone=UTC -Dmaven.test.failure.ignore=true -pl '"!:search-analytics-e2e-test"' + - name: "CMIS API tests" + jdk: openjdk11 + install: + - travis_retry travis_wait 60 mvn --batch-mode -q clean package -DskipTests + - docker login quay.io -u ${QUAY_USERNAME} -p ${QUAY_PASSWORD} + - travis_retry travis_wait 40 docker build -t quay.io/alfresco/insight-engine:latest insight-engine/packaging/target/docker-resources + - mkdir -p e2e-test/target/docker-resources && python e2e-test/python-generator/generator.py --alfresco=quay.io/alfresco/dev:acs-for-search --transformer=LegacyTransformers --search=quay.io/alfresco/insight-engine:latest --spellcheck --output=e2e-test/target/docker-resources + - ./e2e-test/helpers/start-alfresco.sh "e2e-test/target/docker-resources" "quay.io/alfresco/insight-engine:latest" + - ./e2e-test/helpers/wait-service-to-start.sh + script: + - travis_wait 60 mvn test -f e2e-test/pom.xml -DsuiteXmlFile=src/test/resources/cmis-search.xml -Dtest.exclude=Not_InsightEngine,CONFIG_Master_Slave,Not_Bamboo,AGS_302,CONFIG_Sharding_DB_ID_RANGE,CONFIG_Sharding -Dalfresco.server=localhost -Dalfresco.port=8081 -Dmaven.test.failure.ignore=true + - name: "SEARCH API tests" + jdk: openjdk11 + install: + - travis_retry travis_wait 60 mvn --batch-mode -q clean package -DskipTests + - docker login quay.io -u ${QUAY_USERNAME} -p ${QUAY_PASSWORD} + - travis_retry travis_wait 40 docker build -t quay.io/alfresco/insight-engine:latest insight-engine/packaging/target/docker-resources + - mkdir -p e2e-test/target/docker-resources && python e2e-test/python-generator/generator.py --alfresco=quay.io/alfresco/dev:acs-for-search --transformer=LegacyTransformers --search=quay.io/alfresco/insight-engine:latest --spellcheck --output=e2e-test/target/docker-resources + - ./e2e-test/helpers/start-alfresco.sh "e2e-test/target/docker-resources" "quay.io/alfresco/insight-engine:latest" + - ./e2e-test/helpers/wait-service-to-start.sh + script: + - travis_wait 60 mvn test -f e2e-test/pom.xml -DsuiteXmlFile=src/test/resources/SearchSuite.xml -Dtest.exclude=Not_InsightEngine,CONFIG_Master_Slave,Not_Bamboo,AGS_302,CONFIG_Sharding_DB_ID_RANGE,CONFIG_Sharding -Dalfresco.server=localhost -Dalfresco.port=8081 -Dmaven.test.failure.ignore=true + - name: "SQL API tests" + jdk: openjdk11 + install: + - travis_retry travis_wait 60 mvn --batch-mode -q clean package -DskipTests + - docker login quay.io -u ${QUAY_USERNAME} -p ${QUAY_PASSWORD} + - travis_retry travis_wait 40 docker build -t quay.io/alfresco/insight-engine:latest insight-engine/packaging/target/docker-resources + - mkdir -p e2e-test/target/docker-resources && python e2e-test/python-generator/generator.py --alfresco=quay.io/alfresco/dev:acs-for-search --transformer=LegacyTransformers --search=quay.io/alfresco/insight-engine:latest --spellcheck --output=e2e-test/target/docker-resources + - ./e2e-test/helpers/start-alfresco.sh "e2e-test/target/docker-resources" "quay.io/alfresco/insight-engine:latest" + - ./e2e-test/helpers/wait-service-to-start.sh + script: + - travis_wait 60 mvn test -f e2e-test/pom.xml -DsuiteXmlFile=src/test/resources/InsightSuite.xml -Dtest.exclude=Not_InsightEngine,CONFIG_Master_Slave,Not_Bamboo,AGS_302,CONFIG_Sharding_DB_ID_RANGE,CONFIG_Sharding -Dalfresco.server=localhost -Dalfresco.port=8081 -Dmaven.test.failure.ignore=true + - name: "Integration tests" + jdk: openjdk11 + install: + - travis_retry travis_wait 60 mvn --batch-mode -q clean package -DskipTests + script: + - travis_wait 60 mvn -q verify -Dtest=foo -DfailIfNoTests=false -Dtests.timezone=UTC -Dmaven.test.failure.ignore=true -pl '"!:search-analytics-e2e-test"' + - name: "Build and verify Docker images" + jdk: openjdk11 + install: + - travis_retry travis_wait 60 mvn --batch-mode -q clean package -DskipTests + - docker login quay.io -u ${QUAY_USERNAME} -p ${QUAY_PASSWORD} + - travis_retry travis_wait 40 docker build -t quay.io/alfresco/search-services:latest search-services/packaging/target/docker-resources + - travis_retry travis_wait 40 docker build -t quay.io/alfresco/insight-engine:latest insight-engine/packaging/target/docker-resources + - travis_retry travis_wait 40 docker build -t quay.io/alfresco/insight-zeppelin:latest insight-engine/alfresco-insight-zeppelin/target/docker-resources + script: + - travis_wait 60 ./scripts/test-image.sh "${TRAVIS_BUILD_DIR}" "search-services/packaging/target/docker-resources insight-engine/packaging/target/docker-resources insight-engine/alfresco-insight-zeppelin/target/docker-resources" "quay.io/alfresco/search-services quay.io/alfresco/insight-engine quay.io/alfresco/insight-zeppelin" "latest" "alfresco-search-services alfresco-insight-engine zeppelin" + - name: "Whitesource scan - Search Services" + #TODO run only on release branches and master + #if: fork = false AND (branch = master OR branch =~ /release\/.*/) AND type != pull_request + jdk: openjdk11 + install: + - travis_retry travis_wait 60 mvn --batch-mode -q clean package -DskipTests + - mkdir -p whitesource_scan/search-services && unzip search-services/packaging/target/*.zip -d whitesource_scan/search-services + script: + - curl -LJO https://github.com/whitesource/unified-agent-distribution/releases/latest/download/wss-unified-agent.jar + - java -jar wss-unified-agent.jar -apiKey ${WHITESOURCE_API_KEY} -project "SearchServices" -projectVersion "2.0.0-SNAPSHOT" -d ./whitesource_scan/search-services + - name: "Whitesource scan - Insight Engine" + #TODO run only on release branches and master + #if: fork = false AND (branch = master OR branch =~ /release\/.*/) AND type != pull_request + jdk: openjdk11 + install: + - travis_retry travis_wait 60 mvn --batch-mode -q clean package -DskipTests + - mkdir -p whitesource_scan/insight-engine && unzip insight-engine/packaging/target/*.zip -d whitesource_scan/insight-engine + script: + - curl -LJO https://github.com/whitesource/unified-agent-distribution/releases/latest/download/wss-unified-agent.jar + - java -jar wss-unified-agent.jar -apiKey ${WHITESOURCE_API_KEY} -project "InsightEngine" -projectVersion "2.0.0-SNAPSHOT" -d ./whitesource_scan/insight-engine + - name: "Whitesource scan - Zeppelin" + #TODO run only on release branches and master + #if: fork = false AND (branch = master OR branch =~ /release\/.*/) AND type != pull_request + jdk: openjdk11 + install: + - travis_retry travis_wait 60 mvn --batch-mode -q clean package -DskipTests + - mkdir -p whitesource_scan/insight-zeppelin && unzip insight-engine/alfresco-insight-zeppelin/target/*.zip -d whitesource_scan/insight-zeppelin + script: + - curl -LJO https://github.com/whitesource/unified-agent-distribution/releases/latest/download/wss-unified-agent.jar + - java -jar wss-unified-agent.jar -apiKey ${WHITESOURCE_API_KEY} -project "InsightZeppelin" -projectVersion "2.0.0-SNAPSHOT" -d ./whitesource_scan/insight-zeppelin + - stage: Deploy + name: Deploy snapshots to Nexus + #TODO run only on release branches and master + #if: fork = false AND (branch = master OR branch =~ /release\/.*/) AND type != pull_request + install: + - travis_retry travis_wait 60 mvn --batch-mode -q clean package -DskipTests + script: + - travis_wait 60 mvn --batch-mode deploy -DskipTests=true + - name: Deploy dev version to quay.io + #TODO run only on release branches + #if: fork = false AND ( branch =~ /release\/.*/) AND type != pull_request + install: + - travis_retry travis_wait 60 mvn --batch-mode -q clean package -DskipTests + - docker login quay.io -u ${QUAY_USERNAME} -p ${QUAY_PASSWORD} + - travis_retry travis_wait 40 docker build -t quay.io/alfresco/search-services:$DEVELOPMENT_VERSION search-services/packaging/target/docker-resources + - travis_retry travis_wait 40 docker build -t quay.io/alfresco/insight-engine:$DEVELOPMENT_VERSION insight-engine/packaging/target/docker-resources + - travis_retry travis_wait 40 docker build -t quay.io/alfresco/insight-zeppelin:$DEVELOPMENT_VERSION insight-engine/alfresco-insight-zeppelin/target/docker-resources + script: + - travis_retry docker push quay.io/alfresco/search-services:$DEVELOPMENT_VERSION + - travis_retry docker push quay.io/alfresco/insight-engine:$DEVELOPMENT_VERSION + - travis_retry docker push quay.io/alfresco/insight-zeppelin:$DEVELOPMENT_VERSION + - name: Deploy "latest" to quay.io + #TODO run only on master + #if: fork = false AND (branch = master) AND type != pull_request + install: + - travis_retry travis_wait 60 mvn --batch-mode -q clean package -DskipTests + - docker login quay.io -u ${QUAY_USERNAME} -p ${QUAY_PASSWORD} + - travis_retry travis_wait 40 docker build -t quay.io/alfresco/search-services:latest search-services/packaging/target/docker-resources + - travis_retry travis_wait 40 docker build -t quay.io/alfresco/insight-engine:latest insight-engine/packaging/target/docker-resources + - travis_retry travis_wait 40 docker build -t quay.io/alfresco/insight-zeppelin:latest insight-engine/alfresco-insight-zeppelin/target/docker-resources + script: + - travis_retry docker push quay.io/alfresco/search-services:latest + - travis_retry docker push quay.io/alfresco/insight-engine:latest + - travis_retry docker push quay.io/alfresco/insight-zeppelin:latest + - name: "Release and Copy to S3 Staging Bucket" + stage: Release + # TODO release only on releasable branches and if the commit has "release" keyword +# if: fork = false AND (branch = master OR branch =~ /support\/.*/) AND type != pull_request AND commit_message =~ /\[release\]/ +# if: commit_message =~ /\[release\]/ + install: + # Use full history for release + - git checkout -B "${TRAVIS_BRANCH}" + # Add email to link commits to user + - git config user.email "${GIT_EMAIL}" + - travis_retry travis_wait 60 mvn --batch-mode -q clean package -DskipTests + script: + - travis_wait 60 mvn --batch-mode -DreleaseVersion="${RELEASE_VERSION}" -DdevelopmentVersion="${DEVELOPMENT_VERSION}" -Dusername="${GIT_USERNAME}" -Dpassword="${GIT_PASSWORD}" -DscmCommentPrefix="[maven-release-plugin][skip ci]" -DskipTests "-Darguments=-DskipTests -Dadditionalparam=-Xdoclint:none" release:prepare release:perform + deploy: + - provider: s3 + access_key_id: ${AWS_STAGING_ACCESS_KEY} + secret_access_key: ${AWS_STAGING_SECRET_KEY} + region: "eu-west-1" + bucket: "alfresco-artefacts-staging" + glob: "alfresco-search-services-${RELEASE_VERSION}.zip" + upload_dir: "enterprise/SearchServices/${RELEASE_VERSION}" + skip_cleanup: true + acl: private + local_dir: "search-services/packaging/target" + - provider: s3 + access_key_id: ${AWS_STAGING_ACCESS_KEY} + secret_access_key: ${AWS_STAGING_SECRET_KEY} + region: "eu-west-1" + bucket: "alfresco-artefacts-staging" + glob: "alfresco-insight-engine-distribution-${RELEASE_VERSION}.zip" + upload_dir: "enterprise/InsightEngine/${RELEASE_VERSION}" + skip_cleanup: true + acl: private + local_dir: "insight-engine/packaging/target" + - provider: s3 + access_key_id: ${AWS_STAGING_ACCESS_KEY} + secret_access_key: ${AWS_STAGING_SECRET_KEY} + region: "eu-west-1" + bucket: "alfresco-artefacts-staging" + glob: "alfresco-insight-zeppelin-${RELEASE_VERSION}.zip" + upload_dir: "enterprise/InsightZeppelin/${RELEASE_VERSION}" + skip_cleanup: true + acl: private + local_dir: "insight-engine/alfresco-insight-zeppelin/target" + - name: Deploy release version to quay.io + #TODO run only on release branches and if the commit has "release" keyword + #if: fork = false AND ( branch =~ /release\/.*/) AND type != pull_request + #if: commit_message =~ /\[release\]/ + install: + - travis_retry travis_wait 60 mvn --batch-mode -q clean package -DskipTests + - docker login quay.io -u ${QUAY_USERNAME} -p ${QUAY_PASSWORD} + - travis_retry travis_wait 40 docker build -t quay.io/alfresco/search-services:$RELEASE_VERSION search-services/packaging/target/docker-resources + - travis_retry travis_wait 40 docker build -t quay.io/alfresco/insight-engine:$RELEASE_VERSION insight-engine/packaging/target/docker-resources + - travis_retry travis_wait 40 docker build -t quay.io/alfresco/insight-zeppelin:$RELEASE_VERSION insight-engine/alfresco-insight-zeppelin/target/docker-resources + script: + - travis_retry docker push quay.io/alfresco/search-services:$RELEASE_VERSION + - travis_retry docker push quay.io/alfresco/insight-engine:$RELEASE_VERSION + - travis_retry docker push quay.io/alfresco/insight-zeppelin:$RELEASE_VERSION \ No newline at end of file diff --git a/e2e-test/helpers/start-alfresco.sh b/e2e-test/helpers/start-alfresco.sh new file mode 100755 index 000000000..4aa6eda29 --- /dev/null +++ b/e2e-test/helpers/start-alfresco.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +set -eux + +# Start Alfresco and Solr. + +# The location for the docker-compose files. +DOCKER_RESOURCE_FOLDER=$1 +# The search docker image. +SEARCH_IMAGE=$2 + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +export DOCKER_CLIENT_TIMEOUT=120 +export COMPOSE_HTTP_TIMEOUT=120 + +# Pull the image, to ensure we're not using a stale image from the build agent. +docker pull "$SEARCH_IMAGE" + +# Build the images and call docker-compose. +cd "$DOCKER_RESOURCE_FOLDER" +docker-compose up -d --build --force-recreate + +$SCRIPT_DIR/wait-service-to-start.sh diff --git a/e2e-test/helpers/wait-service-to-start.sh b/e2e-test/helpers/wait-service-to-start.sh new file mode 100755 index 000000000..e8ad3e332 --- /dev/null +++ b/e2e-test/helpers/wait-service-to-start.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +set -e # exit if commands fails +set -x # trace what gets exe + +WAIT_INTERVAL=1 +COUNTER=0 +TIMEOUT=2000 +t0=`date +%s` + +ENDPOINT="${1:-http://localhost:8081/alfresco}" + +echo "Waiting for Service to start using endpoint: ${ENDPOINT}" + +until $(curl --output /dev/null --silent --head --fail ${ENDPOINT}) || [ "$COUNTER" -eq "$TIMEOUT" ]; do + printf '.' + sleep $WAIT_INTERVAL + COUNTER=$(($COUNTER+$WAIT_INTERVAL)) +done + +if (("$COUNTER" < "$TIMEOUT")) ; then + t1=`date +%s` + delta=$((($t1 - $t0)/60)) + echo "Service ${ENDPOINT} Started in $delta minutes" +else + echo "Service ${ENDPOINT} could not start in time." + echo "Waited $COUNTER seconds" + exit 1 +fi diff --git a/e2e-test/pom.xml b/e2e-test/pom.xml index 125a7b38a..bdab0b205 100644 --- a/e2e-test/pom.xml +++ b/e2e-test/pom.xml @@ -128,4 +128,37 @@ 12-ea+10 + + + + alfresco-enterprise-releases + https://artifacts.alfresco.com/nexus/content/repositories/enterprise-releases + + true + + + false + + + + alfresco-enterprise-snapshots + https://artifacts.alfresco.com/nexus/content/repositories/enterprise-snapshots + + false + + + true + + + + alfresco-hotfix + https://artifacts.alfresco.com/nexus/content/groups/hotfix + + true + + + false + + + \ No newline at end of file diff --git a/e2e-test/python-generator/README.md b/e2e-test/python-generator/README.md new file mode 100644 index 000000000..e61d02e94 --- /dev/null +++ b/e2e-test/python-generator/README.md @@ -0,0 +1,46 @@ +# About +A python script to generator docker-compose files suitable for use with the automated tests in +the [Search and Insight E2E tests](https://git.alfresco.com/search_discovery/insightengine/tree/master/e2e-test). + +# Installation +The script uses Python 3, which can be installed from [python.org](https://www.python.org/downloads/) or using +a package manager. You will also need the [yaml library](https://pypi.org/project/PyYAML/), which can be installed +using Pip if it is not already present: +``` +pip3 install pyyaml +``` + +# Using the script +The script provides some help with the `-h` option: + +``` +python3 BuildScripts/generator/generator.py -h +usage: generator.py [-h] [-a ALFRESCO]... +... +``` + +For use with ACS 6.2.x the legacy transformers can be included: +``` +python3 BuildScripts/generator/generator.py --alfresco=alfresco/alfresco-content-repository:6.2.0 --transformer=LegacyTransformers +``` + +For ACS 6.0.x and ACS 6.1.x no external transformer is needed: +``` +python3 BuildScripts/generator/generator.py --alfresco=alfresco/alfresco-content-repository:6.1.0 +``` + +For ACS 6.0.0.x and earlier ActiveMQ can be excluded: +``` +python3 BuildScripts/generator/generator.py --alfresco=alfresco/alfresco-content-repository:6.0.0.3 --excludeAMQ +``` + +For ACS 5.2.x the legacy LibreOffice transformer can be used: +``` +python3 BuildScripts/generator/generator.py --alfresco=quay.io/alfresco/alfresco-content-repository-52:5.2.5 --postgres=postgres:9.4 --excludeAMQ --transformer=LibreOffice +``` + +# Starting the containers +To start the containers you also need to build the images - for example: +``` +docker-compose up --build --force-recreate +``` diff --git a/e2e-test/python-generator/generator.py b/e2e-test/python-generator/generator.py new file mode 100644 index 000000000..d541831c8 --- /dev/null +++ b/e2e-test/python-generator/generator.py @@ -0,0 +1,317 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +import argparse +from string import Template +import yaml +import os +from distutils.dir_util import copy_tree + +LIBRE_OFFICE = 'LibreOffice' +LEGACY_TRANSFORMERS = 'LegacyTransformers' + +AMQ_OPTS = '-Dmessaging.broker.url="failover:(nio://activemq:61616)?timeout=3000&jms.useCompression=true"' +TRANSFORM_OPTS = ('-Dtransform.service.enabled=false -Dlocal.transform.service.enabled=true ' + '-DlocalTransform.libreoffice.url=http://libreoffice:8090/ -DlocalTransform.tika.url=http://tika:8090/ ' + '-DlocalTransform.misc.url=http://transform-misc:8090/ -Dlegacy.transform.service.enabled=true ' + '-Djodconverter.url=http://libreoffice:8090/ -Dtika.url=http://tika:8090/ ' + '-Dtransform.misc.url=http://transform-misc:8090/') +SHARE_TRANSFORM_OPTS = ('-DlocalTransform.pdfrenderer.url=http://alfresco-pdf-renderer:8090/ ' + '-Dalfresco-pdf-renderer.url=http://alfresco-pdf-renderer:8090/ ' + '-DlocalTransform.imagemagick.url=http://imagemagick:8090/ -Dimg.url=http://imagemagick:8090/') +SHARDING_OPTS = '-Dsolr.useDynamicShardRegistration=true' +JAVA_OPTS = ('-Ddb.driver=org.postgresql.Driver -Ddb.username=alfresco -Ddb.password=alfresco ' + '-Ddb.url=jdbc:postgresql://postgres:5432/alfresco -Dsolr.port=8983 ' + '-Dindex.subsystem.name=solr6 ' + '-Dalfresco.restApi.basicAuthScheme=true ') +MTLS_OPTS = ('-Dsolr.port.ssl=8983 -Dsolr.secureComms=https ') +HTTP_OPTS = ('-Dsolr.secureComms=none') +JAVA_TOOL_OPTIONS = ('-Dmetadata-keystore.password=mp6yc0UD9e ' + '-Dmetadata-keystore.aliases=metadata ' + '-Dmetadata-keystore.metadata.password=mp6yc0UD9e ' + '-Dmetadata-keystore.metadata.algorithm=AES') +MTLS_JAVA_TOOL_OPTIONS = ('-Dencryption.keystore.type=pkcs12 -Dencryption.cipherAlgorithm=AES/CBC/PKCS5Padding ' + '-Dencryption.keyAlgorithm=AES ' + '-Dssl-truststore.password=kT9X6oe68t -Dssl-keystore.password=kT9X6oe68t ' + '-Dssl-keystore.aliases=ssl-alfresco-ca,ssl-repo ' + '-Dssl-keystore.ssl-alfresco-ca.password=kT9X6oe68t ' + '-Dssl-keystore.ssl-repo.password=kT9X6oe68t ' + '-Dssl-truststore.aliases=alfresco-ca,ssl-repo-client ' + '-Dssl-truststore.alfresco-ca.password=kT9X6oe68t ' + '-Dssl-truststore.ssl-repo-client.password=kT9X6oe68t') + +def getJavaOpts(includeAMQ, includeTransform, includeShare, solrHost, solrBaseUrl, sharding, mtls): + + solrHost = '-Dsolr.host=' + solrHost + shardingOpts = (SHARDING_OPTS if sharding != None else '') + amqOpts = (AMQ_OPTS if includeAMQ else '') + transformOpts = (TRANSFORM_OPTS if includeTransform else '') + solrBaseUrlOpts = '-Dsolr.baseUrl=' + solrBaseUrl + shareTransformOpts = (SHARE_TRANSFORM_OPTS if includeShare and includeTransform else '') + mtlsOpts = (MTLS_OPTS if mtls else HTTP_OPTS) + + return ' '.join([JAVA_OPTS, amqOpts, transformOpts, shareTransformOpts, solrHost, solrBaseUrlOpts, shardingOpts, mtlsOpts]) + +def getJavaToolOptions(mtls): + + mtlsJavaToolOptions = (MTLS_JAVA_TOOL_OPTIONS if mtls else '') + + return ' '.join([JAVA_TOOL_OPTIONS, mtlsJavaToolOptions]) + +def deleteServices(dcYaml, *services): + for service in services: + if service in dcYaml['services'].keys(): + del(dcYaml['services'][service]) + +def getExtraEnvironmentVars(serviceName, replicationType): + """Return a dict of environment variables to add to the search container declaration in docker-compose.yml.""" + extraEnvironmentVars = {} + if replicationType == 'master': + extraEnvironmentVars['REPLICATION_TYPE'] = 'master' + elif replicationType == 'slave': + extraEnvironmentVars.update({'REPLICATION_TYPE': 'slave', + 'REPLICATION_MASTER_HOST': serviceName.replace('slave', 'master'), + 'REPLICATION_MASTER_PORT': '8983', + 'REPLICATION_POLL_INTERVAL': '00:00:10'}) + return extraEnvironmentVars + +def getSolrcoreConfig(sharding, shardId, shardCount, shardRange): + """Returns a list of properties to add to the end of the solrcore.properties file.""" + solrcoreConfig = [] + if sharding != None: + solrcoreConfig.append('solr.port.ssl=8983') + solrcoreConfig.append('shard.instance={}'.format(shardId)) + solrcoreConfig.append('alfresco.port=8080') + solrcoreConfig.append('alfresco.port.ssl=8443') + solrcoreConfig.append('alfresco.baseUrl=/alfresco') + if sharding not in ['DB_ID_RANGE', 'EXPLICIT_ID_FALLBACK_LRIS']: + solrcoreConfig.append('shard.count={}'.format(shardCount)) + if sharding == 'DB_ID_RANGE': + # The first shards each contain 800 nodes by default and the last continues the range to id 100000. + nodesPerShard = shardRange + rangeStart = shardId * nodesPerShard + rangeEnd = (rangeStart + nodesPerShard - 1 if shardId < shardCount - 1 else 100000) + solrcoreConfig.append('shard.range={}-{}'.format(rangeStart, rangeEnd)) + if sharding == 'DATE': + solrcoreConfig.append('shard.key=cm:created') + solrcoreConfig.append('shard.date.grouping={}'.format(12 // shardCount)) + if sharding in ['PROPERTY', 'EXPLICIT_ID', 'EXPLICIT_ID_FALLBACK_LRIS']: + solrcoreConfig.append('shard.key=shard:shardId') + print("SolrConfig for Shard: ", shardId, " : ", solrcoreConfig) + return solrcoreConfig + +def getSolrcoreReplacements(sharding, mtls): + """Returns a dict of replacements to make in the solrcore.properties file.""" + solrcoreReplacements = {} + if sharding != None: + solrcoreReplacements['shard.method=DB_ID'] = 'shard.method={}'.format(sharding) + if mtls: + solrcoreReplacements['alfresco.secureComms=none'] = 'alfresco.secureComms=https' + solrcoreReplacements['alfresco.encryption.ssl.keystore.location=.*'] = 'alfresco.encryption.ssl.keystore.location=\\\\\\/opt\\\\\\/alfresco-search-services\\\\\\/keystore\\\\\\/ssl-repo-client.keystore' + solrcoreReplacements['alfresco.encryption.ssl.keystore.type=.*'] = 'alfresco.encryption.ssl.keystore.type=JCEKS' + solrcoreReplacements['alfresco.encryption.ssl.truststore.location=.*'] = 'alfresco.encryption.ssl.truststore.location=\\\\\\/opt\\\\\\/alfresco-search-services\\\\\\/keystore\\\\\\/ssl-repo-client.truststore' + solrcoreReplacements['alfresco.encryption.ssl.truststore.type=.*'] = 'alfresco.encryption.ssl.truststore.type=JCEKS' + else: + solrcoreReplacements['alfresco.secureComms=https'] = 'alfresco.secureComms=none' + return solrcoreReplacements + +def addAlfrescoMtlsConfig(alfrescoArgsNode): + """Add a list of environment values in Docker Compose Alfresco Service for mTLS.""" + alfrescoArgsNode['TRUSTSTORE_TYPE'] = 'JCEKS' + alfrescoArgsNode['TRUSTSTORE_PASS'] = 'kT9X6oe68t' + alfrescoArgsNode['KEYSTORE_TYPE'] = 'JCEKS' + alfrescoArgsNode['KEYSTORE_PASS'] = 'kT9X6oe68t' + alfrescoArgsNode['SOLR_COMMS'] = 'https' + +def addAlfrescoVolumes(alfrescoNode): + """Add route to keystores folder""" + alfrescoNode['volumes'] = ['./keystores/alfresco:/usr/local/tomcat/alf_data/keystore'] + +def addSolrMtlsConfig(solrEnvNode): + """Add a list of environment values in Docker Compose SOLR Service for mTLS.""" + solrEnvNode['SOLR_SSL_TRUST_STORE'] = '/opt/alfresco-search-services/keystore/ssl-repo-client.truststore' + solrEnvNode['SOLR_SSL_TRUST_STORE_TYPE'] = 'JCEKS' + solrEnvNode['SOLR_SSL_KEY_STORE'] = '/opt/alfresco-search-services/keystore/ssl-repo-client.keystore' + solrEnvNode['SOLR_SSL_KEY_STORE_TYPE'] = 'JCEKS' + solrEnvNode['SOLR_SSL_NEED_CLIENT_AUTH'] = 'true' + +def addSolrOpts(solrEnvNode): + """Add a list of values to add in Docker Compose SOLR_OPTS property for mTLS.""" + solrOptions = ' '.join(['-Dsolr.ssl.checkPeerName=false', + '-Dsolr.allow.unsafe.resourceloading=true']) + solrEnvNode['SOLR_OPTS'] = solrOptions + +def addSolrJavaToolOptions(solrEnvNode): + """Add a list of values to add in Docker Compose JAVA_TOOL_OPTIONS property for mTLS.""" + solrOptions = ' '.join(['-Dsolr.jetty.truststore.password=kT9X6oe68t ', + '-Dsolr.jetty.keystore.password=kT9X6oe68t ', + '-Dssl-keystore.password=kT9X6oe68t', + '-Dssl-keystore.aliases=ssl-alfresco-ca,ssl-repo-client', + '-Dssl-keystore.ssl-alfresco-ca.password=kT9X6oe68t', + '-Dssl-keystore.ssl-repo-client.password=kT9X6oe68t', + '-Dssl-truststore.password=kT9X6oe68t', + '-Dssl-truststore.aliases=ssl-alfresco-ca,ssl-repo,ssl-repo-client', + '-Dssl-truststore.ssl-alfresco-ca.password=kT9X6oe68t', + '-Dssl-truststore.ssl-repo.password=kT9X6oe68t', + '-Dssl-truststore.ssl-repo-client.password=kT9X6oe68t']) + solrEnvNode['JAVA_TOOL_OPTIONS'] = solrOptions + +def addSolrVolumes(solrNode): + """Add route to keystores folder""" + solrNode['volumes'] = ['./keystores/solr:/opt/alfresco-search-services/keystore'] + +def makeSearchNode(outputDirectory, nodeName, externalPort, params, mtls, extraEnvironmentVars={}, solrcoreConfig=[], solrcoreReplacements={}): + # Create a dictionary for the template replacement. + allParams = dict(params) + allParams['SOLR_HOST'] = nodeName + allParams['ALFRESCO_PORT'] = 8443 if mtls else 8080 + allParams['EXTERNAL_PORT'] = externalPort + # Properties to add to solrcore.properties. + allParams['SOLRCORE_PROPERTIES'] = '\\n'.join(solrcoreConfig) + # Replacements to make in solrcore.properties (in an "abc/xyz" format suitable for sed). + allParams['SOLRCORE_REPLACEMENTS'] = ' '.join(map(lambda pair: '"{}/{}"'.format(*pair), solrcoreReplacements.items())) + + # mTLS settings + allParams['ALFRESCO_SECURE_COMMS'] = ('https' if mtls else 'none') + allParams['TRUSTSTORE_TYPE'] = 'JCEKS' + allParams['KEYSTORE_TYPE'] = 'JCEKS' + + # Create a Dockerfile with any extra configuration in. + with open(scriptDir + '/templates/search/Dockerfile.template') as f: + dockerfileTemplate = f.read() + dockerfileString = Template(dockerfileTemplate).substitute(allParams) + if not os.path.isdir('{}/{}'.format(outputDirectory, nodeName)): + os.mkdir('{}/{}'.format(outputDirectory, nodeName)) + with open('{}/{}/Dockerfile'.format(outputDirectory, nodeName), 'w') as f: + dockerfileTemplate = f.write(dockerfileString) + + # Load the search node template. + with open(scriptDir + '/templates/search-node.yml.template') as f: + searchNodeTemplate = f.read() + searchNodeString = Template(searchNodeTemplate).substitute(allParams) + # Read the result as yaml. + searchNodeYaml = yaml.safe_load(searchNodeString) + # Add any extra environment variables. + searchNodeYaml['environment'].update(extraEnvironmentVars) + + # Add mTLS configuration if required + if mtls: + addSolrMtlsConfig(searchNodeYaml['environment']) + addSolrOpts(searchNodeYaml['environment']) + addSolrJavaToolOptions(searchNodeYaml['environment']) + addSolrVolumes(searchNodeYaml) + + return searchNodeYaml + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Generate a docker-compose file for ACS.') + parser.add_argument('-a', '--alfresco', default='quay.io/alfresco/dev:acs-for-search', help='The Alfresco image') + parser.add_argument('-s', '--search', default='quay.io/alfresco/search-services:latest', help='The Search image') + parser.add_argument('-e', '--share', help='The Share image (or omit for no UI)') + parser.add_argument('-p', '--postgres', default='postgres:10.1', help='The Postgres image') + parser.add_argument('-q', '--excludeAMQ', action='store_true', help='Exclude ActiveMQ (i.e. pre-ACS 6.1)') + parser.add_argument('-t', '--transformer', choices=[LIBRE_OFFICE, LEGACY_TRANSFORMERS], help='Use external transformers. ' + + '"{}" for legacy LibreOffice (i.e. ACS 5.2.x). '.format(LIBRE_OFFICE) + + '"{}" for the transformers for use with ACS 6.2.x and later.'.format(LEGACY_TRANSFORMERS)) + parser.add_argument('-c', '--spellcheck', action='store_true', help='Spellcheck Enabled') + parser.add_argument('-ms', '--masterslave', action='store_true', help='Master Slave Enabled') + parser.add_argument('-sh', '--sharding', help='Sharding method (or omit for no sharding). Note that sharding is not supported on SearchServices 1.2.x or earlier.', + choices=['DB_ID', 'DB_ID_RANGE', 'ACL_ID', 'MOD_ACL_ID', 'DATE', 'PROPERTY', 'LRIS', 'EXPLICIT_ID', 'EXPLICIT_ID_FALLBACK_LRIS']) + parser.add_argument('-sc', '--shardCount', type=int, help='Total number of shards to create (default 2)') + parser.add_argument('-sr', '--shardRange', type=int, help='Total number of nodes per shard with DB_ID_RANGE sharding (default 800)') + parser.add_argument('-ct', '--disableCascadeTracking', action='store_true', help='Cascade Tracking Disabled') + parser.add_argument('-sl', '--searchLogLevel', default='WARN', help='The log level for search (default WARN)', + choices=['TRACE', 'DEBUG', 'INFO', 'WARN', 'ERROR']) + parser.add_argument('-o', '--output', default='.', help='The path of the directory to output to') + parser.add_argument('-mtls', '--mtls', action='store_true', help='Use mTLS between SOLR and Alfresco Repository') + args = parser.parse_args() + + # If sharding is selected then the default number of shards is two. + if args.sharding != None and args.shardCount == None: + print('Using default shardCount of 2') + args.shardCount = 2 + elif args.sharding == None and args.shardCount != None: + print('ERROR: shardCount={} specified without sharding method'.format(args.shardCount)) + exit(1) + print('Arguments:', args) + + # If sharding is selected then the default number of nodes per shard is 800. + if args.sharding == "DB_ID_RANGE" and (args.shardRange == None or args.shardRange < 1): + print('Using default shardRange of 800') + args.shardRange = 800 + elif args.sharding != 'DB_ID_RANGE' and args.shardRange != None: + print('ERROR: shardRange={} is only supported for DB_ID_RANGE sharding.') + exit(1) + print('Arguments:', args) + + # Load the template and perform basic token substitution. + scriptDir = os.path.dirname(os.path.realpath(__file__)) + with open(scriptDir + '/templates/docker-compose.yml.template') as f: + template = f.read() + params = { + 'ALFRESCO_IMAGE': args.alfresco, + 'SHARE_IMAGE': args.share, + 'POSTGRES_IMAGE': args.postgres, + 'SEARCH_IMAGE': args.search, + 'SEARCH_LOG_LEVEL': args.searchLogLevel, + 'SEARCH_ENABLE_SPELLCHECK': str(args.spellcheck).lower(), + 'DISABLE_CASCADE_TRACKING': str(args.disableCascadeTracking).lower() + } + dcString = Template(template).substitute(params) + + # Edit the resulting yaml. + dcYaml = yaml.safe_load(dcString) + + # Insert the search node(s). + shardList = range(args.shardCount) if args.sharding != None else [0] + replicationTypes = ['master', 'slave'] if args.masterslave else ['standalone'] + for shardId in shardList: + for replicationType in replicationTypes: + serviceName = 'search_{}_{}'.format(shardId, replicationType) + # Workaround for ShardInfoTest.getShardInfoWithAdminAuthority. + if shardId == 0 and replicationType == 'standalone': + serviceName = 'search' + externalPort = 8083 + 100 * shardId + (1 if replicationType == 'slave' else 0) + dcYaml['services'][serviceName] = makeSearchNode(args.output, serviceName, externalPort, params, args.mtls, + extraEnvironmentVars=getExtraEnvironmentVars(serviceName, replicationType), + solrcoreConfig=getSolrcoreConfig(args.sharding, shardId, args.shardCount, args.shardRange), + solrcoreReplacements=getSolrcoreReplacements(args.sharding, args.mtls)) + + # Point Alfresco at whichever Solr node came last in the list. + solrHost = serviceName + solrBaseUrl = '/solr-slave' if args.masterslave else '/solr' + + javaOpts = getJavaOpts(not args.excludeAMQ, args.transformer == LEGACY_TRANSFORMERS, args.share != None, solrHost, solrBaseUrl, args.sharding, args.mtls) + dcYaml['services']['alfresco']['environment']['JAVA_OPTS'] = javaOpts + javaToolOpts = getJavaToolOptions(args.mtls) + dcYaml['services']['alfresco']['environment']['JAVA_TOOL_OPTIONS'] = javaToolOpts + if args.mtls: + addAlfrescoMtlsConfig(dcYaml['services']['alfresco']['build']['args']) + addAlfrescoVolumes(dcYaml['services']['alfresco']) + + if not args.share: + deleteServices(dcYaml, 'share', 'alfresco-pdf-renderer', 'imagemagick') + if args.excludeAMQ: + deleteServices(dcYaml, 'activemq') + if args.transformer != LEGACY_TRANSFORMERS: + deleteServices(dcYaml, 'libreoffice', 'tika', 'transform-misc', 'shared-file-store', 'alfresco-pdf-renderer', 'imagemagick') + del(dcYaml['volumes']['shared-file-store-volume']) + if args.transformer == LIBRE_OFFICE: + dcYaml['services']['libreoffice'] = {'image': 'xcgd/libreoffice'} + + # Output the yaml. + with open(args.output + '/docker-compose.yml', 'w') as f: + f.write(yaml.safe_dump(dcYaml)) + + # Create an Alfresco Dockerfile with any extra configuration in. + with open(scriptDir + '/templates/alfresco/Dockerfile.template') as f: + dockerfileTemplate = f.read() + dockerfileString = Template(dockerfileTemplate).substitute(params) + if not os.path.isdir('{}/{}'.format(args.output, 'alfresco')): + os.mkdir('{}/{}'.format(args.output, 'alfresco')) + with open('{}/{}/Dockerfile'.format(args.output, 'alfresco'), 'w') as f: + dockerfileTemplate = f.write(dockerfileString) + + # Copy the keystores (when using mTLS) + if args.mtls: + copy_tree(scriptDir + '/keystores', args.output + '/keystores') diff --git a/e2e-test/python-generator/keystores/alfresco/keystore b/e2e-test/python-generator/keystores/alfresco/keystore new file mode 100644 index 000000000..439bc1f86 Binary files /dev/null and b/e2e-test/python-generator/keystores/alfresco/keystore differ diff --git a/e2e-test/python-generator/keystores/alfresco/ssl.keystore b/e2e-test/python-generator/keystores/alfresco/ssl.keystore new file mode 100644 index 000000000..c0cff56f8 Binary files /dev/null and b/e2e-test/python-generator/keystores/alfresco/ssl.keystore differ diff --git a/e2e-test/python-generator/keystores/alfresco/ssl.truststore b/e2e-test/python-generator/keystores/alfresco/ssl.truststore new file mode 100644 index 000000000..33da2c362 Binary files /dev/null and b/e2e-test/python-generator/keystores/alfresco/ssl.truststore differ diff --git a/e2e-test/python-generator/keystores/solr/ssl-repo-client.keystore b/e2e-test/python-generator/keystores/solr/ssl-repo-client.keystore new file mode 100644 index 000000000..f1474dd04 Binary files /dev/null and b/e2e-test/python-generator/keystores/solr/ssl-repo-client.keystore differ diff --git a/e2e-test/python-generator/keystores/solr/ssl-repo-client.truststore b/e2e-test/python-generator/keystores/solr/ssl-repo-client.truststore new file mode 100644 index 000000000..c09e6891a Binary files /dev/null and b/e2e-test/python-generator/keystores/solr/ssl-repo-client.truststore differ diff --git a/e2e-test/python-generator/templates/alfresco/Dockerfile.template b/e2e-test/python-generator/templates/alfresco/Dockerfile.template new file mode 100755 index 000000000..3438f963e --- /dev/null +++ b/e2e-test/python-generator/templates/alfresco/Dockerfile.template @@ -0,0 +1,49 @@ +FROM ${ALFRESCO_IMAGE} + +ENV TOMCAT_DIR=/usr/local/tomcat +ENV ALF_DATA_DIR=$${TOMCAT_DIR}/alf_data + +# COMMS +ARG SOLR_COMMS +ENV SOLR_COMMS $$SOLR_COMMS + +# SSL +ARG TRUSTSTORE_TYPE +ARG TRUSTSTORE_PASS +ARG KEYSTORE_TYPE +ARG KEYSTORE_PASS + +ENV TRUSTSTORE_TYPE=$$TRUSTSTORE_TYPE \ + TRUSTSTORE_PASS=$$TRUSTSTORE_PASS \ + KEYSTORE_TYPE=$$KEYSTORE_TYPE \ + KEYSTORE_PASS=$$KEYSTORE_PASS + +USER root + +# Default value in 'repository.properties' is 'dir.keystore=classpath:alfresco/keystore' +RUN if [ "$$SOLR_COMMS" == "https" ] ; then \ + echo -e "\n\ + dir.keystore=$${ALF_DATA_DIR}/keystore\n\ + alfresco.encryption.ssl.keystore.type=$${TRUSTSTORE_TYPE}\n\ + alfresco.encryption.ssl.truststore.type=$${KEYSTORE_TYPE}\n\ + " >> $${TOMCAT_DIR}/shared/classes/alfresco-global.properties; \ + fi + +# Enable SSL by adding the proper Connector to server.xml +RUN if [ "$$SOLR_COMMS" == "https" ] ; then \ + sed -i "s/\ +[[:space:]]\+<\/Engine>/\n\ + <\/Engine>\n\ + \n\ + <\/Connector>/g" $${TOMCAT_DIR}/conf/server.xml; \ + fi + +# Expose keystore folder +# Useless for 'none'/'http' communications with SOLR +VOLUME ["$$ALF_DATA_DIR/keystore"] diff --git a/e2e-test/python-generator/templates/docker-compose.yml.template b/e2e-test/python-generator/templates/docker-compose.yml.template new file mode 100644 index 000000000..b2c3f0145 --- /dev/null +++ b/e2e-test/python-generator/templates/docker-compose.yml.template @@ -0,0 +1,101 @@ +version: '3' +services: + alfresco: + build: + context: ./alfresco + args: + SOLR_COMMS: none + environment: + JAVA_OPTS : "This will be populated by the generator script" + JAVA_TOOL_OPTIONS : "This will be populated by the generator script" + ports: + - "7203:7203" #JMX connect via service:jmx:rmi:///jndi/rmi://localhost:7203/jmxrmi + - "5005:5005" #Java debugging + - "8081:8080" #Browser port for Alfresco + share: + image: ${SHARE_IMAGE} + environment: + - REPO_HOST=alfresco + - REPO_PORT=8080 + ports: + - 8082:8080 #Browser port for Share + postgres: + image: ${POSTGRES_IMAGE} + environment: + - POSTGRES_PASSWORD=alfresco + - POSTGRES_USER=alfresco + - POSTGRES_DB=alfresco + ports: + - 5432:5432 + activemq: + image: alfresco/alfresco-activemq:5.15.6 + ports: + - 8161:8161 # Web Console + - 5672:5672 # AMQP + - 61616:61616 # OpenWire + - 61613:61613 # STOMP + alfresco-pdf-renderer: + image: alfresco/alfresco-pdf-renderer:2.1.0-EA4 + environment: + JAVA_OPTS: " -Xms256m -Xmx512m" + ACTIVEMQ_URL: "nio://activemq:61616" + FILE_STORE_URL: "http://shared-file-store:8099/alfresco/api/-default-/private/sfs/versions/1/file" + ports: + - 8090:8090 + links: + - activemq + imagemagick: + image: alfresco/alfresco-imagemagick:2.1.0-EA4 + environment: + JAVA_OPTS: " -Xms256m -Xmx512m" + ACTIVEMQ_URL: "nio://activemq:61616" + FILE_STORE_URL: "http://shared-file-store:8099/alfresco/api/-default-/private/sfs/versions/1/file" + ports: + - 8091:8090 + links: + - activemq + libreoffice: + image: alfresco/alfresco-libreoffice:2.1.0-EA4 + environment: + JAVA_OPTS: " -Xms256m -Xmx512m" + ACTIVEMQ_URL: "nio://activemq:61616" + FILE_STORE_URL: "http://shared-file-store:8099/alfresco/api/-default-/private/sfs/versions/1/file" + ports: + - 8092:8090 + links: + - activemq + tika: + image: alfresco/alfresco-tika:2.1.0-EA4 + environment: + JAVA_OPTS: " -Xms256m -Xmx512m" + ACTIVEMQ_URL: "nio://activemq:61616" + FILE_STORE_URL: "http://shared-file-store:8099/alfresco/api/-default-/private/sfs/versions/1/file" + ports: + - 8093:8090 + links: + - activemq + transform-misc: + image: alfresco/alfresco-transform-misc:2.1.0-EA4 + environment: + JAVA_OPTS: " -Xms256m -Xmx512m" + ACTIVEMQ_URL: "nio://activemq:61616" + FILE_STORE_URL: "http://shared-file-store:8099/alfresco/api/-default-/private/sfs/versions/1/file" + ports: + - 8094:8090 + links: + - activemq + shared-file-store: + image: alfresco/alfresco-shared-file-store:0.5.3 + environment: + JAVA_OPTS: " -Xms256m -Xmx512m" + scheduler.content.age.millis: 86400000 + scheduler.cleanup.interval: 86400000 + ports: + - 8099:8099 + volumes: + - shared-file-store-volume:/tmp/Alfresco/sfs +volumes: + shared-file-store-volume: + driver_opts: + type: tmpfs + device: tmpfs diff --git a/e2e-test/python-generator/templates/search-node.yml.template b/e2e-test/python-generator/templates/search-node.yml.template new file mode 100644 index 000000000..502be27f7 --- /dev/null +++ b/e2e-test/python-generator/templates/search-node.yml.template @@ -0,0 +1,17 @@ +build: + context: ./${SOLR_HOST} +environment: + #Solr needs to know how to register itself with Alfresco + SOLR_ALFRESCO_HOST: "alfresco" + SOLR_ALFRESCO_PORT: "${ALFRESCO_PORT}" + #Alfresco needs to know how to call solr + SOLR_SOLR_HOST: "${SOLR_HOST}" + SOLR_SOLR_PORT: "8983" + #Create the default alfresco and archive cores + SOLR_CREATE_ALFRESCO_DEFAULTS: "alfresco,archive" + #Enable Spellcheck by setting to true + ENABLE_SPELLCHECK: "${SEARCH_ENABLE_SPELLCHECK}" + #Disable Cascade Tracking + DISABLE_CASCADE_TRACKING: "${DISABLE_CASCADE_TRACKING}" +ports: + - ${EXTERNAL_PORT}:8983 #Browser port diff --git a/e2e-test/python-generator/templates/search/Dockerfile.template b/e2e-test/python-generator/templates/search/Dockerfile.template new file mode 100644 index 000000000..cd2db8ae2 --- /dev/null +++ b/e2e-test/python-generator/templates/search/Dockerfile.template @@ -0,0 +1,32 @@ +FROM ${SEARCH_IMAGE} + +# Create search_config_setup.sh if it does not exist (e.g. on SearchServices 1.2.x or earlier). +USER root +RUN touch $${DIST_DIR}/solr/bin/search_config_setup.sh \ + && chown solr:solr $${DIST_DIR}/solr/bin/search_config_setup.sh +USER solr + +RUN replacementPairs=(${SOLRCORE_REPLACEMENTS}); \ + for replacementPair in $${replacementPairs[@]}; \ + do \ + sed -i '/^bash.*/i sed -i "'"s/$$replacementPair/g"'" $${DIST_DIR}/solrhome/templates/rerank/conf/solrcore.properties\n' \ + $${DIST_DIR}/solr/bin/search_config_setup.sh; \ + done; \ + if [[ "${SOLRCORE_PROPERTIES}" != "" ]]; \ + then \ + sed -i '/^bash.*/i echo "\n${SOLRCORE_PROPERTIES}" >> $${DIST_DIR}/solrhome/templates/rerank/conf/solrcore.properties\n' \ + $${DIST_DIR}/solr/bin/search_config_setup.sh; \ + fi + +USER root +RUN mkdir -p /opt/alfresco-search-services/keystore \ + && chown -R solr:solr /opt/alfresco-search-services/keystore +USER solr + +# Set the search log level if requested. +RUN if [ "${SEARCH_LOG_LEVEL}" != "" ] ; then \ + sed -i '/^bash.*/i sed -i "'"s/log4j.rootLogger=WARN, file, CONSOLE/log4j.rootLogger=${SEARCH_LOG_LEVEL}, file, CONSOLE/g"'" $${DIST_DIR}/logs/log4j.properties\n' \ + $${DIST_DIR}/solr/bin/search_config_setup.sh; \ +fi + +VOLUME ["/opt/alfresco-search-services/keystore"] diff --git a/pom.xml b/pom.xml index 9bb3bdfd7..d4b1a9402 100644 --- a/pom.xml +++ b/pom.xml @@ -12,18 +12,18 @@ Alfresco Search And Insight Parent - alfresco-internal + alfresco-enterprise-releases https://artifacts.alfresco.com/nexus/content/repositories/enterprise-releases/ - alfresco-internal-snapshots + alfresco-enterprise-snapshots https://artifacts.alfresco.com/nexus/content/repositories/enterprise-snapshots/ - scm:git:ssh://git@github.com/Alfresco/InsightEngine.git - scm:git:ssh://git@github.com/Alfresco/InsightEngine.git - scm:git:ssh://git@github.com/Alfresco/InsightEngine.git + scm:git:https://github.com/Alfresco/InsightEngine.git + scm:git:https://github.com/Alfresco/InsightEngine.git + https://github.com/Alfresco/InsightEngine HEAD diff --git a/scripts/test-image.sh b/scripts/test-image.sh new file mode 100755 index 000000000..a666d14eb --- /dev/null +++ b/scripts/test-image.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +set -eux + +# Sanity test of the docker images. + +# The root directory of the project. +PROJECT_DIRECTORY=$1 +# The relative paths to the dockerfiles. +DOCKER_RESOURCE_HOMES=($2) +# Names of the Docker images to test. +IMAGE_NAMES=($3) +# The version of the image. +IMAGE_VERSION=$4 +# The directories (within /opt) on the images that our products should have been installed in. +DIST_DIRS=($5) + +# The number of projects being processed. +COUNT=${#IMAGE_NAMES[@]} + +for (( i = 0; i < $COUNT; i++ )) +do + IMAGE_NAME=${IMAGE_NAMES[$i]} + DOCKER_RESOURCE_HOME=${DOCKER_RESOURCE_HOMES[$i]} + # There is a variable called DIST_DIR in the image, so use something different. + DIR=${DIST_DIRS[$i]} + + DOCKER_IMAGE_VERSIONED="$IMAGE_NAME:$IMAGE_VERSION" + + cd "$PROJECT_DIRECTORY/$DOCKER_RESOURCE_HOME" + + if [ "$DIR" != "zeppelin" ] + then + docker run --rm $DOCKER_IMAGE_VERSIONED [ -d /opt/$DIR/solr ] || (echo "solr dir does not exist" && exit 1) + docker run --rm $DOCKER_IMAGE_VERSIONED [ -d /opt/$DIR/data/alfrescoModels ] || (echo "alfrescoModels dir does not exist" && exit 1) + docker run --rm $DOCKER_IMAGE_VERSIONED [ -e /opt/$DIR/solr.in.sh ] || (echo "solr.in.sh does not exist" && exit 1) + docker run --rm $DOCKER_IMAGE_VERSIONED grep -q Alfresco /opt/$DIR/solr.in.sh || (echo "solr.in.sh does not contain Alfresco config" && exit 1) + docker run --rm $DOCKER_IMAGE_VERSIONED grep -q Alfresco /opt/$DIR/solr.in.cmd || (echo "solr.in.cmd does not contain Alfresco config" && exit 1) + docker run --rm $DOCKER_IMAGE_VERSIONED grep -q LOG4J_PROPS /opt/$DIR/solr.in.sh || (echo "solr.in.sh does not contain LOG4J_PROPS" && exit 1) + docker run --rm $DOCKER_IMAGE_VERSIONED grep -q LOG4J_CONFIG /opt/$DIR/solr.in.cmd || (echo "solr.in.cmd does not contain LOG4J_CONFIG" && exit 1) + docker run --rm $DOCKER_IMAGE_VERSIONED [ -e /opt/$DIR/solrhome/conf/shared.properties ] || (echo "shared.properties does not exist" && exit 1) + docker run --rm $DOCKER_IMAGE_VERSIONED /opt/$DIR/solr/bin/solr start + fi +done diff --git a/search-services/alfresco-search/pom.xml b/search-services/alfresco-search/pom.xml index ae60aa88a..21b272eff 100644 --- a/search-services/alfresco-search/pom.xml +++ b/search-services/alfresco-search/pom.xml @@ -212,43 +212,6 @@ - - - alfresco-public - https://artifacts.alfresco.com/nexus/content/groups/public - - true - - - false - - - - alfresco-public-snapshots - https://artifacts.alfresco.com/nexus/content/groups/public-snapshots - - false - - - true - - - - central - Central Repository - https://repo.maven.apache.org/maven2 - default - - false - - - - maven-restlet - Public online Restlet repository - http://maven.restlet.talend.com - - - alfresco-solr diff --git a/search-services/alfresco-solrclient-lib/pom.xml b/search-services/alfresco-solrclient-lib/pom.xml index 1382b6471..12d286b2d 100644 --- a/search-services/alfresco-solrclient-lib/pom.xml +++ b/search-services/alfresco-solrclient-lib/pom.xml @@ -10,17 +10,6 @@ 2.0.0-SNAPSHOT - - - alfresco-internal - https://artifacts.alfresco.com/nexus/content/repositories/releases - - - alfresco-internal-snapshots - https://artifacts.alfresco.com/nexus/content/repositories/snapshots - - - 8.135 2.10.3 diff --git a/search-services/packaging/pom.xml b/search-services/packaging/pom.xml index 8db3085d8..05c2767ad 100644 --- a/search-services/packaging/pom.xml +++ b/search-services/packaging/pom.xml @@ -8,7 +8,6 @@ UTF-8 UTF-8 ${maven.build.timestamp} - ${bamboo_repository_revision_number} org.alfresco diff --git a/search-services/pom.xml b/search-services/pom.xml index 4d2570772..5665651e4 100644 --- a/search-services/pom.xml +++ b/search-services/pom.xml @@ -23,4 +23,52 @@ alfresco-search packaging + + + + alfresco-public-releases + https://artifacts.alfresco.com/nexus/content/repositories/releases + + + alfresco-public-snapshots + https://artifacts.alfresco.com/nexus/content/repositories/snapshots + + + + + + alfresco-public + https://artifacts.alfresco.com/nexus/content/groups/public + + true + + + false + + + + alfresco-public-snapshots + https://artifacts.alfresco.com/nexus/content/groups/public-snapshots + + false + + + true + + + + central + Central Repository + https://repo.maven.apache.org/maven2 + default + + false + + + + maven-restlet + Public online Restlet repository + http://maven.restlet.talend.com + + diff --git a/wss-unified-agent.config b/wss-unified-agent.config new file mode 100644 index 000000000..9501519e7 --- /dev/null +++ b/wss-unified-agent.config @@ -0,0 +1,228 @@ +#################################################################### +# WhiteSource Unified-Agent configuration file +#################################################################### +########################################## +# GENERAL SCAN MODE: Files and Package Managers +########################################## + +checkPolicies=true +forceCheckAllDependencies=true +forceUpdate=true +forceUpdate.failBuildOnPolicyViolation=true +offline=false +#ignoreSourceFiles=true +#scanComment= +#updateInventory=false +#resolveAllDependencies=false +#failErrorLevel=ALL +#requireKnownSha1=false +#generateScanReport=true +#scanReportTimeoutMinutes=10 +#excludeDependenciesFromNodes=.*commons-io.*,.*maven-model + +#projectPerFolder=true +#projectPerFolderIncludes= +#projectPerFolderExcludes= + +#wss.connectionTimeoutMinutes=60 +# Change the below URL to your WhiteSource server. +# Use the 'WhiteSource Server URL' which can be retrieved +# from your 'Profile' page on the 'Server URLs' panel. +# Then, add the '/agent' path to it. +wss.url=https://saas.whitesourcesoftware.com/agent + +#npm.resolveDependencies=false +#npm.ignoreSourceFiles=false +#npm.includeDevDependencies=true +#npm.runPreStep=true +#npm.ignoreNpmLsErrors=true +#npm.ignoreScripts=true +#npm.yarnProject=true +#npm.accessToken= +#npm.identifyByNameAndVersion=true + +#bower.resolveDependencies=false +#bower.ignoreSourceFiles=true +#bower.runPreStep=true + +#nuget.resolvePackagesConfigFiles=false +#nuget.resolveCsProjFiles=false +#nuget.resolveDependencies=false +#nuget.restoreDependencies=true +#nuget.ignoreSourceFiles=true +#nuget.runPreStep=true +#nuget.resolveNuspecFiles=false + +#python.resolveDependencies=false +#python.ignoreSourceFiles=false +#python.ignorePipInstallErrors=true +#python.installVirtualenv=true +#python.resolveHierarchyTree=false +#python.requirementsFileIncludes=requirements.txt +#python.resolveSetupPyFiles=true +#python.runPipenvPreStep=true +#python.pipenvDevDependencies=true +#python.IgnorePipenvInstallErrors=true + +#maven.ignoredScopes=test provided +maven.resolveDependencies=true +#maven.ignoreSourceFiles=true +#maven.aggregateModules=true +maven.ignorePomModules=false +#maven.runPreStep=true +#maven.ignoreMvnTreeErrors=true +#maven.environmentPath= +#maven.m2RepositoryPath= + +#gradle.ignoredScopes= +#gradle.resolveDependencies=false +#gradle.runAssembleCommand=false +#gradle.runPreStep=true +#gradle.ignoreSourceFiles=true +#gradle.aggregateModules=true +#gradle.preferredEnvironment=wrapper +#gradle.localRepositoryPath= + +#paket.resolveDependencies=false +#paket.ignoredGroups= +#paket.ignoreSourceFiles=false +#paket.runPreStep=true +#paket.exePath= + +#go.resolveDependencies=false +#go.collectDependenciesAtRuntime=true +#go.dependencyManager= +#go.ignoreSourceFiles=true +#go.glide.ignoreTestPackages=false +#go.gogradle.enableTaskAlias=true + +#ruby.resolveDependencies = false +#ruby.ignoreSourceFiles = false +#ruby.installMissingGems = true +#ruby.runBundleInstall = true +#ruby.overwriteGemFile = true + +#sbt.resolveDependencies=false +#sbt.ignoreSourceFiles=true +#sbt.aggregateModules=true +#sbt.runPreStep=true +#sbt.targetFolder= + +#php.resolveDependencies=false +#php.runPreStep=true +#php.includeDevDependencies=true + +#html.resolveDependencies=false + +#cocoapods.resolveDependencies=false +#cocoapods.runPreStep=true +#cocoapods.ignoreSourceFiles=false + +#hex.resolveDependencies=false +#hex.runPreStep=true +#hex.ignoreSourceFiles=false +#hex.aggregateModules=true + +################################## +# Organization tokens: +################################## +#apiKey= + +#userKey is required if WhiteSource administrator has enabled "Enforce user level access" option +#userKey= + +#projectName= +#projectVersion= +#projectToken= + +productName=SearchAndDiscovery +#productVersion= +#productToken= +#updateType=APPEND +#requesterEmail=user@provider.com + +######################################################################################### +# Includes/Excludes Glob patterns - PLEASE USE ONLY ONE EXCLUDE LINE AND ONE INCLUDE LINE +######################################################################################### +#includes=**/*.c **/*.cc **/*.cp **/*.cpp **/*.cxx **/*.c++ **/*.h **/*.hpp **/*.hxx + +#includes=**/*.m **/*.mm **/*.js **/*.php +includes=**/*.jar +#includes=**/*.gem **/*.rb +#includes=**/*.dll **/*.cs **/*.nupkg +#includes=**/*.tgz **/*.deb **/*.gzip **/*.rpm **/*.tar.bz2 +#includes=**/*.zip **/*.tar.gz **/*.egg **/*.whl **/*.py + +## Exclude file extensions or specific directories by adding **/*. or **/** +excludes=**/*sources.jar **/*javadoc.jar + +case.sensitive.glob=false +followSymbolicLinks=true + +################################## +# Archive properties +################################## +#archiveExtractionDepth=2 +#archiveIncludes=**/*.war **/*.ear +#archiveExcludes=**/*sources.jar + +################################## +# Proxy settings +################################## +#proxy.host= +#proxy.port= +#proxy.user= +#proxy.pass= + +################################## +# SCM settings +################################## +#scm.type= +#scm.user= +#scm.pass= +#scm.ppk= +#scm.url= +#scm.branch= +#scm.tag= +#scm.npmInstall= +#scm.npmInstallTimeoutMinutes= +#scm.repositoriesFile= + +############################################## +# SCAN MODE: Linux package manager settings +############################################## +#scanPackageManager=true + +################################## +# SCAN MODE: Docker images +################################## +#docker.scanImages=true +#docker.includes=.*.* +#docker.excludes= +#docker.pull.enable=true +#docker.pull.images=.*.* +#docker.pull.maxImages=10 +#docker.pull.tags=.*.* +#docker.pull.digest= +#docker.delete.force=true +#docker.login.sudo=false + +#docker.aws.enable=true +#docker.aws.registryIds= + +################################## +# SCAN MODE: Docker containers +################################## +#docker.scanContainers=true +#docker.containerIncludes=.*.* +#docker.containerExcludes= + +################################ +# Serverless settings +################################ +#serverless.provider= +#serverless.scanFunctions=true +#serverless.includes= +#serverless.excludes= +#serverless.region= +#serverless.maxFunctions=10