From cb32feb1fdbb1fa6a7fb2bb533cf7b389cf2c6f7 Mon Sep 17 00:00:00 2001 From: Alex Mukha Date: Wed, 8 Jul 2020 10:02:28 +0100 Subject: [PATCH 01/40] Add simple unit test job --- .travis.yml | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 000000000..1d81d1e99 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,43 @@ +os: linux +dist: xenial +sudo: required +language: java +jdk: + - openjdk11 + +services: + - docker + +cache: + directories: + - $HOME/.m2 +# the cache can grow constantly +before_cache: + - rm -rf $HOME/.m2/repository/org/alfresco/* + +#branches: +# only: +# - master +# - /release\/.*/ + +stages: + - name: test + if: commit_message !~ /\[skip tests\]/ + - release + - publish + +# This should not be required on community build +before_install: + - "cp .travis.settings.xml $HOME/.m2/settings.xml" + +jobs: + include: + - stage: test + name: "Unit tests" + jdk: openjdk11 + install: + - travis_retry travis_wait 40 mvn --batch-mode clean package -DskipTests + script: + - travis_wait 60 mvn --batch-mode package -DfailIfNoTests=false -Dmaven.wagon.http.pool=false -Dtests.timezone=UTC -Dmaven.test.failure.ignore=true -pl '"!:search-analytics-e2e-test"' + + From c5c02bfbb58546fd519e9f2b54ea95f56615d81f Mon Sep 17 00:00:00 2001 From: Alex Mukha Date: Wed, 8 Jul 2020 10:08:39 +0100 Subject: [PATCH 02/40] Remove deprecated sudo --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 1d81d1e99..01f51e613 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,5 @@ os: linux dist: xenial -sudo: required language: java jdk: - openjdk11 From 1aeb1bc42613240daf3cc3a38dd8892f7d7fe42e Mon Sep 17 00:00:00 2001 From: Alex Mukha Date: Wed, 8 Jul 2020 10:09:34 +0100 Subject: [PATCH 03/40] Comment out settings.xml --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 01f51e613..941e322cc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,8 +26,8 @@ stages: - publish # This should not be required on community build -before_install: - - "cp .travis.settings.xml $HOME/.m2/settings.xml" +#before_install: +# - "cp .travis.settings.xml $HOME/.m2/settings.xml" jobs: include: From 6c717e91d560b8aa9180b9f02fdb3ef0e669c751 Mon Sep 17 00:00:00 2001 From: Alex Mukha Date: Wed, 8 Jul 2020 11:24:48 +0100 Subject: [PATCH 04/40] Add settings.xml --- .travis.settings.xml | 9 +++++++++ .travis.yml | 4 ++-- 2 files changed, 11 insertions(+), 2 deletions(-) create mode 100644 .travis.settings.xml diff --git a/.travis.settings.xml b/.travis.settings.xml new file mode 100644 index 000000000..f0a5039be --- /dev/null +++ b/.travis.settings.xml @@ -0,0 +1,9 @@ + + + + alfresco-internal + ${env.MAVEN_USERNAME} + ${env.MAVEN_PASSWORD} + + + diff --git a/.travis.yml b/.travis.yml index 941e322cc..01f51e613 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,8 +26,8 @@ stages: - publish # This should not be required on community build -#before_install: -# - "cp .travis.settings.xml $HOME/.m2/settings.xml" +before_install: + - "cp .travis.settings.xml $HOME/.m2/settings.xml" jobs: include: From 3bf3677779b8cdf9cf208ae26ffa61d5b5f494d3 Mon Sep 17 00:00:00 2001 From: Alex Mukha Date: Wed, 8 Jul 2020 12:11:35 +0100 Subject: [PATCH 05/40] Add config for CMIS API tests --- .travis.yml | 13 +- e2e-test/helpers/start-alfresco.sh | 23 ++ e2e-test/helpers/wait-service-to-start.sh | 28 ++ e2e-test/python-generator/README.md | 46 +++ e2e-test/python-generator/generator.py | 314 ++++++++++++++++++ .../keystores/alfresco/keystore | Bin 0 -> 327 bytes .../keystores/alfresco/ssl.keystore | Bin 0 -> 3288 bytes .../keystores/alfresco/ssl.truststore | Bin 0 -> 1854 bytes .../keystores/solr/ssl-repo-client.keystore | Bin 0 -> 3298 bytes .../keystores/solr/ssl-repo-client.truststore | Bin 0 -> 2883 bytes .../templates/alfresco/Dockerfile.template | 49 +++ .../templates/docker-compose.yml.template | 101 ++++++ .../templates/search-node.yml.template | 17 + .../templates/search/Dockerfile.template | 26 ++ 14 files changed, 616 insertions(+), 1 deletion(-) create mode 100755 e2e-test/helpers/start-alfresco.sh create mode 100755 e2e-test/helpers/wait-service-to-start.sh create mode 100644 e2e-test/python-generator/README.md create mode 100644 e2e-test/python-generator/generator.py create mode 100644 e2e-test/python-generator/keystores/alfresco/keystore create mode 100644 e2e-test/python-generator/keystores/alfresco/ssl.keystore create mode 100644 e2e-test/python-generator/keystores/alfresco/ssl.truststore create mode 100644 e2e-test/python-generator/keystores/solr/ssl-repo-client.keystore create mode 100644 e2e-test/python-generator/keystores/solr/ssl-repo-client.truststore create mode 100755 e2e-test/python-generator/templates/alfresco/Dockerfile.template create mode 100644 e2e-test/python-generator/templates/docker-compose.yml.template create mode 100644 e2e-test/python-generator/templates/search-node.yml.template create mode 100644 e2e-test/python-generator/templates/search/Dockerfile.template diff --git a/.travis.yml b/.travis.yml index 01f51e613..a8f8144c8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -38,5 +38,16 @@ jobs: - travis_retry travis_wait 40 mvn --batch-mode clean package -DskipTests script: - travis_wait 60 mvn --batch-mode package -DfailIfNoTests=false -Dmaven.wagon.http.pool=false -Dtests.timezone=UTC -Dmaven.test.failure.ignore=true -pl '"!:search-analytics-e2e-test"' - + - name: "CMIS API tests" + jdk: openjdk11 + install: + - travis_retry travis_wait 40 mvn --batch-mode clean package -DskipTests + - travis_retry travis_wait 40 docker build -t quay.io/alfresco/search-services:latest search-services/packaging/target/docker-resources + - travis_retry travis_wait 40 docker build -t quay.io/alfresco/insight-engine:latest insight-engine/packaging/target/docker-resources + - travis_retry travis_wait 40 docker build -t quay.io/alfresco/insight-zeppelin:latest insight-engine/alfresco-insight-zeppelin/target/docker-resources + - mkdir -p e2e-test/target/docker-resources && python e2e-test/python-generator/generator.py --alfresco=quay.io/alfresco/dev:acs-for-search --transformer=LegacyTransformers --search=quay.io/alfresco/insight-engine:latest --spellcheck --output=e2e-test/target/docker-resources + - ./e2e-test/helpers/start-alfresco.sh "e2e-test/target/docker-resources" "quay.io/alfresco/insight-engine:latest" + - ./e2e-test/helpers/wait-service-to-start.sh + script: + - travis_wait 60 mvn test -f e2e-test/pom.xml -DsuiteXmlFile=src/test/resources/cmis-search.xml -Dtest.exclude=Not_InsightEngine,CONFIG_Master_Slave,Not_Bamboo,AGS_302,CONFIG_Sharding_DB_ID_RANGE,CONFIG_Sharding -Dalfresco.server=localhost -Dalfresco.port=8081 -Dmaven.test.failure.ignore=true diff --git a/e2e-test/helpers/start-alfresco.sh b/e2e-test/helpers/start-alfresco.sh new file mode 100755 index 000000000..4aa6eda29 --- /dev/null +++ b/e2e-test/helpers/start-alfresco.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +set -eux + +# Start Alfresco and Solr. + +# The location for the docker-compose files. +DOCKER_RESOURCE_FOLDER=$1 +# The search docker image. +SEARCH_IMAGE=$2 + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +export DOCKER_CLIENT_TIMEOUT=120 +export COMPOSE_HTTP_TIMEOUT=120 + +# Pull the image, to ensure we're not using a stale image from the build agent. +docker pull "$SEARCH_IMAGE" + +# Build the images and call docker-compose. +cd "$DOCKER_RESOURCE_FOLDER" +docker-compose up -d --build --force-recreate + +$SCRIPT_DIR/wait-service-to-start.sh diff --git a/e2e-test/helpers/wait-service-to-start.sh b/e2e-test/helpers/wait-service-to-start.sh new file mode 100755 index 000000000..e8ad3e332 --- /dev/null +++ b/e2e-test/helpers/wait-service-to-start.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +set -e # exit if commands fails +set -x # trace what gets exe + +WAIT_INTERVAL=1 +COUNTER=0 +TIMEOUT=2000 +t0=`date +%s` + +ENDPOINT="${1:-http://localhost:8081/alfresco}" + +echo "Waiting for Service to start using endpoint: ${ENDPOINT}" + +until $(curl --output /dev/null --silent --head --fail ${ENDPOINT}) || [ "$COUNTER" -eq "$TIMEOUT" ]; do + printf '.' + sleep $WAIT_INTERVAL + COUNTER=$(($COUNTER+$WAIT_INTERVAL)) +done + +if (("$COUNTER" < "$TIMEOUT")) ; then + t1=`date +%s` + delta=$((($t1 - $t0)/60)) + echo "Service ${ENDPOINT} Started in $delta minutes" +else + echo "Service ${ENDPOINT} could not start in time." + echo "Waited $COUNTER seconds" + exit 1 +fi diff --git a/e2e-test/python-generator/README.md b/e2e-test/python-generator/README.md new file mode 100644 index 000000000..e61d02e94 --- /dev/null +++ b/e2e-test/python-generator/README.md @@ -0,0 +1,46 @@ +# About +A python script to generator docker-compose files suitable for use with the automated tests in +the [Search and Insight E2E tests](https://git.alfresco.com/search_discovery/insightengine/tree/master/e2e-test). + +# Installation +The script uses Python 3, which can be installed from [python.org](https://www.python.org/downloads/) or using +a package manager. You will also need the [yaml library](https://pypi.org/project/PyYAML/), which can be installed +using Pip if it is not already present: +``` +pip3 install pyyaml +``` + +# Using the script +The script provides some help with the `-h` option: + +``` +python3 BuildScripts/generator/generator.py -h +usage: generator.py [-h] [-a ALFRESCO]... +... +``` + +For use with ACS 6.2.x the legacy transformers can be included: +``` +python3 BuildScripts/generator/generator.py --alfresco=alfresco/alfresco-content-repository:6.2.0 --transformer=LegacyTransformers +``` + +For ACS 6.0.x and ACS 6.1.x no external transformer is needed: +``` +python3 BuildScripts/generator/generator.py --alfresco=alfresco/alfresco-content-repository:6.1.0 +``` + +For ACS 6.0.0.x and earlier ActiveMQ can be excluded: +``` +python3 BuildScripts/generator/generator.py --alfresco=alfresco/alfresco-content-repository:6.0.0.3 --excludeAMQ +``` + +For ACS 5.2.x the legacy LibreOffice transformer can be used: +``` +python3 BuildScripts/generator/generator.py --alfresco=quay.io/alfresco/alfresco-content-repository-52:5.2.5 --postgres=postgres:9.4 --excludeAMQ --transformer=LibreOffice +``` + +# Starting the containers +To start the containers you also need to build the images - for example: +``` +docker-compose up --build --force-recreate +``` diff --git a/e2e-test/python-generator/generator.py b/e2e-test/python-generator/generator.py new file mode 100644 index 000000000..99cf11d68 --- /dev/null +++ b/e2e-test/python-generator/generator.py @@ -0,0 +1,314 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +import argparse +from string import Template +import yaml +import os +from distutils.dir_util import copy_tree + +LIBRE_OFFICE = 'LibreOffice' +LEGACY_TRANSFORMERS = 'LegacyTransformers' + +AMQ_OPTS = '-Dmessaging.broker.url="failover:(nio://activemq:61616)?timeout=3000&jms.useCompression=true"' +TRANSFORM_OPTS = ('-Dtransform.service.enabled=false -Dlocal.transform.service.enabled=true ' + '-DlocalTransform.libreoffice.url=http://libreoffice:8090/ -DlocalTransform.tika.url=http://tika:8090/ ' + '-DlocalTransform.misc.url=http://transform-misc:8090/ -Dlegacy.transform.service.enabled=true ' + '-Djodconverter.url=http://libreoffice:8090/ -Dtika.url=http://tika:8090/ ' + '-Dtransform.misc.url=http://transform-misc:8090/') +SHARE_TRANSFORM_OPTS = ('-DlocalTransform.pdfrenderer.url=http://alfresco-pdf-renderer:8090/ ' + '-Dalfresco-pdf-renderer.url=http://alfresco-pdf-renderer:8090/ ' + '-DlocalTransform.imagemagick.url=http://imagemagick:8090/ -Dimg.url=http://imagemagick:8090/') +SHARDING_OPTS = '-Dsolr.useDynamicShardRegistration=true' +JAVA_OPTS = ('-Ddb.driver=org.postgresql.Driver -Ddb.username=alfresco -Ddb.password=alfresco ' + '-Ddb.url=jdbc:postgresql://postgres:5432/alfresco -Dsolr.port=8983 ' + '-Dindex.subsystem.name=solr6 ' + '-Dalfresco.restApi.basicAuthScheme=true ') +MTLS_OPTS = ('-Dsolr.port.ssl=8983 -Dsolr.secureComms=https ') +HTTP_OPTS = ('-Dsolr.secureComms=none') +JAVA_TOOL_OPTIONS = ('-Dmetadata-keystore.password=mp6yc0UD9e ' + '-Dmetadata-keystore.aliases=metadata ' + '-Dmetadata-keystore.metadata.password=mp6yc0UD9e ' + '-Dmetadata-keystore.metadata.algorithm=AES') +MTLS_JAVA_TOOL_OPTIONS = ('-Dencryption.keystore.type=pkcs12 -Dencryption.cipherAlgorithm=AES/CBC/PKCS5Padding ' + '-Dencryption.keyAlgorithm=AES ' + '-Dssl-truststore.password=kT9X6oe68t -Dssl-keystore.password=kT9X6oe68t ' + '-Dssl-keystore.aliases=ssl-alfresco-ca,ssl-repo ' + '-Dssl-keystore.ssl-alfresco-ca.password=kT9X6oe68t ' + '-Dssl-keystore.ssl-repo.password=kT9X6oe68t ' + '-Dssl-truststore.aliases=alfresco-ca,ssl-repo-client ' + '-Dssl-truststore.alfresco-ca.password=kT9X6oe68t ' + '-Dssl-truststore.ssl-repo-client.password=kT9X6oe68t') + +def getJavaOpts(includeAMQ, includeTransform, includeShare, solrHost, solrBaseUrl, sharding, mtls): + + solrHost = '-Dsolr.host=' + solrHost + shardingOpts = (SHARDING_OPTS if sharding != None else '') + amqOpts = (AMQ_OPTS if includeAMQ else '') + transformOpts = (TRANSFORM_OPTS if includeTransform else '') + solrBaseUrlOpts = '-Dsolr.baseUrl=' + solrBaseUrl + shareTransformOpts = (SHARE_TRANSFORM_OPTS if includeShare and includeTransform else '') + mtlsOpts = (MTLS_OPTS if mtls else HTTP_OPTS) + + return ' '.join([JAVA_OPTS, amqOpts, transformOpts, shareTransformOpts, solrHost, solrBaseUrlOpts, shardingOpts, mtlsOpts]) + +def getJavaToolOptions(mtls): + + mtlsJavaToolOptions = (MTLS_JAVA_TOOL_OPTIONS if mtls else '') + + return ' '.join([JAVA_TOOL_OPTIONS, mtlsJavaToolOptions]) + +def deleteServices(dcYaml, *services): + for service in services: + if service in dcYaml['services'].keys(): + del(dcYaml['services'][service]) + +def getExtraEnvironmentVars(serviceName, replicationType): + """Return a dict of environment variables to add to the search container declaration in docker-compose.yml.""" + extraEnvironmentVars = {} + if replicationType == 'master': + extraEnvironmentVars['REPLICATION_TYPE'] = 'master' + elif replicationType == 'slave': + extraEnvironmentVars.update({'REPLICATION_TYPE': 'slave', + 'REPLICATION_MASTER_HOST': serviceName.replace('slave', 'master'), + 'REPLICATION_MASTER_PORT': '8983', + 'REPLICATION_POLL_INTERVAL': '00:00:10'}) + return extraEnvironmentVars + +def getSolrcoreConfig(sharding, shardId, shardCount, shardRange): + """Returns a list of properties to add to the end of the solrcore.properties file.""" + solrcoreConfig = [] + if sharding != None: + solrcoreConfig.append('solr.port.ssl=8983') + solrcoreConfig.append('shard.instance={}'.format(shardId)) + solrcoreConfig.append('alfresco.port=8080') + solrcoreConfig.append('alfresco.port.ssl=8443') + solrcoreConfig.append('alfresco.baseUrl=/alfresco') + if sharding not in ['DB_ID_RANGE', 'EXPLICIT_ID_FALLBACK_LRIS']: + solrcoreConfig.append('shard.count={}'.format(shardCount)) + if sharding == 'DB_ID_RANGE': + # The first shards each contain 800 nodes by default and the last continues the range to id 100000. + nodesPerShard = shardRange + rangeStart = shardId * nodesPerShard + rangeEnd = (rangeStart + nodesPerShard - 1 if shardId < shardCount - 1 else 100000) + solrcoreConfig.append('shard.range={}-{}'.format(rangeStart, rangeEnd)) + if sharding == 'DATE': + solrcoreConfig.append('shard.key=cm:created') + solrcoreConfig.append('shard.date.grouping={}'.format(12 // shardCount)) + if sharding in ['PROPERTY', 'EXPLICIT_ID', 'EXPLICIT_ID_FALLBACK_LRIS']: + solrcoreConfig.append('shard.key=shard:shardId') + print("SolrConfig for Shard: ", shardId, " : ", solrcoreConfig) + return solrcoreConfig + +def getSolrcoreReplacements(sharding, mtls): + """Returns a dict of replacements to make in the solrcore.properties file.""" + solrcoreReplacements = {} + if sharding != None: + solrcoreReplacements['shard.method=DB_ID'] = 'shard.method={}'.format(sharding) + if mtls: + solrcoreReplacements['alfresco.secureComms=none'] = 'alfresco.secureComms=https' + solrcoreReplacements['alfresco.encryption.ssl.keystore.location=.*'] = 'alfresco.encryption.ssl.keystore.location=\\\\\\/opt\\\\\\/alfresco-search-services\\\\\\/keystore\\\\\\/ssl-repo-client.keystore' + solrcoreReplacements['alfresco.encryption.ssl.keystore.type=.*'] = 'alfresco.encryption.ssl.keystore.type=JCEKS' + solrcoreReplacements['alfresco.encryption.ssl.truststore.location=.*'] = 'alfresco.encryption.ssl.truststore.location=\\\\\\/opt\\\\\\/alfresco-search-services\\\\\\/keystore\\\\\\/ssl-repo-client.truststore' + solrcoreReplacements['alfresco.encryption.ssl.truststore.type=.*'] = 'alfresco.encryption.ssl.truststore.type=JCEKS' + else: + solrcoreReplacements['alfresco.secureComms=https'] = 'alfresco.secureComms=none' + return solrcoreReplacements + +def addAlfrescoMtlsConfig(alfrescoArgsNode): + """Add a list of environment values in Docker Compose Alfresco Service for mTLS.""" + alfrescoArgsNode['TRUSTSTORE_TYPE'] = 'JCEKS' + alfrescoArgsNode['TRUSTSTORE_PASS'] = 'kT9X6oe68t' + alfrescoArgsNode['KEYSTORE_TYPE'] = 'JCEKS' + alfrescoArgsNode['KEYSTORE_PASS'] = 'kT9X6oe68t' + alfrescoArgsNode['SOLR_COMMS'] = 'https' + +def addAlfrescoVolumes(alfrescoNode): + """Add route to keystores folder""" + alfrescoNode['volumes'] = ['./keystores/alfresco:/usr/local/tomcat/alf_data/keystore'] + +def addSolrMtlsConfig(solrEnvNode): + """Add a list of environment values in Docker Compose SOLR Service for mTLS.""" + solrEnvNode['SOLR_SSL_TRUST_STORE'] = '/opt/alfresco-search-services/keystore/ssl-repo-client.truststore' + solrEnvNode['SOLR_SSL_TRUST_STORE_TYPE'] = 'JCEKS' + solrEnvNode['SOLR_SSL_KEY_STORE'] = '/opt/alfresco-search-services/keystore/ssl-repo-client.keystore' + solrEnvNode['SOLR_SSL_KEY_STORE_TYPE'] = 'JCEKS' + solrEnvNode['SOLR_SSL_NEED_CLIENT_AUTH'] = 'true' + +def addSolrOpts(solrEnvNode): + """Add a list of values to add in Docker Compose SOLR_OPTS property for mTLS.""" + solrOptions = ' '.join(['-Dsolr.ssl.checkPeerName=false', + '-Dsolr.allow.unsafe.resourceloading=true']) + solrEnvNode['SOLR_OPTS'] = solrOptions + +def addSolrJavaToolOptions(solrEnvNode): + """Add a list of values to add in Docker Compose JAVA_TOOL_OPTIONS property for mTLS.""" + solrOptions = ' '.join(['-Dsolr.jetty.truststore.password=kT9X6oe68t ', + '-Dsolr.jetty.keystore.password=kT9X6oe68t ', + '-Dssl-keystore.password=kT9X6oe68t', + '-Dssl-keystore.aliases=ssl-alfresco-ca,ssl-repo-client', + '-Dssl-keystore.ssl-alfresco-ca.password=kT9X6oe68t', + '-Dssl-keystore.ssl-repo-client.password=kT9X6oe68t', + '-Dssl-truststore.password=kT9X6oe68t', + '-Dssl-truststore.aliases=ssl-alfresco-ca,ssl-repo,ssl-repo-client', + '-Dssl-truststore.ssl-alfresco-ca.password=kT9X6oe68t', + '-Dssl-truststore.ssl-repo.password=kT9X6oe68t', + '-Dssl-truststore.ssl-repo-client.password=kT9X6oe68t']) + solrEnvNode['JAVA_TOOL_OPTIONS'] = solrOptions + +def addSolrVolumes(solrNode): + """Add route to keystores folder""" + solrNode['volumes'] = ['./keystores/solr:/opt/alfresco-search-services/keystore'] + +def makeSearchNode(outputDirectory, nodeName, externalPort, params, mtls, extraEnvironmentVars={}, solrcoreConfig=[], solrcoreReplacements={}): + # Create a dictionary for the template replacement. + allParams = dict(params) + allParams['SOLR_HOST'] = nodeName + allParams['ALFRESCO_PORT'] = 8443 if mtls else 8080 + allParams['EXTERNAL_PORT'] = externalPort + # Properties to add to solrcore.properties. + allParams['SOLRCORE_PROPERTIES'] = '\\n'.join(solrcoreConfig) + # Replacements to make in solrcore.properties (in an "abc/xyz" format suitable for sed). + allParams['SOLRCORE_REPLACEMENTS'] = ' '.join(map(lambda pair: '"{}/{}"'.format(*pair), solrcoreReplacements.items())) + + # mTLS settings + allParams['ALFRESCO_SECURE_COMMS'] = ('https' if mtls else 'none') + allParams['TRUSTSTORE_TYPE'] = 'JCEKS' + allParams['KEYSTORE_TYPE'] = 'JCEKS' + + # Create a Dockerfile with any extra configuration in. + with open(scriptDir + '/templates/search/Dockerfile.template') as f: + dockerfileTemplate = f.read() + dockerfileString = Template(dockerfileTemplate).substitute(allParams) + if not os.path.isdir('{}/{}'.format(outputDirectory, nodeName)): + os.mkdir('{}/{}'.format(outputDirectory, nodeName)) + with open('{}/{}/Dockerfile'.format(outputDirectory, nodeName), 'w') as f: + dockerfileTemplate = f.write(dockerfileString) + + # Load the search node template. + with open(scriptDir + '/templates/search-node.yml.template') as f: + searchNodeTemplate = f.read() + searchNodeString = Template(searchNodeTemplate).substitute(allParams) + # Read the result as yaml. + searchNodeYaml = yaml.safe_load(searchNodeString) + # Add any extra environment variables. + searchNodeYaml['environment'].update(extraEnvironmentVars) + + # Add mTLS configuration if required + if mtls: + addSolrMtlsConfig(searchNodeYaml['environment']) + addSolrOpts(searchNodeYaml['environment']) + addSolrJavaToolOptions(searchNodeYaml['environment']) + addSolrVolumes(searchNodeYaml) + + return searchNodeYaml + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Generate a docker-compose file for ACS.') + parser.add_argument('-a', '--alfresco', default='quay.io/alfresco/dev:acs-for-search', help='The Alfresco image') + parser.add_argument('-s', '--search', default='quay.io/alfresco/search-services:latest', help='The Search image') + parser.add_argument('-e', '--share', help='The Share image (or omit for no UI)') + parser.add_argument('-p', '--postgres', default='postgres:10.1', help='The Postgres image') + parser.add_argument('-q', '--excludeAMQ', action='store_true', help='Exclude ActiveMQ (i.e. pre-ACS 6.1)') + parser.add_argument('-t', '--transformer', choices=[LIBRE_OFFICE, LEGACY_TRANSFORMERS], help='Use external transformers. ' + + '"{}" for legacy LibreOffice (i.e. ACS 5.2.x). '.format(LIBRE_OFFICE) + + '"{}" for the transformers for use with ACS 6.2.x and later.'.format(LEGACY_TRANSFORMERS)) + parser.add_argument('-c', '--spellcheck', action='store_true', help='Spellcheck Enabled') + parser.add_argument('-ms', '--masterslave', action='store_true', help='Master Slave Enabled') + parser.add_argument('-sh', '--sharding', help='Sharding method (or omit for no sharding). Note that sharding is not supported on SearchServices 1.2.x or earlier.', + choices=['DB_ID', 'DB_ID_RANGE', 'ACL_ID', 'MOD_ACL_ID', 'DATE', 'PROPERTY', 'LRIS', 'EXPLICIT_ID', 'EXPLICIT_ID_FALLBACK_LRIS']) + parser.add_argument('-sc', '--shardCount', type=int, help='Total number of shards to create (default 2)') + parser.add_argument('-sr', '--shardRange', type=int, help='Total number of nodes per shard with DB_ID_RANGE sharding (default 800)') + parser.add_argument('-ct', '--disableCascadeTracking', action='store_true', help='Cascade Tracking Disabled') + parser.add_argument('-o', '--output', default='.', help='The path of the directory to output to') + parser.add_argument('-mtls', '--mtls', action='store_true', help='Use mTLS between SOLR and Alfresco Repository') + args = parser.parse_args() + + # If sharding is selected then the default number of shards is two. + if args.sharding != None and args.shardCount == None: + print('Using default shardCount of 2') + args.shardCount = 2 + elif args.sharding == None and args.shardCount != None: + print('ERROR: shardCount={} specified without sharding method'.format(args.shardCount)) + exit(1) + print('Arguments:', args) + + # If sharding is selected then the default number of nodes per shard is 800. + if args.sharding == "DB_ID_RANGE" and (args.shardRange == None or args.shardRange < 1): + print('Using default shardRange of 800') + args.shardRange = 800 + elif args.sharding != 'DB_ID_RANGE' and args.shardRange != None: + print('ERROR: shardRange={} is only supported for DB_ID_RANGE sharding.') + exit(1) + print('Arguments:', args) + + # Load the template and perform basic token substitution. + scriptDir = os.path.dirname(os.path.realpath(__file__)) + with open(scriptDir + '/templates/docker-compose.yml.template') as f: + template = f.read() + params = { + 'ALFRESCO_IMAGE': args.alfresco, + 'SHARE_IMAGE': args.share, + 'POSTGRES_IMAGE': args.postgres, + 'SEARCH_IMAGE': args.search, + 'SEARCH_ENABLE_SPELLCHECK': str(args.spellcheck).lower(), + 'DISABLE_CASCADE_TRACKING': str(args.disableCascadeTracking).lower() + } + dcString = Template(template).substitute(params) + + # Edit the resulting yaml. + dcYaml = yaml.safe_load(dcString) + + # Insert the search node(s). + shardList = range(args.shardCount) if args.sharding != None else [0] + replicationTypes = ['master', 'slave'] if args.masterslave else ['standalone'] + for shardId in shardList: + for replicationType in replicationTypes: + serviceName = 'search_{}_{}'.format(shardId, replicationType) + # Workaround for ShardInfoTest.getShardInfoWithAdminAuthority. + if shardId == 0 and replicationType == 'standalone': + serviceName = 'search' + externalPort = 8083 + 100 * shardId + (1 if replicationType == 'slave' else 0) + dcYaml['services'][serviceName] = makeSearchNode(args.output, serviceName, externalPort, params, args.mtls, + extraEnvironmentVars=getExtraEnvironmentVars(serviceName, replicationType), + solrcoreConfig=getSolrcoreConfig(args.sharding, shardId, args.shardCount, args.shardRange), + solrcoreReplacements=getSolrcoreReplacements(args.sharding, args.mtls)) + + # Point Alfresco at whichever Solr node came last in the list. + solrHost = serviceName + solrBaseUrl = '/solr-slave' if args.masterslave else '/solr' + + javaOpts = getJavaOpts(not args.excludeAMQ, args.transformer == LEGACY_TRANSFORMERS, args.share != None, solrHost, solrBaseUrl, args.sharding, args.mtls) + dcYaml['services']['alfresco']['environment']['JAVA_OPTS'] = javaOpts + javaToolOpts = getJavaToolOptions(args.mtls) + dcYaml['services']['alfresco']['environment']['JAVA_TOOL_OPTIONS'] = javaToolOpts + if args.mtls: + addAlfrescoMtlsConfig(dcYaml['services']['alfresco']['build']['args']) + addAlfrescoVolumes(dcYaml['services']['alfresco']) + + if not args.share: + deleteServices(dcYaml, 'share', 'alfresco-pdf-renderer', 'imagemagick') + if args.excludeAMQ: + deleteServices(dcYaml, 'activemq') + if args.transformer != LEGACY_TRANSFORMERS: + deleteServices(dcYaml, 'libreoffice', 'tika', 'transform-misc', 'shared-file-store', 'alfresco-pdf-renderer', 'imagemagick') + del(dcYaml['volumes']['shared-file-store-volume']) + if args.transformer == LIBRE_OFFICE: + dcYaml['services']['libreoffice'] = {'image': 'xcgd/libreoffice'} + + # Output the yaml. + with open(args.output + '/docker-compose.yml', 'w') as f: + f.write(yaml.safe_dump(dcYaml)) + + # Create an Alfresco Dockerfile with any extra configuration in. + with open(scriptDir + '/templates/alfresco/Dockerfile.template') as f: + dockerfileTemplate = f.read() + dockerfileString = Template(dockerfileTemplate).substitute(params) + if not os.path.isdir('{}/{}'.format(args.output, 'alfresco')): + os.mkdir('{}/{}'.format(args.output, 'alfresco')) + with open('{}/{}/Dockerfile'.format(args.output, 'alfresco'), 'w') as f: + dockerfileTemplate = f.write(dockerfileString) + + # Copy the keystores (when using mTLS) + if args.mtls: + copy_tree(scriptDir + '/keystores', args.output + '/keystores') diff --git a/e2e-test/python-generator/keystores/alfresco/keystore b/e2e-test/python-generator/keystores/alfresco/keystore new file mode 100644 index 0000000000000000000000000000000000000000..439bc1f861272ed97542fa9242f42ffd04e16988 GIT binary patch literal 327 zcmXqLVsvI=WHxC0%f_kI=F#?@myw-uLF0Rt#y19yFAW->p$Ok#X}oICc*&sgJR3Ju zH4hgf>w+qS3S92M!f1Tik zH9Q_)YD(SXXZ-Z$HgqwNhr5we#8601fFYM5m7#8E2#ZiiW^Ss2 zp{a$bg_(hwfuV_^p@E%&q5&rxt2Q4qlN2ihi%7tUi}~TpGSzd_qXZSXHa(ri<~xB! f#N|lal#HE$%&VpJjhV!L%WjK|cV%K`Y+C>T8B%IO literal 0 HcmV?d00001 diff --git a/e2e-test/python-generator/keystores/alfresco/ssl.keystore b/e2e-test/python-generator/keystores/alfresco/ssl.keystore new file mode 100644 index 0000000000000000000000000000000000000000..c0cff56f8b1b130dd2ce09fdfbf9175e17aedae3 GIT binary patch literal 3288 zcmeH}do+~$8pr3Icdll}Z5ZW#t7NbAPJ>Kp8>vtdhQu!0#K>)CG(&Ec88U~YLTqtd z3Po-0lx&v>gR&LbF4cAy*>0OfWvYWSlU=9PY3)DGT4(jo{^PgS`#kUSuJwC=&+q&B zKCP{-tuPo2{&C6ic!3)jfj(R&&x^CcivfcHN8KL$VhuwcFfxItlF~^i8a^gj4!Iy4 zfWwszYnREr+s#`T#k{T5Fr-}A>fq)C5l~n|p^b2X001~742mM-AS?=WL7<3m>+K%` z3?A;XhfIVNq;Jr8nY|1@ZzkK9$?zs?K^oGe44!29=U%kw9G~D&2A8RA7wm0BmWO1e z1vos~g&n}=gtEzM&}wN0#4B2b@Pauk?T=qtSwfmBgiR1+44IKPQH(c{-Bk$2(vb3J z`1Lm^$ocfl03-|)B*Gv;JX|1vO@>%jj_+@-npB|{k;_4ub;DVMvgm8r2L-R9=JtlD zFN$+)>qRM7en_7kq5R?Hq>$f~B%fY-`Gnb+m#%lXR=s#4!H}eQFzyE)0p&iESNf{* z;en9$g%?DmPSr92Td3?#C3RQaD<~O1T-Mk?khPO-Q=HyY7jqtaEe9Y8;2;14@;o7r z_f64|MUW6mSj<3u!=3)AoAkRDnJK5~Vk6f+NK4WH4L!svh{XSL0bW`_L?9s$0G8i( zP(x1ggg{8HlJL_5kX#y2^5^`R-^g7#~ZnGaWmGazA5TSa<49a3&lxf0l+X)NpP7# zoE0rJdrL%cMIwO9kU9$c4GKU2hHxSP$ssT?V$W`Uk^j$d5G2qL(QrQ8Db`y*apH)- z#cb(dZ%|b~+LFgg%E#9xx2cl(sojZuP>g)6>q1LRw@d0w{_ah$(dnDzI{S>bF0nc} z>A1?+B8xGH{rC5u#Vz-8Ot+N-3UnWvE#!Q+G+cyE?LXGI&ZI;QxJ~^;j(>9Sj=Jpd z<^1H;P5yPK2QYEC4q)_o=5)tZvctup$0A*&{IH~U>JjCz|28-(k6J(-p#nrWN@92{3Pd zbMlnnrGc?W5%C6w_FymjEO})7`L=y;f7W*g4$l=OHpOjrWzhpvj+4%3)}Lu{5tQHY zCxR9&O>Qm+e4fN~nfExI5MQrTs3+dF+O^>HG^jsyWNB;E*{*R#BZTK9Rkk1tslw&sG+Gr7XuB^B@zvVU-%n6Kew-5M5*r4H9`>2!zCH&3o6{62r%gX_n)|^!zF;K( zgJ!aQ-b-=Xgwew5%_7G9W3Ywz< zE))&0-&?E%uW$$^@l>p*&;}5Sw*!C_uP7Qs`A>>~SWD917sO=K>2})IOg59t2xfX~ z)0y03Os=*SlN;>kn_Inmg($CV}_JdW*%Y{xhCr&LCp&C1!a)n_Lrf5 zNpLD(Ao%K6EBecWigb$j=t0O_h$xNMAn=e33L{}V5{U)@NJ*MhKoB5YG&Ydq#R&A} z@Pa?nj?}Jp&cJ$qwD<5q9UZOSg}$`kYUB0%s)W!mb!H8&`9cX5Wp^N~r~+;CPsig5 zLLz2v=O*{4OGndPTPpgD@rX+&eu&&WME({DoMWbfPpna&(y#YsO+^t6Vr1b4p_j< z#K^=XlBBSW|F>81_op1ccFAfPPq7Jz3^L$l8#k_1$?-y_Ry6a3t>BaBQgjp4L|L5ggAsS^Vc5UbWr5j%+Z995|m(Pdq zywF?kgUu5-SIuV>Yh+?(WMEvJU=Rn66j?qNF%}U{j{Ap>Y`ynpL+~E^*^|41f9QqZ zHIN5JfUGi$gn?KCb_F~j1;Q+>2F#3%|G_aR%MWC+umF=%8*%^u6Ad%a=M6`{uVCic ztKo3v*0X|}&t`q@VlPh?%ROqn|L8Q`kjvTO!XGNKFL7ATJ-H%owt+^u+|QnRk+s=gH3oi(>rkHzOTL+F3T z-fwrzR{3#v-YS}a?TJx)*Zwl-0A~sQ;^G|LqSOLlcF4(0%_{+BiNeSeCr-h$ z1T!!@Fh2%jA4ry95@1C0%fP2E)C^z*%mBuq3}6P%07g*;m9R{SC5Muc8MO>FAem7S zJu?OY1F|@?B)_Oq!5JK!2KbWx<@U`^uX&p;On|L>xprUR3I_I72@!97|9z4dSiJqsJ~PQm)^&v+4V32CMQHnduDxD3 z*?DDa^ZLh^-}%^0YB;0p8+pX9({EL3#_Sn9GRse|stD*7Pg#3s3;)luO{{;Lu9`$b zl716okwFt<9=HetCSN8424I=Rnb79J*!IJTkr9-9*&Gav5kkByYGNw>1*v(#!9EJ^ zsd=eIi6yBi3c;yGWvN99&Z$KunQ58HKry5oWmb2Bzdr2P`RmK8%!CE2HYW%t#~L*5 zgJh}3Z3c~-v1h7uzN3ox1h!t~-pU zzZ^E(cR}Mt*tBAXllGStXK@AXf870bfg;<^|IL+bITwyFKAaf%%=HCFdsd3^dBc;& z|NMOg*UVg@Hp9?x5A%wPN|VJl?!Eo>JiEVj^y4-`lX-d3K4E+xle=G>%USV9kR|)i cHGauea=oD&UI|J%%dUKLv}lRl!@X5e07h+FmjD0& literal 0 HcmV?d00001 diff --git a/e2e-test/python-generator/keystores/solr/ssl-repo-client.keystore b/e2e-test/python-generator/keystores/solr/ssl-repo-client.keystore new file mode 100644 index 0000000000000000000000000000000000000000..f1474dd04f3cbe8fed7fe9e6f68daf658e75e181 GIT binary patch literal 3298 zcmeH}do+~$8pr3I8^g>nrbfhZOWPs#J4P;vjoBp`F2t(2z;g{ba zH~-@^0}v1xHyHwQ&qKLf$V{kd{d1>|h8byta^m+ev9HFj{E8K+K(&`FpINjDmD^m+ zx48cxy)-jx;WyG8%??-GmL{2XyD(aRil$7DRJgzTB1xSf<(81i#vonii*7I1cR7dt zv@(fDC^c+hP-Qz^4G6!~HJ8*p_o{7e!C)!a9;sQ&f5rY2Rgn)6xKJ1X0g609kL{+& zW8nn&4wUHA`z?2#yu4xG_)SiFqW!AY@ljAg&;WTA_%4tT^Z5d_umBH7fG_~q+U}sd zgy0Dd7hI+Ij}Jg_X<&jsk7->ggQ1($Onx4k3?7`!ew`>1P9p@g8aB6H(sbzaKaO9B z@P8y~aHXTnBL`HYU*8)ZAEDxIQnS`lFgzO z?hjCXF4f?7B~A-YnSnTN0m;9Q=vG*ZpC4jntIDpG52v{{RlNZd1d;$C5^Oe8lf?)Y z$bsq02;>NIAujE0V*)Ld3Up8i${|r2NB|C~L-7D80T*3&pVRLQ)Xjs!U>FlP4;o_( zfWzTGsMwJ;$;H-a%l&nx2XfsElbmrEBNA#{u?2%!=0B$PR#98D-5tb{+`=H^bI!&E znL}FAeRra&v!gWX59LKu>aYiTtIhJ%L!1&a%!B*vUf|FAMi$Mb+O9A!+B9_Kcww(K z@UoeSmNsMGCLiQwWV)z?cTAeK4)*fHY)?n+nm^8j{%{iYMtLNriGN|CBuM?x!oyhU`_!Ep?NLTU1{kB<~rP z^ot;m`dN*6dncJ05Rc$D?Q1vkI@}ya<|7WaJlv-}@$=Hm8De`J;@f9phljN9N` z@BM?1VeNSfvwm(%Ut3HPD_0ARJWH|}(&@DmS6zqpnI^AQP*wZF>c;yLzY6O~U%zqt z3IFX>_0fbVf08r>lA5aUb=%473Es)8=BjQD-qvzA^gCgjR);*BmmR{y-BYP8T?gyz zM=j1@R(V>C44Bzxs6bCNYD}J8rBCIh8I8ZfE68rtP~7hB7+FpnP|Hl&TB45bT$3Jb zt9S6;gSE^A(ERBE&nA_aIvmbD1%n+wAdy02_RmS$=``CvdpSCfrWH0{5UVDM1zIHiPRB4n^ZgwfFq%0p6W|pFNIUsB6iAKN*({WPjLUZ1fi(b-l zu0{K`l9R_8?|2ig?j|>wQ66rS{XM{80(iqF1dH+x;80266Xkt*8vZw~wFS5)3308v zjcaXJF!H^He8x}y2kTG=)!tc$(x=v8Cy+hcmlMQ_R3vYUAN(^=_no_CHiO|k%BVCX z4fHmq_#s_v4R%*XC+##!I(zDAR{QVknTrk8y$=Sf__k#bkqS@eS*y2;3X(;4U-ETj zA`#a^UV+NF#?BhnucHS-(#hpZ|YjwMOc zt43~1tcBe~Z1VbbT;7422e3dM5V&o)0@y-9h`?b{p7BS3_*F9i5W*H|0+Rld!oc?u z_Sgh70`2W7isp^*ZoJk#2Kfyo`LU9>*8^z zZ+YTr?jV={&H%WzAoteiE#NBmY%!Pn#cvqz^J9x}rug7UhyoBVj8)-iPz)(5Ks*8= z0t3Ju!Xyrk0mXSJW)O|a^a)~fK9P{ny!PMIsMLxL8TL6q79ZbQMqPjG+mvx8*_w zuJUr(-Yg=q5mxq-ayp@=X?VFu#K!Q%L_AI>FYpA#3HyqcG~LZC+rYv7H-3?j`D9f#yiwCM`2_QILf6({|3{#rV#)D literal 0 HcmV?d00001 diff --git a/e2e-test/python-generator/keystores/solr/ssl-repo-client.truststore b/e2e-test/python-generator/keystores/solr/ssl-repo-client.truststore new file mode 100644 index 0000000000000000000000000000000000000000..c09e6891a0e5e89969690e53b91828140b6e4e16 GIT binary patch literal 2883 zcmX?i?%X*B1_mY|W(L#z#l<Q3<2haEeAeR-9cuwd2Z1mWaZgT{TpV3bv6 zY20SexVb@uljHuOBU|si*$}+Pe)i<9;2(P7cNaFcENN^U@D$c0lBBSW|F>81_op1c zcFAfPPq7Jz3<4)oS$;;w|DZr(U0@&#@{cMDj{z4Ohc+7{D=RxQqk#xWP>_Y!fV+t^ zCqFqcCnLYO1UXRvvk)^dHJBdIUZdAscss+&nfuZIubk^{WbU21_3y4bjHkaGHrsbW z<3-rCVuq9UmlbDm1?_*_{dIvN+s^;Zm25c|jxau)82HTf1xI^Uit%~Flg9u2eFfLd zT%k6@&~Oj)ii=8<#WwD}{q;P%zjgHEHbIkldC@*$d>@m$U!2QX@kfv)`_DCgNknmw z2+fqb$%&BS;AkqmIAD5h(8L5R5|{!Y#R2w!8QcYgA+Ufj0TmF&-~z%hYQWPM-eTf) zspGDfVJCOL7Pnk!xPzHX{n6Y9eC(?@Pb~df|H-#h=I@XBo`()kUN(KkhbPAGl7j@7 z9GfUGW6Ppii|5HoDV6eve!S||5)+E)m|o1w7WIDdman_cM3i3q{!Exvarb{-&K06j zmSWd-?q9m`Wzx2zM|k;s_|6Nx^*-12k1V+Twzc5T-R6lWt*aZ)1n&7Cx5vbg zVe07vZlO}MuI8$~6xy9Nw^fhD=QTsHP^(5z%15difhoU;x>cjHfg&)Dfz`AW z?y3=QV!zwGb=6l&=8Y(r!_AZ!qSNZe9#k*#h zOIxhF9VhO&ZrX0oACDMR_^%rm?N+l+F3eiwBYe=RT&``gbH?W@ehe*#msKonm^p97 zzVBH-961gJ&uVONi1tzWP~Ojc?(ChbGK^-&zJ+|rb1FT2nT~a$ulmf^jBSeE(^5-! z7{5QmDO6;tIa(*eo0XtC(e*vUoi%=OX>5g0_qwq!40G0By<~THd4){sKJJrKHd?az zL{zNY$?h>Hutu;=nB$YHQB?iH!WrSGciuIOT`~7jz_idbjTQg?c}?a|JJ0B$b?oc| z1|D|P(uP;T{dNCpci7rlIJ`CV^vs)aM6kI$H#_g%fxITZ3+I#1@n~LQS^4{?#e~B$ UK6_e{>&hArKW8|-kn?aG0D2FY!vFvP literal 0 HcmV?d00001 diff --git a/e2e-test/python-generator/templates/alfresco/Dockerfile.template b/e2e-test/python-generator/templates/alfresco/Dockerfile.template new file mode 100755 index 000000000..3438f963e --- /dev/null +++ b/e2e-test/python-generator/templates/alfresco/Dockerfile.template @@ -0,0 +1,49 @@ +FROM ${ALFRESCO_IMAGE} + +ENV TOMCAT_DIR=/usr/local/tomcat +ENV ALF_DATA_DIR=$${TOMCAT_DIR}/alf_data + +# COMMS +ARG SOLR_COMMS +ENV SOLR_COMMS $$SOLR_COMMS + +# SSL +ARG TRUSTSTORE_TYPE +ARG TRUSTSTORE_PASS +ARG KEYSTORE_TYPE +ARG KEYSTORE_PASS + +ENV TRUSTSTORE_TYPE=$$TRUSTSTORE_TYPE \ + TRUSTSTORE_PASS=$$TRUSTSTORE_PASS \ + KEYSTORE_TYPE=$$KEYSTORE_TYPE \ + KEYSTORE_PASS=$$KEYSTORE_PASS + +USER root + +# Default value in 'repository.properties' is 'dir.keystore=classpath:alfresco/keystore' +RUN if [ "$$SOLR_COMMS" == "https" ] ; then \ + echo -e "\n\ + dir.keystore=$${ALF_DATA_DIR}/keystore\n\ + alfresco.encryption.ssl.keystore.type=$${TRUSTSTORE_TYPE}\n\ + alfresco.encryption.ssl.truststore.type=$${KEYSTORE_TYPE}\n\ + " >> $${TOMCAT_DIR}/shared/classes/alfresco-global.properties; \ + fi + +# Enable SSL by adding the proper Connector to server.xml +RUN if [ "$$SOLR_COMMS" == "https" ] ; then \ + sed -i "s/\ +[[:space:]]\+<\/Engine>/\n\ + <\/Engine>\n\ + \n\ + <\/Connector>/g" $${TOMCAT_DIR}/conf/server.xml; \ + fi + +# Expose keystore folder +# Useless for 'none'/'http' communications with SOLR +VOLUME ["$$ALF_DATA_DIR/keystore"] diff --git a/e2e-test/python-generator/templates/docker-compose.yml.template b/e2e-test/python-generator/templates/docker-compose.yml.template new file mode 100644 index 000000000..b2c3f0145 --- /dev/null +++ b/e2e-test/python-generator/templates/docker-compose.yml.template @@ -0,0 +1,101 @@ +version: '3' +services: + alfresco: + build: + context: ./alfresco + args: + SOLR_COMMS: none + environment: + JAVA_OPTS : "This will be populated by the generator script" + JAVA_TOOL_OPTIONS : "This will be populated by the generator script" + ports: + - "7203:7203" #JMX connect via service:jmx:rmi:///jndi/rmi://localhost:7203/jmxrmi + - "5005:5005" #Java debugging + - "8081:8080" #Browser port for Alfresco + share: + image: ${SHARE_IMAGE} + environment: + - REPO_HOST=alfresco + - REPO_PORT=8080 + ports: + - 8082:8080 #Browser port for Share + postgres: + image: ${POSTGRES_IMAGE} + environment: + - POSTGRES_PASSWORD=alfresco + - POSTGRES_USER=alfresco + - POSTGRES_DB=alfresco + ports: + - 5432:5432 + activemq: + image: alfresco/alfresco-activemq:5.15.6 + ports: + - 8161:8161 # Web Console + - 5672:5672 # AMQP + - 61616:61616 # OpenWire + - 61613:61613 # STOMP + alfresco-pdf-renderer: + image: alfresco/alfresco-pdf-renderer:2.1.0-EA4 + environment: + JAVA_OPTS: " -Xms256m -Xmx512m" + ACTIVEMQ_URL: "nio://activemq:61616" + FILE_STORE_URL: "http://shared-file-store:8099/alfresco/api/-default-/private/sfs/versions/1/file" + ports: + - 8090:8090 + links: + - activemq + imagemagick: + image: alfresco/alfresco-imagemagick:2.1.0-EA4 + environment: + JAVA_OPTS: " -Xms256m -Xmx512m" + ACTIVEMQ_URL: "nio://activemq:61616" + FILE_STORE_URL: "http://shared-file-store:8099/alfresco/api/-default-/private/sfs/versions/1/file" + ports: + - 8091:8090 + links: + - activemq + libreoffice: + image: alfresco/alfresco-libreoffice:2.1.0-EA4 + environment: + JAVA_OPTS: " -Xms256m -Xmx512m" + ACTIVEMQ_URL: "nio://activemq:61616" + FILE_STORE_URL: "http://shared-file-store:8099/alfresco/api/-default-/private/sfs/versions/1/file" + ports: + - 8092:8090 + links: + - activemq + tika: + image: alfresco/alfresco-tika:2.1.0-EA4 + environment: + JAVA_OPTS: " -Xms256m -Xmx512m" + ACTIVEMQ_URL: "nio://activemq:61616" + FILE_STORE_URL: "http://shared-file-store:8099/alfresco/api/-default-/private/sfs/versions/1/file" + ports: + - 8093:8090 + links: + - activemq + transform-misc: + image: alfresco/alfresco-transform-misc:2.1.0-EA4 + environment: + JAVA_OPTS: " -Xms256m -Xmx512m" + ACTIVEMQ_URL: "nio://activemq:61616" + FILE_STORE_URL: "http://shared-file-store:8099/alfresco/api/-default-/private/sfs/versions/1/file" + ports: + - 8094:8090 + links: + - activemq + shared-file-store: + image: alfresco/alfresco-shared-file-store:0.5.3 + environment: + JAVA_OPTS: " -Xms256m -Xmx512m" + scheduler.content.age.millis: 86400000 + scheduler.cleanup.interval: 86400000 + ports: + - 8099:8099 + volumes: + - shared-file-store-volume:/tmp/Alfresco/sfs +volumes: + shared-file-store-volume: + driver_opts: + type: tmpfs + device: tmpfs diff --git a/e2e-test/python-generator/templates/search-node.yml.template b/e2e-test/python-generator/templates/search-node.yml.template new file mode 100644 index 000000000..502be27f7 --- /dev/null +++ b/e2e-test/python-generator/templates/search-node.yml.template @@ -0,0 +1,17 @@ +build: + context: ./${SOLR_HOST} +environment: + #Solr needs to know how to register itself with Alfresco + SOLR_ALFRESCO_HOST: "alfresco" + SOLR_ALFRESCO_PORT: "${ALFRESCO_PORT}" + #Alfresco needs to know how to call solr + SOLR_SOLR_HOST: "${SOLR_HOST}" + SOLR_SOLR_PORT: "8983" + #Create the default alfresco and archive cores + SOLR_CREATE_ALFRESCO_DEFAULTS: "alfresco,archive" + #Enable Spellcheck by setting to true + ENABLE_SPELLCHECK: "${SEARCH_ENABLE_SPELLCHECK}" + #Disable Cascade Tracking + DISABLE_CASCADE_TRACKING: "${DISABLE_CASCADE_TRACKING}" +ports: + - ${EXTERNAL_PORT}:8983 #Browser port diff --git a/e2e-test/python-generator/templates/search/Dockerfile.template b/e2e-test/python-generator/templates/search/Dockerfile.template new file mode 100644 index 000000000..70d97b9a7 --- /dev/null +++ b/e2e-test/python-generator/templates/search/Dockerfile.template @@ -0,0 +1,26 @@ +FROM ${SEARCH_IMAGE} + +# Create search_config_setup.sh if it does not exist (e.g. on SearchServices 1.2.x or earlier). +USER root +RUN touch $${DIST_DIR}/solr/bin/search_config_setup.sh \ + && chown solr:solr $${DIST_DIR}/solr/bin/search_config_setup.sh +USER solr + +RUN replacementPairs=(${SOLRCORE_REPLACEMENTS}); \ + for replacementPair in $${replacementPairs[@]}; \ + do \ + sed -i '/^bash.*/i sed -i "'"s/$$replacementPair/g"'" $${DIST_DIR}/solrhome/templates/rerank/conf/solrcore.properties\n' \ + $${DIST_DIR}/solr/bin/search_config_setup.sh; \ + done; \ + if [[ "${SOLRCORE_PROPERTIES}" != "" ]]; \ + then \ + sed -i '/^bash.*/i echo "\n${SOLRCORE_PROPERTIES}" >> $${DIST_DIR}/solrhome/templates/rerank/conf/solrcore.properties\n' \ + $${DIST_DIR}/solr/bin/search_config_setup.sh; \ + fi + +USER root +RUN mkdir -p /opt/alfresco-search-services/keystore \ + && chown -R solr:solr /opt/alfresco-search-services/keystore +USER solr + +VOLUME ["/opt/alfresco-search-services/keystore"] From 071baea49ef6b2701011249450a412809e7d8693 Mon Sep 17 00:00:00 2001 From: Alex Mukha Date: Wed, 8 Jul 2020 12:13:47 +0100 Subject: [PATCH 06/40] Add quiet flag to install commands --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index a8f8144c8..3c0e3b998 100644 --- a/.travis.yml +++ b/.travis.yml @@ -35,13 +35,13 @@ jobs: name: "Unit tests" jdk: openjdk11 install: - - travis_retry travis_wait 40 mvn --batch-mode clean package -DskipTests + - travis_retry travis_wait 40 mvn --batch-mode -q clean package -DskipTests script: - travis_wait 60 mvn --batch-mode package -DfailIfNoTests=false -Dmaven.wagon.http.pool=false -Dtests.timezone=UTC -Dmaven.test.failure.ignore=true -pl '"!:search-analytics-e2e-test"' - name: "CMIS API tests" jdk: openjdk11 install: - - travis_retry travis_wait 40 mvn --batch-mode clean package -DskipTests + - travis_retry travis_wait 40 mvn --batch-mode -q clean package -DskipTests - travis_retry travis_wait 40 docker build -t quay.io/alfresco/search-services:latest search-services/packaging/target/docker-resources - travis_retry travis_wait 40 docker build -t quay.io/alfresco/insight-engine:latest insight-engine/packaging/target/docker-resources - travis_retry travis_wait 40 docker build -t quay.io/alfresco/insight-zeppelin:latest insight-engine/alfresco-insight-zeppelin/target/docker-resources From 7c5d22b39f82ff3ec5848da71a65555c7d7adf92 Mon Sep 17 00:00:00 2001 From: Alex Mukha Date: Wed, 8 Jul 2020 12:37:26 +0100 Subject: [PATCH 07/40] Rename stages --- .travis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 3c0e3b998..e63d64034 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,10 +20,10 @@ before_cache: # - /release\/.*/ stages: - - name: test + - name: Tests if: commit_message !~ /\[skip tests\]/ - - release - - publish + - Release + - Publish # This should not be required on community build before_install: From 5ad53074f12275e4d1260d5ab419329fd463a8e3 Mon Sep 17 00:00:00 2001 From: Alex Mukha Date: Wed, 8 Jul 2020 14:35:47 +0100 Subject: [PATCH 08/40] Change distribution management and repository URls --- .travis.settings.xml | 17 ++++++- e2e-test/pom.xml | 23 +++++++++ pom.xml | 4 +- search-services/alfresco-search/pom.xml | 37 -------------- .../alfresco-solrclient-lib/pom.xml | 11 ----- search-services/pom.xml | 48 +++++++++++++++++++ 6 files changed, 89 insertions(+), 51 deletions(-) diff --git a/.travis.settings.xml b/.travis.settings.xml index f0a5039be..bdc0b98db 100644 --- a/.travis.settings.xml +++ b/.travis.settings.xml @@ -1,7 +1,22 @@ - alfresco-internal + alfresco-public-releases + ${env.MAVEN_USERNAME} + ${env.MAVEN_PASSWORD} + + + alfresco-public-snapshots + ${env.MAVEN_USERNAME} + ${env.MAVEN_PASSWORD} + + + alfresco-enterprise-releases + ${env.MAVEN_USERNAME} + ${env.MAVEN_PASSWORD} + + + alfresco-enterprise-snapshots ${env.MAVEN_USERNAME} ${env.MAVEN_PASSWORD} diff --git a/e2e-test/pom.xml b/e2e-test/pom.xml index e6f71d099..a9fbb8c70 100644 --- a/e2e-test/pom.xml +++ b/e2e-test/pom.xml @@ -128,4 +128,27 @@ 12-ea+10 + + + + alfresco-enterprise-releases + https://artifacts.alfresco.com/nexus/content/repositories/enterprise-releases + + true + + + false + + + + alfresco-enterprise-snapshots + https://artifacts.alfresco.com/nexus/content/repositories/enterprise-snapshots + + false + + + true + + + \ No newline at end of file diff --git a/pom.xml b/pom.xml index 9bb3bdfd7..9aaa80001 100644 --- a/pom.xml +++ b/pom.xml @@ -12,11 +12,11 @@ Alfresco Search And Insight Parent - alfresco-internal + alfresco-enterprise-releases https://artifacts.alfresco.com/nexus/content/repositories/enterprise-releases/ - alfresco-internal-snapshots + alfresco-enterprise-snapshots https://artifacts.alfresco.com/nexus/content/repositories/enterprise-snapshots/ diff --git a/search-services/alfresco-search/pom.xml b/search-services/alfresco-search/pom.xml index 091ce549a..57d12ef46 100644 --- a/search-services/alfresco-search/pom.xml +++ b/search-services/alfresco-search/pom.xml @@ -206,43 +206,6 @@ - - - alfresco-public - https://artifacts.alfresco.com/nexus/content/groups/public - - true - - - false - - - - alfresco-public-snapshots - https://artifacts.alfresco.com/nexus/content/groups/public-snapshots - - false - - - true - - - - central - Central Repository - https://repo.maven.apache.org/maven2 - default - - false - - - - maven-restlet - Public online Restlet repository - http://maven.restlet.talend.com - - - alfresco-solr diff --git a/search-services/alfresco-solrclient-lib/pom.xml b/search-services/alfresco-solrclient-lib/pom.xml index be8043b36..9235252de 100644 --- a/search-services/alfresco-solrclient-lib/pom.xml +++ b/search-services/alfresco-solrclient-lib/pom.xml @@ -10,17 +10,6 @@ 2.0.0-SNAPSHOT - - - alfresco-internal - https://artifacts.alfresco.com/nexus/content/repositories/releases - - - alfresco-internal-snapshots - https://artifacts.alfresco.com/nexus/content/repositories/snapshots - - - 8.123 2.10.3 diff --git a/search-services/pom.xml b/search-services/pom.xml index 4d2570772..5665651e4 100644 --- a/search-services/pom.xml +++ b/search-services/pom.xml @@ -23,4 +23,52 @@ alfresco-search packaging + + + + alfresco-public-releases + https://artifacts.alfresco.com/nexus/content/repositories/releases + + + alfresco-public-snapshots + https://artifacts.alfresco.com/nexus/content/repositories/snapshots + + + + + + alfresco-public + https://artifacts.alfresco.com/nexus/content/groups/public + + true + + + false + + + + alfresco-public-snapshots + https://artifacts.alfresco.com/nexus/content/groups/public-snapshots + + false + + + true + + + + central + Central Repository + https://repo.maven.apache.org/maven2 + default + + false + + + + maven-restlet + Public online Restlet repository + http://maven.restlet.talend.com + + From d191bc85ca05e82f5fbe760445968c8a52e3558a Mon Sep 17 00:00:00 2001 From: Alex Mukha Date: Wed, 8 Jul 2020 15:44:04 +0100 Subject: [PATCH 09/40] Add internal releases repository --- .travis.settings.xml | 5 +++++ e2e-test/pom.xml | 11 +++++++++++ 2 files changed, 16 insertions(+) diff --git a/.travis.settings.xml b/.travis.settings.xml index bdc0b98db..e90586603 100644 --- a/.travis.settings.xml +++ b/.travis.settings.xml @@ -20,5 +20,10 @@ ${env.MAVEN_USERNAME} ${env.MAVEN_PASSWORD} + + alfresco-internal-releases + ${env.MAVEN_USERNAME} + ${env.MAVEN_PASSWORD} + diff --git a/e2e-test/pom.xml b/e2e-test/pom.xml index a9fbb8c70..ebf335a79 100644 --- a/e2e-test/pom.xml +++ b/e2e-test/pom.xml @@ -150,5 +150,16 @@ true + + + alfresco-internal-releases + https://artifacts.alfresco.com/nexus/content/repositories/internal-releases + + true + + + false + + \ No newline at end of file From 121a045e78dfb834969c45b95d5ab84735d1b61b Mon Sep 17 00:00:00 2001 From: Alex Mukha Date: Wed, 8 Jul 2020 16:27:08 +0100 Subject: [PATCH 10/40] Increase timeouts for install scripts --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index e63d64034..c68647688 100644 --- a/.travis.yml +++ b/.travis.yml @@ -35,13 +35,13 @@ jobs: name: "Unit tests" jdk: openjdk11 install: - - travis_retry travis_wait 40 mvn --batch-mode -q clean package -DskipTests + - travis_retry travis_wait 60 mvn --batch-mode -q clean package -DskipTests script: - travis_wait 60 mvn --batch-mode package -DfailIfNoTests=false -Dmaven.wagon.http.pool=false -Dtests.timezone=UTC -Dmaven.test.failure.ignore=true -pl '"!:search-analytics-e2e-test"' - name: "CMIS API tests" jdk: openjdk11 install: - - travis_retry travis_wait 40 mvn --batch-mode -q clean package -DskipTests + - travis_retry travis_wait 60 mvn --batch-mode -q clean package -DskipTests - travis_retry travis_wait 40 docker build -t quay.io/alfresco/search-services:latest search-services/packaging/target/docker-resources - travis_retry travis_wait 40 docker build -t quay.io/alfresco/insight-engine:latest insight-engine/packaging/target/docker-resources - travis_retry travis_wait 40 docker build -t quay.io/alfresco/insight-zeppelin:latest insight-engine/alfresco-insight-zeppelin/target/docker-resources From fcba92611aa7420dcdd57ea7a77bcb1ead4cf087 Mon Sep 17 00:00:00 2001 From: Alex Mukha Date: Wed, 8 Jul 2020 17:05:14 +0100 Subject: [PATCH 11/40] Rename test stage --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index c68647688..d5c7d975d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -31,7 +31,7 @@ before_install: jobs: include: - - stage: test + - stage: Tests name: "Unit tests" jdk: openjdk11 install: From 1027da8be59e847fcdcda91d7025ed52cd320f0b Mon Sep 17 00:00:00 2001 From: Alex Mukha Date: Wed, 8 Jul 2020 17:52:02 +0100 Subject: [PATCH 12/40] Add HF repository --- .travis.settings.xml | 3 ++- e2e-test/pom.xml | 5 ++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.travis.settings.xml b/.travis.settings.xml index e90586603..0ce1436a4 100644 --- a/.travis.settings.xml +++ b/.travis.settings.xml @@ -20,8 +20,9 @@ ${env.MAVEN_USERNAME} ${env.MAVEN_PASSWORD} + - alfresco-internal-releases + alfresco-hotfix ${env.MAVEN_USERNAME} ${env.MAVEN_PASSWORD} diff --git a/e2e-test/pom.xml b/e2e-test/pom.xml index ebf335a79..c294bd34f 100644 --- a/e2e-test/pom.xml +++ b/e2e-test/pom.xml @@ -150,10 +150,9 @@ true - - alfresco-internal-releases - https://artifacts.alfresco.com/nexus/content/repositories/internal-releases + alfresco-hotfix + https://artifacts.alfresco.com/nexus/content/groups/hotfix true From 89721afe934bc176fae0909000f75046e72fba9e Mon Sep 17 00:00:00 2001 From: Alex Mukha Date: Wed, 8 Jul 2020 19:44:43 +0100 Subject: [PATCH 13/40] Add quay.io login --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index d5c7d975d..3a4b8da61 100644 --- a/.travis.yml +++ b/.travis.yml @@ -42,6 +42,7 @@ jobs: jdk: openjdk11 install: - travis_retry travis_wait 60 mvn --batch-mode -q clean package -DskipTests + - docker login quay.io -u ${QUAY_USERNAME} -p ${QUAY_PASSWORD} - travis_retry travis_wait 40 docker build -t quay.io/alfresco/search-services:latest search-services/packaging/target/docker-resources - travis_retry travis_wait 40 docker build -t quay.io/alfresco/insight-engine:latest insight-engine/packaging/target/docker-resources - travis_retry travis_wait 40 docker build -t quay.io/alfresco/insight-zeppelin:latest insight-engine/alfresco-insight-zeppelin/target/docker-resources From 77c7d6eeb1c28198f91b3763977917b9b9f9c456 Mon Sep 17 00:00:00 2001 From: Alex Mukha Date: Wed, 8 Jul 2020 20:11:05 +0100 Subject: [PATCH 14/40] Add Search API tests --- .travis.yml | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 3a4b8da61..cabbd9e07 100644 --- a/.travis.yml +++ b/.travis.yml @@ -51,4 +51,16 @@ jobs: - ./e2e-test/helpers/wait-service-to-start.sh script: - travis_wait 60 mvn test -f e2e-test/pom.xml -DsuiteXmlFile=src/test/resources/cmis-search.xml -Dtest.exclude=Not_InsightEngine,CONFIG_Master_Slave,Not_Bamboo,AGS_302,CONFIG_Sharding_DB_ID_RANGE,CONFIG_Sharding -Dalfresco.server=localhost -Dalfresco.port=8081 -Dmaven.test.failure.ignore=true - + - name: "SEARCH API tests" + jdk: openjdk11 + install: + - travis_retry travis_wait 60 mvn --batch-mode -q clean package -DskipTests + - docker login quay.io -u ${QUAY_USERNAME} -p ${QUAY_PASSWORD} + - travis_retry travis_wait 40 docker build -t quay.io/alfresco/search-services:latest search-services/packaging/target/docker-resources + - travis_retry travis_wait 40 docker build -t quay.io/alfresco/insight-engine:latest insight-engine/packaging/target/docker-resources + - travis_retry travis_wait 40 docker build -t quay.io/alfresco/insight-zeppelin:latest insight-engine/alfresco-insight-zeppelin/target/docker-resources + - mkdir -p e2e-test/target/docker-resources && python e2e-test/python-generator/generator.py --alfresco=quay.io/alfresco/dev:acs-for-search --transformer=LegacyTransformers --search=quay.io/alfresco/insight-engine:latest --spellcheck --output=e2e-test/target/docker-resources + - ./e2e-test/helpers/start-alfresco.sh "e2e-test/target/docker-resources" "quay.io/alfresco/insight-engine:latest" + - ./e2e-test/helpers/wait-service-to-start.sh + script: + - travis_wait 60 mvn clean test -DsuiteXmlFile=src/test/resources/SearchSuite.xml -Dtest.exclude=Not_InsightEngine,CONFIG_Master_Slave,Not_Bamboo,AGS_302,CONFIG_Sharding_DB_ID_RANGE,CONFIG_Sharding -Dalfresco.server=localhost -Dalfresco.port=8081 -Dmaven.test.failure.ignore=true From 30a2cb46e6968e70f21fdd82490f7ed65f9add9a Mon Sep 17 00:00:00 2001 From: Alex Mukha Date: Wed, 8 Jul 2020 20:38:49 +0100 Subject: [PATCH 15/40] Correct maven command for Search API tests --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index cabbd9e07..ae5d233ba 100644 --- a/.travis.yml +++ b/.travis.yml @@ -63,4 +63,4 @@ jobs: - ./e2e-test/helpers/start-alfresco.sh "e2e-test/target/docker-resources" "quay.io/alfresco/insight-engine:latest" - ./e2e-test/helpers/wait-service-to-start.sh script: - - travis_wait 60 mvn clean test -DsuiteXmlFile=src/test/resources/SearchSuite.xml -Dtest.exclude=Not_InsightEngine,CONFIG_Master_Slave,Not_Bamboo,AGS_302,CONFIG_Sharding_DB_ID_RANGE,CONFIG_Sharding -Dalfresco.server=localhost -Dalfresco.port=8081 -Dmaven.test.failure.ignore=true + - travis_wait 60 mvn clean test -f e2e-test/pom.xml -DsuiteXmlFile=src/test/resources/SearchSuite.xml -Dtest.exclude=Not_InsightEngine,CONFIG_Master_Slave,Not_Bamboo,AGS_302,CONFIG_Sharding_DB_ID_RANGE,CONFIG_Sharding -Dalfresco.server=localhost -Dalfresco.port=8081 -Dmaven.test.failure.ignore=true From f592ce4c6519e633c3874943461794c4e9c0bddd Mon Sep 17 00:00:00 2001 From: Alex Mukha Date: Wed, 8 Jul 2020 20:40:06 +0100 Subject: [PATCH 16/40] Enable branch restrictions --- .travis.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index ae5d233ba..21bac00ba 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,10 +14,10 @@ cache: before_cache: - rm -rf $HOME/.m2/repository/org/alfresco/* -#branches: -# only: -# - master -# - /release\/.*/ +branches: + only: + - master + - /release\/.*/ stages: - name: Tests From 45930b9b5ea9b1b9acf19f629f667b1986168530 Mon Sep 17 00:00:00 2001 From: Alex Mukha Date: Thu, 9 Jul 2020 13:14:31 +0100 Subject: [PATCH 17/40] Add SQL API tests --- .travis.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.travis.yml b/.travis.yml index 21bac00ba..c251f4d26 100644 --- a/.travis.yml +++ b/.travis.yml @@ -64,3 +64,16 @@ jobs: - ./e2e-test/helpers/wait-service-to-start.sh script: - travis_wait 60 mvn clean test -f e2e-test/pom.xml -DsuiteXmlFile=src/test/resources/SearchSuite.xml -Dtest.exclude=Not_InsightEngine,CONFIG_Master_Slave,Not_Bamboo,AGS_302,CONFIG_Sharding_DB_ID_RANGE,CONFIG_Sharding -Dalfresco.server=localhost -Dalfresco.port=8081 -Dmaven.test.failure.ignore=true + - name: "SQL API tests" + jdk: openjdk11 + install: + - travis_retry travis_wait 60 mvn --batch-mode -q clean package -DskipTests + - docker login quay.io -u ${QUAY_USERNAME} -p ${QUAY_PASSWORD} + - travis_retry travis_wait 40 docker build -t quay.io/alfresco/search-services:latest search-services/packaging/target/docker-resources + - travis_retry travis_wait 40 docker build -t quay.io/alfresco/insight-engine:latest insight-engine/packaging/target/docker-resources + - travis_retry travis_wait 40 docker build -t quay.io/alfresco/insight-zeppelin:latest insight-engine/alfresco-insight-zeppelin/target/docker-resources + - mkdir -p e2e-test/target/docker-resources && python e2e-test/python-generator/generator.py --alfresco=quay.io/alfresco/dev:acs-for-search --transformer=LegacyTransformers --search=quay.io/alfresco/insight-engine:latest --spellcheck --output=e2e-test/target/docker-resources + - ./e2e-test/helpers/start-alfresco.sh "e2e-test/target/docker-resources" "quay.io/alfresco/insight-engine:latest" + - ./e2e-test/helpers/wait-service-to-start.sh + script: + - travis_wait 60 mvn clean test -f e2e-test/pom.xml -DsuiteXmlFile=src/test/resources/InsightSuite.xml -Dtest.exclude=Not_InsightEngine,CONFIG_Master_Slave,Not_Bamboo,AGS_302,CONFIG_Sharding_DB_ID_RANGE,CONFIG_Sharding -Dalfresco.server=localhost -Dalfresco.port=8081 -Dmaven.test.failure.ignore=true \ No newline at end of file From 0e8568f20ce61fc84bdf3eed2c30ebdd86b3e909 Mon Sep 17 00:00:00 2001 From: Alex Mukha Date: Thu, 9 Jul 2020 13:21:39 +0100 Subject: [PATCH 18/40] Add Integration tests --- .travis.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index c251f4d26..1d9760a31 100644 --- a/.travis.yml +++ b/.travis.yml @@ -76,4 +76,10 @@ jobs: - ./e2e-test/helpers/start-alfresco.sh "e2e-test/target/docker-resources" "quay.io/alfresco/insight-engine:latest" - ./e2e-test/helpers/wait-service-to-start.sh script: - - travis_wait 60 mvn clean test -f e2e-test/pom.xml -DsuiteXmlFile=src/test/resources/InsightSuite.xml -Dtest.exclude=Not_InsightEngine,CONFIG_Master_Slave,Not_Bamboo,AGS_302,CONFIG_Sharding_DB_ID_RANGE,CONFIG_Sharding -Dalfresco.server=localhost -Dalfresco.port=8081 -Dmaven.test.failure.ignore=true \ No newline at end of file + - travis_wait 60 mvn clean test -f e2e-test/pom.xml -DsuiteXmlFile=src/test/resources/InsightSuite.xml -Dtest.exclude=Not_InsightEngine,CONFIG_Master_Slave,Not_Bamboo,AGS_302,CONFIG_Sharding_DB_ID_RANGE,CONFIG_Sharding -Dalfresco.server=localhost -Dalfresco.port=8081 -Dmaven.test.failure.ignore=true + - name: "Integration tests" + jdk: openjdk11 + install: + - travis_retry travis_wait 60 mvn --batch-mode -q clean package -DskipTests + script: + - travis_wait 60 mvn -q verify -Dtest=foo -DfailIfNoTests=false -Dtests.timezone=UTC -Dmaven.test.failure.ignore=true -pl '"!:search-analytics-e2e-test"' \ No newline at end of file From a2414f37eeb73f8f5abfc9f3b17d23f02ffe10ff Mon Sep 17 00:00:00 2001 From: Alex Mukha Date: Thu, 9 Jul 2020 13:23:02 +0100 Subject: [PATCH 19/40] Cleanup maven commands --- .travis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1d9760a31..adcaa37d6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -37,7 +37,7 @@ jobs: install: - travis_retry travis_wait 60 mvn --batch-mode -q clean package -DskipTests script: - - travis_wait 60 mvn --batch-mode package -DfailIfNoTests=false -Dmaven.wagon.http.pool=false -Dtests.timezone=UTC -Dmaven.test.failure.ignore=true -pl '"!:search-analytics-e2e-test"' + - travis_wait 60 mvn --batch-mode test -DfailIfNoTests=false -Dtests.timezone=UTC -Dmaven.test.failure.ignore=true -pl '"!:search-analytics-e2e-test"' - name: "CMIS API tests" jdk: openjdk11 install: @@ -63,7 +63,7 @@ jobs: - ./e2e-test/helpers/start-alfresco.sh "e2e-test/target/docker-resources" "quay.io/alfresco/insight-engine:latest" - ./e2e-test/helpers/wait-service-to-start.sh script: - - travis_wait 60 mvn clean test -f e2e-test/pom.xml -DsuiteXmlFile=src/test/resources/SearchSuite.xml -Dtest.exclude=Not_InsightEngine,CONFIG_Master_Slave,Not_Bamboo,AGS_302,CONFIG_Sharding_DB_ID_RANGE,CONFIG_Sharding -Dalfresco.server=localhost -Dalfresco.port=8081 -Dmaven.test.failure.ignore=true + - travis_wait 60 mvn test -f e2e-test/pom.xml -DsuiteXmlFile=src/test/resources/SearchSuite.xml -Dtest.exclude=Not_InsightEngine,CONFIG_Master_Slave,Not_Bamboo,AGS_302,CONFIG_Sharding_DB_ID_RANGE,CONFIG_Sharding -Dalfresco.server=localhost -Dalfresco.port=8081 -Dmaven.test.failure.ignore=true - name: "SQL API tests" jdk: openjdk11 install: @@ -76,7 +76,7 @@ jobs: - ./e2e-test/helpers/start-alfresco.sh "e2e-test/target/docker-resources" "quay.io/alfresco/insight-engine:latest" - ./e2e-test/helpers/wait-service-to-start.sh script: - - travis_wait 60 mvn clean test -f e2e-test/pom.xml -DsuiteXmlFile=src/test/resources/InsightSuite.xml -Dtest.exclude=Not_InsightEngine,CONFIG_Master_Slave,Not_Bamboo,AGS_302,CONFIG_Sharding_DB_ID_RANGE,CONFIG_Sharding -Dalfresco.server=localhost -Dalfresco.port=8081 -Dmaven.test.failure.ignore=true + - travis_wait 60 mvn test -f e2e-test/pom.xml -DsuiteXmlFile=src/test/resources/InsightSuite.xml -Dtest.exclude=Not_InsightEngine,CONFIG_Master_Slave,Not_Bamboo,AGS_302,CONFIG_Sharding_DB_ID_RANGE,CONFIG_Sharding -Dalfresco.server=localhost -Dalfresco.port=8081 -Dmaven.test.failure.ignore=true - name: "Integration tests" jdk: openjdk11 install: From 992357c90e09150bac4d86dd65e0a5f25063e392 Mon Sep 17 00:00:00 2001 From: Alex Mukha Date: Thu, 9 Jul 2020 13:33:52 +0100 Subject: [PATCH 20/40] Fix unit test mvn cleanup --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index adcaa37d6..489f06bec 100644 --- a/.travis.yml +++ b/.travis.yml @@ -37,7 +37,7 @@ jobs: install: - travis_retry travis_wait 60 mvn --batch-mode -q clean package -DskipTests script: - - travis_wait 60 mvn --batch-mode test -DfailIfNoTests=false -Dtests.timezone=UTC -Dmaven.test.failure.ignore=true -pl '"!:search-analytics-e2e-test"' + - travis_wait 60 mvn --batch-mode package -DfailIfNoTests=false -Dtests.timezone=UTC -Dmaven.test.failure.ignore=true -pl '"!:search-analytics-e2e-test"' - name: "CMIS API tests" jdk: openjdk11 install: From b79554f3b9884e4b17ba4655a2699f736b67193c Mon Sep 17 00:00:00 2001 From: Alex Mukha Date: Thu, 9 Jul 2020 14:04:53 +0100 Subject: [PATCH 21/40] Add docker images verification --- .travis.yml | 12 +++++++++++- scripts/test-image.sh | 43 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+), 1 deletion(-) create mode 100755 scripts/test-image.sh diff --git a/.travis.yml b/.travis.yml index 489f06bec..cfc87c8b7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -82,4 +82,14 @@ jobs: install: - travis_retry travis_wait 60 mvn --batch-mode -q clean package -DskipTests script: - - travis_wait 60 mvn -q verify -Dtest=foo -DfailIfNoTests=false -Dtests.timezone=UTC -Dmaven.test.failure.ignore=true -pl '"!:search-analytics-e2e-test"' \ No newline at end of file + - travis_wait 60 mvn -q verify -Dtest=foo -DfailIfNoTests=false -Dtests.timezone=UTC -Dmaven.test.failure.ignore=true -pl '"!:search-analytics-e2e-test"' + - name: "Build and verify Docker images" + jdk: openjdk11 + install: + - travis_retry travis_wait 60 mvn --batch-mode -q clean package -DskipTests + - docker login quay.io -u ${QUAY_USERNAME} -p ${QUAY_PASSWORD} + - travis_retry travis_wait 40 docker build -t quay.io/alfresco/search-services:latest search-services/packaging/target/docker-resources + - travis_retry travis_wait 40 docker build -t quay.io/alfresco/insight-engine:latest insight-engine/packaging/target/docker-resources + - travis_retry travis_wait 40 docker build -t quay.io/alfresco/insight-zeppelin:latest insight-engine/alfresco-insight-zeppelin/target/docker-resources + script: + - travis_wait 60 ./scripts/test-image.sh "${TRAVIS_BUILD_DIR}" "search-services/packaging/target/docker-resources insight-engine/packaging/target/docker-resources insight-engine/alfresco-insight-zeppelin/target/docker-resources" "quay.io/alfresco/search-services quay.io/alfresco/insight-engine quay.io/alfresco/insight-zeppelin" "latest" "alfresco-search-services alfresco-insight-engine zeppelin" \ No newline at end of file diff --git a/scripts/test-image.sh b/scripts/test-image.sh new file mode 100755 index 000000000..a666d14eb --- /dev/null +++ b/scripts/test-image.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +set -eux + +# Sanity test of the docker images. + +# The root directory of the project. +PROJECT_DIRECTORY=$1 +# The relative paths to the dockerfiles. +DOCKER_RESOURCE_HOMES=($2) +# Names of the Docker images to test. +IMAGE_NAMES=($3) +# The version of the image. +IMAGE_VERSION=$4 +# The directories (within /opt) on the images that our products should have been installed in. +DIST_DIRS=($5) + +# The number of projects being processed. +COUNT=${#IMAGE_NAMES[@]} + +for (( i = 0; i < $COUNT; i++ )) +do + IMAGE_NAME=${IMAGE_NAMES[$i]} + DOCKER_RESOURCE_HOME=${DOCKER_RESOURCE_HOMES[$i]} + # There is a variable called DIST_DIR in the image, so use something different. + DIR=${DIST_DIRS[$i]} + + DOCKER_IMAGE_VERSIONED="$IMAGE_NAME:$IMAGE_VERSION" + + cd "$PROJECT_DIRECTORY/$DOCKER_RESOURCE_HOME" + + if [ "$DIR" != "zeppelin" ] + then + docker run --rm $DOCKER_IMAGE_VERSIONED [ -d /opt/$DIR/solr ] || (echo "solr dir does not exist" && exit 1) + docker run --rm $DOCKER_IMAGE_VERSIONED [ -d /opt/$DIR/data/alfrescoModels ] || (echo "alfrescoModels dir does not exist" && exit 1) + docker run --rm $DOCKER_IMAGE_VERSIONED [ -e /opt/$DIR/solr.in.sh ] || (echo "solr.in.sh does not exist" && exit 1) + docker run --rm $DOCKER_IMAGE_VERSIONED grep -q Alfresco /opt/$DIR/solr.in.sh || (echo "solr.in.sh does not contain Alfresco config" && exit 1) + docker run --rm $DOCKER_IMAGE_VERSIONED grep -q Alfresco /opt/$DIR/solr.in.cmd || (echo "solr.in.cmd does not contain Alfresco config" && exit 1) + docker run --rm $DOCKER_IMAGE_VERSIONED grep -q LOG4J_PROPS /opt/$DIR/solr.in.sh || (echo "solr.in.sh does not contain LOG4J_PROPS" && exit 1) + docker run --rm $DOCKER_IMAGE_VERSIONED grep -q LOG4J_CONFIG /opt/$DIR/solr.in.cmd || (echo "solr.in.cmd does not contain LOG4J_CONFIG" && exit 1) + docker run --rm $DOCKER_IMAGE_VERSIONED [ -e /opt/$DIR/solrhome/conf/shared.properties ] || (echo "shared.properties does not exist" && exit 1) + docker run --rm $DOCKER_IMAGE_VERSIONED /opt/$DIR/solr/bin/solr start + fi +done From ccdda07ec11653aa882e0439d312d017a590a31a Mon Sep 17 00:00:00 2001 From: Alex Mukha Date: Fri, 10 Jul 2020 16:23:15 +0100 Subject: [PATCH 22/40] Cleanup unnecessary docker image creation --- .travis.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index cfc87c8b7..e40f7f485 100644 --- a/.travis.yml +++ b/.travis.yml @@ -43,9 +43,7 @@ jobs: install: - travis_retry travis_wait 60 mvn --batch-mode -q clean package -DskipTests - docker login quay.io -u ${QUAY_USERNAME} -p ${QUAY_PASSWORD} - - travis_retry travis_wait 40 docker build -t quay.io/alfresco/search-services:latest search-services/packaging/target/docker-resources - travis_retry travis_wait 40 docker build -t quay.io/alfresco/insight-engine:latest insight-engine/packaging/target/docker-resources - - travis_retry travis_wait 40 docker build -t quay.io/alfresco/insight-zeppelin:latest insight-engine/alfresco-insight-zeppelin/target/docker-resources - mkdir -p e2e-test/target/docker-resources && python e2e-test/python-generator/generator.py --alfresco=quay.io/alfresco/dev:acs-for-search --transformer=LegacyTransformers --search=quay.io/alfresco/insight-engine:latest --spellcheck --output=e2e-test/target/docker-resources - ./e2e-test/helpers/start-alfresco.sh "e2e-test/target/docker-resources" "quay.io/alfresco/insight-engine:latest" - ./e2e-test/helpers/wait-service-to-start.sh @@ -56,9 +54,7 @@ jobs: install: - travis_retry travis_wait 60 mvn --batch-mode -q clean package -DskipTests - docker login quay.io -u ${QUAY_USERNAME} -p ${QUAY_PASSWORD} - - travis_retry travis_wait 40 docker build -t quay.io/alfresco/search-services:latest search-services/packaging/target/docker-resources - travis_retry travis_wait 40 docker build -t quay.io/alfresco/insight-engine:latest insight-engine/packaging/target/docker-resources - - travis_retry travis_wait 40 docker build -t quay.io/alfresco/insight-zeppelin:latest insight-engine/alfresco-insight-zeppelin/target/docker-resources - mkdir -p e2e-test/target/docker-resources && python e2e-test/python-generator/generator.py --alfresco=quay.io/alfresco/dev:acs-for-search --transformer=LegacyTransformers --search=quay.io/alfresco/insight-engine:latest --spellcheck --output=e2e-test/target/docker-resources - ./e2e-test/helpers/start-alfresco.sh "e2e-test/target/docker-resources" "quay.io/alfresco/insight-engine:latest" - ./e2e-test/helpers/wait-service-to-start.sh @@ -69,9 +65,7 @@ jobs: install: - travis_retry travis_wait 60 mvn --batch-mode -q clean package -DskipTests - docker login quay.io -u ${QUAY_USERNAME} -p ${QUAY_PASSWORD} - - travis_retry travis_wait 40 docker build -t quay.io/alfresco/search-services:latest search-services/packaging/target/docker-resources - travis_retry travis_wait 40 docker build -t quay.io/alfresco/insight-engine:latest insight-engine/packaging/target/docker-resources - - travis_retry travis_wait 40 docker build -t quay.io/alfresco/insight-zeppelin:latest insight-engine/alfresco-insight-zeppelin/target/docker-resources - mkdir -p e2e-test/target/docker-resources && python e2e-test/python-generator/generator.py --alfresco=quay.io/alfresco/dev:acs-for-search --transformer=LegacyTransformers --search=quay.io/alfresco/insight-engine:latest --spellcheck --output=e2e-test/target/docker-resources - ./e2e-test/helpers/start-alfresco.sh "e2e-test/target/docker-resources" "quay.io/alfresco/insight-engine:latest" - ./e2e-test/helpers/wait-service-to-start.sh From 9e5d5347e48f1ad5711c88bd1a6522bb17205258 Mon Sep 17 00:00:00 2001 From: Alex Mukha Date: Fri, 10 Jul 2020 17:35:33 +0100 Subject: [PATCH 23/40] Add whitesource scans --- .travis.yml | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index e40f7f485..e33b70fc8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -86,4 +86,34 @@ jobs: - travis_retry travis_wait 40 docker build -t quay.io/alfresco/insight-engine:latest insight-engine/packaging/target/docker-resources - travis_retry travis_wait 40 docker build -t quay.io/alfresco/insight-zeppelin:latest insight-engine/alfresco-insight-zeppelin/target/docker-resources script: - - travis_wait 60 ./scripts/test-image.sh "${TRAVIS_BUILD_DIR}" "search-services/packaging/target/docker-resources insight-engine/packaging/target/docker-resources insight-engine/alfresco-insight-zeppelin/target/docker-resources" "quay.io/alfresco/search-services quay.io/alfresco/insight-engine quay.io/alfresco/insight-zeppelin" "latest" "alfresco-search-services alfresco-insight-engine zeppelin" \ No newline at end of file + - travis_wait 60 ./scripts/test-image.sh "${TRAVIS_BUILD_DIR}" "search-services/packaging/target/docker-resources insight-engine/packaging/target/docker-resources insight-engine/alfresco-insight-zeppelin/target/docker-resources" "quay.io/alfresco/search-services quay.io/alfresco/insight-engine quay.io/alfresco/insight-zeppelin" "latest" "alfresco-search-services alfresco-insight-engine zeppelin" + - name: "Whitesource scan - Search Services" + #TODO run only on release branches + #if: fork = false AND (branch = master OR branch =~ /release\/.*/) AND type != pull_request + jdk: openjdk11 + install: + - travis_retry travis_wait 60 mvn --batch-mode -q clean package -DskipTests + - mkdir -p whitesource_scan/search-services && unzip search-services/packaging/target/*.zip -d whitesource_scan/search-services + script: + - curl -LJO https://github.com/whitesource/unified-agent-distribution/releases/latest/download/wss-unified-agent.jar + - java -jar wss-unified-agent.jar -apiKey ${WHITESOURCE_API_KEY} -productName SearchAndDiscovery -projectName SearchServices -projectVersion 2.0.0-SNAPSHOT -d ./whitesource_scan/search-services + - name: "Whitesource scan - Insight Engine" + #TODO run only on release branches + #if: fork = false AND (branch = master OR branch =~ /release\/.*/) AND type != pull_request + jdk: openjdk11 + install: + - travis_retry travis_wait 60 mvn --batch-mode -q clean package -DskipTests + - mkdir -p whitesource_scan/insight-engine && unzip insight-engine/packaging/target/*.zip -d whitesource_scan/insight-engine + script: + - curl -LJO https://github.com/whitesource/unified-agent-distribution/releases/latest/download/wss-unified-agent.jar + - java -jar wss-unified-agent.jar -apiKey ${WHITESOURCE_API_KEY} -productName SearchAndDiscovery -projectName InsightEngine -projectVersion 2.0.0-SNAPSHOT -d ./whitesource_scan/insight-engine + - name: "Whitesource scan - Zeppelin" + #TODO run only on release branches + #if: fork = false AND (branch = master OR branch =~ /release\/.*/) AND type != pull_request + jdk: openjdk11 + install: + - travis_retry travis_wait 60 mvn --batch-mode -q clean package -DskipTests + - mkdir -p whitesource_scan/insight-zeppelin && unzip alfresco-insight-zeppelin/packaging/target/*.zip -d whitesource_scan/insight-zeppelin + script: + - curl -LJO https://github.com/whitesource/unified-agent-distribution/releases/latest/download/wss-unified-agent.jar + - java -jar wss-unified-agent.jar -apiKey ${WHITESOURCE_API_KEY} -productName SearchAndDiscovery -projectName InsightZeppelin -projectVersion 2.0.0-SNAPSHOT -d ./whitesource_scan/insight-zeppelin From 6266e7018ecf3f3a40d935d437964e72a0689d6f Mon Sep 17 00:00:00 2001 From: Alex Mukha Date: Fri, 10 Jul 2020 18:11:24 +0100 Subject: [PATCH 24/40] Correct whitesource cli params --- .travis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index e33b70fc8..2a317626c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -96,7 +96,7 @@ jobs: - mkdir -p whitesource_scan/search-services && unzip search-services/packaging/target/*.zip -d whitesource_scan/search-services script: - curl -LJO https://github.com/whitesource/unified-agent-distribution/releases/latest/download/wss-unified-agent.jar - - java -jar wss-unified-agent.jar -apiKey ${WHITESOURCE_API_KEY} -productName SearchAndDiscovery -projectName SearchServices -projectVersion 2.0.0-SNAPSHOT -d ./whitesource_scan/search-services + - java -jar wss-unified-agent.jar -apiKey ${WHITESOURCE_API_KEY} -product "SearchAndDiscovery" -project "SearchServices" -projectVersion "2.0.0-SNAPSHOT" -d ./whitesource_scan/search-services - name: "Whitesource scan - Insight Engine" #TODO run only on release branches #if: fork = false AND (branch = master OR branch =~ /release\/.*/) AND type != pull_request @@ -106,7 +106,7 @@ jobs: - mkdir -p whitesource_scan/insight-engine && unzip insight-engine/packaging/target/*.zip -d whitesource_scan/insight-engine script: - curl -LJO https://github.com/whitesource/unified-agent-distribution/releases/latest/download/wss-unified-agent.jar - - java -jar wss-unified-agent.jar -apiKey ${WHITESOURCE_API_KEY} -productName SearchAndDiscovery -projectName InsightEngine -projectVersion 2.0.0-SNAPSHOT -d ./whitesource_scan/insight-engine + - java -jar wss-unified-agent.jar -apiKey ${WHITESOURCE_API_KEY} -product "SearchAndDiscovery" -project "InsightEngine" -projectVersion "2.0.0-SNAPSHOT" -d ./whitesource_scan/insight-engine - name: "Whitesource scan - Zeppelin" #TODO run only on release branches #if: fork = false AND (branch = master OR branch =~ /release\/.*/) AND type != pull_request @@ -116,4 +116,4 @@ jobs: - mkdir -p whitesource_scan/insight-zeppelin && unzip alfresco-insight-zeppelin/packaging/target/*.zip -d whitesource_scan/insight-zeppelin script: - curl -LJO https://github.com/whitesource/unified-agent-distribution/releases/latest/download/wss-unified-agent.jar - - java -jar wss-unified-agent.jar -apiKey ${WHITESOURCE_API_KEY} -productName SearchAndDiscovery -projectName InsightZeppelin -projectVersion 2.0.0-SNAPSHOT -d ./whitesource_scan/insight-zeppelin + - java -jar wss-unified-agent.jar -apiKey ${WHITESOURCE_API_KEY} -product "SearchAndDiscovery" -project "InsightZeppelin" -projectVersion "2.0.0-SNAPSHOT" -d ./whitesource_scan/insight-zeppelin From f47f8e85fbdc8fb180380cb65f65f59669f02e6e Mon Sep 17 00:00:00 2001 From: Alex Mukha Date: Fri, 10 Jul 2020 18:13:41 +0100 Subject: [PATCH 25/40] Correct zeppelin folder location --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 2a317626c..9dc9836b6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -113,7 +113,7 @@ jobs: jdk: openjdk11 install: - travis_retry travis_wait 60 mvn --batch-mode -q clean package -DskipTests - - mkdir -p whitesource_scan/insight-zeppelin && unzip alfresco-insight-zeppelin/packaging/target/*.zip -d whitesource_scan/insight-zeppelin + - mkdir -p whitesource_scan/insight-zeppelin && unzip insight-engine/alfresco-insight-zeppelin/packaging/target/*.zip -d whitesource_scan/insight-zeppelin script: - curl -LJO https://github.com/whitesource/unified-agent-distribution/releases/latest/download/wss-unified-agent.jar - java -jar wss-unified-agent.jar -apiKey ${WHITESOURCE_API_KEY} -product "SearchAndDiscovery" -project "InsightZeppelin" -projectVersion "2.0.0-SNAPSHOT" -d ./whitesource_scan/insight-zeppelin From c5e827afc8ffa49627dce2b5732a1e900b1d195f Mon Sep 17 00:00:00 2001 From: Alex Mukha Date: Fri, 10 Jul 2020 19:41:51 +0100 Subject: [PATCH 26/40] Add defaut config file for whitesource --- .travis.yml | 6 +- .wss-unified-agent.config | 228 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 231 insertions(+), 3 deletions(-) create mode 100644 .wss-unified-agent.config diff --git a/.travis.yml b/.travis.yml index 9dc9836b6..a0fd93370 100644 --- a/.travis.yml +++ b/.travis.yml @@ -96,7 +96,7 @@ jobs: - mkdir -p whitesource_scan/search-services && unzip search-services/packaging/target/*.zip -d whitesource_scan/search-services script: - curl -LJO https://github.com/whitesource/unified-agent-distribution/releases/latest/download/wss-unified-agent.jar - - java -jar wss-unified-agent.jar -apiKey ${WHITESOURCE_API_KEY} -product "SearchAndDiscovery" -project "SearchServices" -projectVersion "2.0.0-SNAPSHOT" -d ./whitesource_scan/search-services + - java -jar wss-unified-agent.jar -apiKey ${WHITESOURCE_API_KEY} -project "SearchServices" -projectVersion "2.0.0-SNAPSHOT" -d ./whitesource_scan/search-services - name: "Whitesource scan - Insight Engine" #TODO run only on release branches #if: fork = false AND (branch = master OR branch =~ /release\/.*/) AND type != pull_request @@ -106,7 +106,7 @@ jobs: - mkdir -p whitesource_scan/insight-engine && unzip insight-engine/packaging/target/*.zip -d whitesource_scan/insight-engine script: - curl -LJO https://github.com/whitesource/unified-agent-distribution/releases/latest/download/wss-unified-agent.jar - - java -jar wss-unified-agent.jar -apiKey ${WHITESOURCE_API_KEY} -product "SearchAndDiscovery" -project "InsightEngine" -projectVersion "2.0.0-SNAPSHOT" -d ./whitesource_scan/insight-engine + - java -jar wss-unified-agent.jar -apiKey ${WHITESOURCE_API_KEY} -project "InsightEngine" -projectVersion "2.0.0-SNAPSHOT" -d ./whitesource_scan/insight-engine - name: "Whitesource scan - Zeppelin" #TODO run only on release branches #if: fork = false AND (branch = master OR branch =~ /release\/.*/) AND type != pull_request @@ -116,4 +116,4 @@ jobs: - mkdir -p whitesource_scan/insight-zeppelin && unzip insight-engine/alfresco-insight-zeppelin/packaging/target/*.zip -d whitesource_scan/insight-zeppelin script: - curl -LJO https://github.com/whitesource/unified-agent-distribution/releases/latest/download/wss-unified-agent.jar - - java -jar wss-unified-agent.jar -apiKey ${WHITESOURCE_API_KEY} -product "SearchAndDiscovery" -project "InsightZeppelin" -projectVersion "2.0.0-SNAPSHOT" -d ./whitesource_scan/insight-zeppelin + - java -jar wss-unified-agent.jar -apiKey ${WHITESOURCE_API_KEY} -project "InsightZeppelin" -projectVersion "2.0.0-SNAPSHOT" -d ./whitesource_scan/insight-zeppelin diff --git a/.wss-unified-agent.config b/.wss-unified-agent.config new file mode 100644 index 000000000..9501519e7 --- /dev/null +++ b/.wss-unified-agent.config @@ -0,0 +1,228 @@ +#################################################################### +# WhiteSource Unified-Agent configuration file +#################################################################### +########################################## +# GENERAL SCAN MODE: Files and Package Managers +########################################## + +checkPolicies=true +forceCheckAllDependencies=true +forceUpdate=true +forceUpdate.failBuildOnPolicyViolation=true +offline=false +#ignoreSourceFiles=true +#scanComment= +#updateInventory=false +#resolveAllDependencies=false +#failErrorLevel=ALL +#requireKnownSha1=false +#generateScanReport=true +#scanReportTimeoutMinutes=10 +#excludeDependenciesFromNodes=.*commons-io.*,.*maven-model + +#projectPerFolder=true +#projectPerFolderIncludes= +#projectPerFolderExcludes= + +#wss.connectionTimeoutMinutes=60 +# Change the below URL to your WhiteSource server. +# Use the 'WhiteSource Server URL' which can be retrieved +# from your 'Profile' page on the 'Server URLs' panel. +# Then, add the '/agent' path to it. +wss.url=https://saas.whitesourcesoftware.com/agent + +#npm.resolveDependencies=false +#npm.ignoreSourceFiles=false +#npm.includeDevDependencies=true +#npm.runPreStep=true +#npm.ignoreNpmLsErrors=true +#npm.ignoreScripts=true +#npm.yarnProject=true +#npm.accessToken= +#npm.identifyByNameAndVersion=true + +#bower.resolveDependencies=false +#bower.ignoreSourceFiles=true +#bower.runPreStep=true + +#nuget.resolvePackagesConfigFiles=false +#nuget.resolveCsProjFiles=false +#nuget.resolveDependencies=false +#nuget.restoreDependencies=true +#nuget.ignoreSourceFiles=true +#nuget.runPreStep=true +#nuget.resolveNuspecFiles=false + +#python.resolveDependencies=false +#python.ignoreSourceFiles=false +#python.ignorePipInstallErrors=true +#python.installVirtualenv=true +#python.resolveHierarchyTree=false +#python.requirementsFileIncludes=requirements.txt +#python.resolveSetupPyFiles=true +#python.runPipenvPreStep=true +#python.pipenvDevDependencies=true +#python.IgnorePipenvInstallErrors=true + +#maven.ignoredScopes=test provided +maven.resolveDependencies=true +#maven.ignoreSourceFiles=true +#maven.aggregateModules=true +maven.ignorePomModules=false +#maven.runPreStep=true +#maven.ignoreMvnTreeErrors=true +#maven.environmentPath= +#maven.m2RepositoryPath= + +#gradle.ignoredScopes= +#gradle.resolveDependencies=false +#gradle.runAssembleCommand=false +#gradle.runPreStep=true +#gradle.ignoreSourceFiles=true +#gradle.aggregateModules=true +#gradle.preferredEnvironment=wrapper +#gradle.localRepositoryPath= + +#paket.resolveDependencies=false +#paket.ignoredGroups= +#paket.ignoreSourceFiles=false +#paket.runPreStep=true +#paket.exePath= + +#go.resolveDependencies=false +#go.collectDependenciesAtRuntime=true +#go.dependencyManager= +#go.ignoreSourceFiles=true +#go.glide.ignoreTestPackages=false +#go.gogradle.enableTaskAlias=true + +#ruby.resolveDependencies = false +#ruby.ignoreSourceFiles = false +#ruby.installMissingGems = true +#ruby.runBundleInstall = true +#ruby.overwriteGemFile = true + +#sbt.resolveDependencies=false +#sbt.ignoreSourceFiles=true +#sbt.aggregateModules=true +#sbt.runPreStep=true +#sbt.targetFolder= + +#php.resolveDependencies=false +#php.runPreStep=true +#php.includeDevDependencies=true + +#html.resolveDependencies=false + +#cocoapods.resolveDependencies=false +#cocoapods.runPreStep=true +#cocoapods.ignoreSourceFiles=false + +#hex.resolveDependencies=false +#hex.runPreStep=true +#hex.ignoreSourceFiles=false +#hex.aggregateModules=true + +################################## +# Organization tokens: +################################## +#apiKey= + +#userKey is required if WhiteSource administrator has enabled "Enforce user level access" option +#userKey= + +#projectName= +#projectVersion= +#projectToken= + +productName=SearchAndDiscovery +#productVersion= +#productToken= +#updateType=APPEND +#requesterEmail=user@provider.com + +######################################################################################### +# Includes/Excludes Glob patterns - PLEASE USE ONLY ONE EXCLUDE LINE AND ONE INCLUDE LINE +######################################################################################### +#includes=**/*.c **/*.cc **/*.cp **/*.cpp **/*.cxx **/*.c++ **/*.h **/*.hpp **/*.hxx + +#includes=**/*.m **/*.mm **/*.js **/*.php +includes=**/*.jar +#includes=**/*.gem **/*.rb +#includes=**/*.dll **/*.cs **/*.nupkg +#includes=**/*.tgz **/*.deb **/*.gzip **/*.rpm **/*.tar.bz2 +#includes=**/*.zip **/*.tar.gz **/*.egg **/*.whl **/*.py + +## Exclude file extensions or specific directories by adding **/*. or **/** +excludes=**/*sources.jar **/*javadoc.jar + +case.sensitive.glob=false +followSymbolicLinks=true + +################################## +# Archive properties +################################## +#archiveExtractionDepth=2 +#archiveIncludes=**/*.war **/*.ear +#archiveExcludes=**/*sources.jar + +################################## +# Proxy settings +################################## +#proxy.host= +#proxy.port= +#proxy.user= +#proxy.pass= + +################################## +# SCM settings +################################## +#scm.type= +#scm.user= +#scm.pass= +#scm.ppk= +#scm.url= +#scm.branch= +#scm.tag= +#scm.npmInstall= +#scm.npmInstallTimeoutMinutes= +#scm.repositoriesFile= + +############################################## +# SCAN MODE: Linux package manager settings +############################################## +#scanPackageManager=true + +################################## +# SCAN MODE: Docker images +################################## +#docker.scanImages=true +#docker.includes=.*.* +#docker.excludes= +#docker.pull.enable=true +#docker.pull.images=.*.* +#docker.pull.maxImages=10 +#docker.pull.tags=.*.* +#docker.pull.digest= +#docker.delete.force=true +#docker.login.sudo=false + +#docker.aws.enable=true +#docker.aws.registryIds= + +################################## +# SCAN MODE: Docker containers +################################## +#docker.scanContainers=true +#docker.containerIncludes=.*.* +#docker.containerExcludes= + +################################ +# Serverless settings +################################ +#serverless.provider= +#serverless.scanFunctions=true +#serverless.includes= +#serverless.excludes= +#serverless.region= +#serverless.maxFunctions=10 From 5788dc65614c1729543093e37e5312075a4827f5 Mon Sep 17 00:00:00 2001 From: Alex Mukha Date: Fri, 10 Jul 2020 19:42:54 +0100 Subject: [PATCH 27/40] Rename whitesource config to be picked up --- .wss-unified-agent.config => wss-unified-agent.config | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .wss-unified-agent.config => wss-unified-agent.config (100%) diff --git a/.wss-unified-agent.config b/wss-unified-agent.config similarity index 100% rename from .wss-unified-agent.config rename to wss-unified-agent.config From 9bdf0b4e567e917e81782a91a9c766eb09de7437 Mon Sep 17 00:00:00 2001 From: Alex Mukha Date: Fri, 10 Jul 2020 20:02:59 +0100 Subject: [PATCH 28/40] Change the location of zepplin zip --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index a0fd93370..fc3a6003b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -113,7 +113,7 @@ jobs: jdk: openjdk11 install: - travis_retry travis_wait 60 mvn --batch-mode -q clean package -DskipTests - - mkdir -p whitesource_scan/insight-zeppelin && unzip insight-engine/alfresco-insight-zeppelin/packaging/target/*.zip -d whitesource_scan/insight-zeppelin + - mkdir -p whitesource_scan/insight-zeppelin && unzip insight-engine/alfresco-insight-zeppelin/target/*.zip -d whitesource_scan/insight-zeppelin script: - curl -LJO https://github.com/whitesource/unified-agent-distribution/releases/latest/download/wss-unified-agent.jar - java -jar wss-unified-agent.jar -apiKey ${WHITESOURCE_API_KEY} -project "InsightZeppelin" -projectVersion "2.0.0-SNAPSHOT" -d ./whitesource_scan/insight-zeppelin From 3f0b95e965d111cfd1938dbe2f0392b3d009d606 Mon Sep 17 00:00:00 2001 From: Alex Mukha Date: Mon, 13 Jul 2020 17:28:57 +0100 Subject: [PATCH 29/40] Add deploy stage --- .travis.yml | 47 ++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 44 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index fc3a6003b..cd9acafdf 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,9 +22,16 @@ branches: stages: - name: Tests if: commit_message !~ /\[skip tests\]/ + - Deploy - Release - Publish +env: + global: + # Release version has to start with real version (6.3.0-....) for the docker image to build successfully. + - RELEASE_VERSION=2.0.0-A1 + - DEVELOPMENT_VERSION=2.0.0-SNAPSHOT + # This should not be required on community build before_install: - "cp .travis.settings.xml $HOME/.m2/settings.xml" @@ -88,7 +95,7 @@ jobs: script: - travis_wait 60 ./scripts/test-image.sh "${TRAVIS_BUILD_DIR}" "search-services/packaging/target/docker-resources insight-engine/packaging/target/docker-resources insight-engine/alfresco-insight-zeppelin/target/docker-resources" "quay.io/alfresco/search-services quay.io/alfresco/insight-engine quay.io/alfresco/insight-zeppelin" "latest" "alfresco-search-services alfresco-insight-engine zeppelin" - name: "Whitesource scan - Search Services" - #TODO run only on release branches + #TODO run only on release branches and master #if: fork = false AND (branch = master OR branch =~ /release\/.*/) AND type != pull_request jdk: openjdk11 install: @@ -98,7 +105,7 @@ jobs: - curl -LJO https://github.com/whitesource/unified-agent-distribution/releases/latest/download/wss-unified-agent.jar - java -jar wss-unified-agent.jar -apiKey ${WHITESOURCE_API_KEY} -project "SearchServices" -projectVersion "2.0.0-SNAPSHOT" -d ./whitesource_scan/search-services - name: "Whitesource scan - Insight Engine" - #TODO run only on release branches + #TODO run only on release branches and master #if: fork = false AND (branch = master OR branch =~ /release\/.*/) AND type != pull_request jdk: openjdk11 install: @@ -108,7 +115,7 @@ jobs: - curl -LJO https://github.com/whitesource/unified-agent-distribution/releases/latest/download/wss-unified-agent.jar - java -jar wss-unified-agent.jar -apiKey ${WHITESOURCE_API_KEY} -project "InsightEngine" -projectVersion "2.0.0-SNAPSHOT" -d ./whitesource_scan/insight-engine - name: "Whitesource scan - Zeppelin" - #TODO run only on release branches + #TODO run only on release branches and master #if: fork = false AND (branch = master OR branch =~ /release\/.*/) AND type != pull_request jdk: openjdk11 install: @@ -117,3 +124,37 @@ jobs: script: - curl -LJO https://github.com/whitesource/unified-agent-distribution/releases/latest/download/wss-unified-agent.jar - java -jar wss-unified-agent.jar -apiKey ${WHITESOURCE_API_KEY} -project "InsightZeppelin" -projectVersion "2.0.0-SNAPSHOT" -d ./whitesource_scan/insight-zeppelin + - stage: Deploy + name: Deploy to Nexus + #TODO run only on release branches and master + #if: fork = false AND (branch = master OR branch =~ /release\/.*/) AND type != pull_request + install: + - travis_retry travis_wait 60 mvn --batch-mode -q clean package -DskipTests + script: + - travis_wait 60 mvn --batch-mode deploy -DskipTests=true + - name: Deploy "$DEVELOPMENT_VERSION" to quay.io + #TODO run only on release branches + #if: fork = false AND ( branch =~ /release\/.*/) AND type != pull_request + install: + - travis_retry travis_wait 60 mvn --batch-mode -q clean package -DskipTests + - docker login quay.io -u ${QUAY_USERNAME} -p ${QUAY_PASSWORD} + - travis_retry travis_wait 40 docker build -t quay.io/alfresco/search-services:$DEVELOPMENT_VERSION search-services/packaging/target/docker-resources + - travis_retry travis_wait 40 docker build -t quay.io/alfresco/insight-engine:$DEVELOPMENT_VERSION insight-engine/packaging/target/docker-resources + - travis_retry travis_wait 40 docker build -t quay.io/alfresco/insight-zeppelin:$DEVELOPMENT_VERSION insight-engine/alfresco-insight-zeppelin/target/docker-resources + script: + - docker push quay.io/alfresco/search-services:$DEVELOPMENT_VERSION + - docker push quay.io/alfresco/insight-engine:$DEVELOPMENT_VERSION + - docker push quay.io/alfresco/insight-zeppelin:$DEVELOPMENT_VERSION + - name: Deploy "latest" to quay.io + #TODO run only on master + #if: fork = false AND (branch = master) AND type != pull_request + install: + - travis_retry travis_wait 60 mvn --batch-mode -q clean package -DskipTests + - docker login quay.io -u ${QUAY_USERNAME} -p ${QUAY_PASSWORD} + - travis_retry travis_wait 40 docker build -t quay.io/alfresco/search-services:latest search-services/packaging/target/docker-resources + - travis_retry travis_wait 40 docker build -t quay.io/alfresco/insight-engine:latest insight-engine/packaging/target/docker-resources + - travis_retry travis_wait 40 docker build -t quay.io/alfresco/insight-zeppelin:latest insight-engine/alfresco-insight-zeppelin/target/docker-resources + script: + - docker push quay.io/alfresco/search-services:latest + - docker push quay.io/alfresco/insight-engine:latest + - docker push quay.io/alfresco/insight-zeppelin:latest From 4d2152ad5143a24be7e9d189caeebf84f3de93c6 Mon Sep 17 00:00:00 2001 From: Alex Mukha Date: Fri, 17 Jul 2020 16:15:24 +0100 Subject: [PATCH 30/40] Change alpha version --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index cd9acafdf..9f65dd852 100644 --- a/.travis.yml +++ b/.travis.yml @@ -28,8 +28,8 @@ stages: env: global: - # Release version has to start with real version (6.3.0-....) for the docker image to build successfully. - - RELEASE_VERSION=2.0.0-A1 + # Release version has to start with real version (2.0.0-....) for the docker image to build successfully. + - RELEASE_VERSION=2.0.0-A2 - DEVELOPMENT_VERSION=2.0.0-SNAPSHOT # This should not be required on community build From 8e6f33767c70d8becc3d62a80ef3b3c1abab0e58 Mon Sep 17 00:00:00 2001 From: Alex Mukha Date: Fri, 17 Jul 2020 16:16:30 +0100 Subject: [PATCH 31/40] Rename deploy job --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 9f65dd852..939bdc369 100644 --- a/.travis.yml +++ b/.travis.yml @@ -132,7 +132,7 @@ jobs: - travis_retry travis_wait 60 mvn --batch-mode -q clean package -DskipTests script: - travis_wait 60 mvn --batch-mode deploy -DskipTests=true - - name: Deploy "$DEVELOPMENT_VERSION" to quay.io + - name: Deploy dev version to quay.io #TODO run only on release branches #if: fork = false AND ( branch =~ /release\/.*/) AND type != pull_request install: From 1d0762d49c9fe68eea68dbe33b0b1b1ba3ae360c Mon Sep 17 00:00:00 2001 From: Alex Mukha Date: Fri, 17 Jul 2020 16:32:59 +0100 Subject: [PATCH 32/40] Add retry to docker push --- .travis.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 939bdc369..b1b488104 100644 --- a/.travis.yml +++ b/.travis.yml @@ -142,9 +142,9 @@ jobs: - travis_retry travis_wait 40 docker build -t quay.io/alfresco/insight-engine:$DEVELOPMENT_VERSION insight-engine/packaging/target/docker-resources - travis_retry travis_wait 40 docker build -t quay.io/alfresco/insight-zeppelin:$DEVELOPMENT_VERSION insight-engine/alfresco-insight-zeppelin/target/docker-resources script: - - docker push quay.io/alfresco/search-services:$DEVELOPMENT_VERSION - - docker push quay.io/alfresco/insight-engine:$DEVELOPMENT_VERSION - - docker push quay.io/alfresco/insight-zeppelin:$DEVELOPMENT_VERSION + - travis_retry docker push quay.io/alfresco/search-services:$DEVELOPMENT_VERSION + - travis_retry docker push quay.io/alfresco/insight-engine:$DEVELOPMENT_VERSION + - travis_retry docker push quay.io/alfresco/insight-zeppelin:$DEVELOPMENT_VERSION - name: Deploy "latest" to quay.io #TODO run only on master #if: fork = false AND (branch = master) AND type != pull_request @@ -155,6 +155,6 @@ jobs: - travis_retry travis_wait 40 docker build -t quay.io/alfresco/insight-engine:latest insight-engine/packaging/target/docker-resources - travis_retry travis_wait 40 docker build -t quay.io/alfresco/insight-zeppelin:latest insight-engine/alfresco-insight-zeppelin/target/docker-resources script: - - docker push quay.io/alfresco/search-services:latest - - docker push quay.io/alfresco/insight-engine:latest - - docker push quay.io/alfresco/insight-zeppelin:latest + - travis_retry docker push quay.io/alfresco/search-services:latest + - travis_retry docker push quay.io/alfresco/insight-engine:latest + - travis_retry docker push quay.io/alfresco/insight-zeppelin:latest From ec12b702a002ec0989437cede78ef49e67fa6f52 Mon Sep 17 00:00:00 2001 From: Alex Mukha Date: Fri, 17 Jul 2020 16:41:56 +0100 Subject: [PATCH 33/40] Allow configuring the search log level in the Python generator. --- e2e-test/python-generator/generator.py | 3 +++ .../python-generator/templates/search/Dockerfile.template | 6 ++++++ 2 files changed, 9 insertions(+) diff --git a/e2e-test/python-generator/generator.py b/e2e-test/python-generator/generator.py index 99cf11d68..d541831c8 100644 --- a/e2e-test/python-generator/generator.py +++ b/e2e-test/python-generator/generator.py @@ -220,6 +220,8 @@ if __name__ == '__main__': parser.add_argument('-sc', '--shardCount', type=int, help='Total number of shards to create (default 2)') parser.add_argument('-sr', '--shardRange', type=int, help='Total number of nodes per shard with DB_ID_RANGE sharding (default 800)') parser.add_argument('-ct', '--disableCascadeTracking', action='store_true', help='Cascade Tracking Disabled') + parser.add_argument('-sl', '--searchLogLevel', default='WARN', help='The log level for search (default WARN)', + choices=['TRACE', 'DEBUG', 'INFO', 'WARN', 'ERROR']) parser.add_argument('-o', '--output', default='.', help='The path of the directory to output to') parser.add_argument('-mtls', '--mtls', action='store_true', help='Use mTLS between SOLR and Alfresco Repository') args = parser.parse_args() @@ -251,6 +253,7 @@ if __name__ == '__main__': 'SHARE_IMAGE': args.share, 'POSTGRES_IMAGE': args.postgres, 'SEARCH_IMAGE': args.search, + 'SEARCH_LOG_LEVEL': args.searchLogLevel, 'SEARCH_ENABLE_SPELLCHECK': str(args.spellcheck).lower(), 'DISABLE_CASCADE_TRACKING': str(args.disableCascadeTracking).lower() } diff --git a/e2e-test/python-generator/templates/search/Dockerfile.template b/e2e-test/python-generator/templates/search/Dockerfile.template index 70d97b9a7..cd2db8ae2 100644 --- a/e2e-test/python-generator/templates/search/Dockerfile.template +++ b/e2e-test/python-generator/templates/search/Dockerfile.template @@ -23,4 +23,10 @@ RUN mkdir -p /opt/alfresco-search-services/keystore \ && chown -R solr:solr /opt/alfresco-search-services/keystore USER solr +# Set the search log level if requested. +RUN if [ "${SEARCH_LOG_LEVEL}" != "" ] ; then \ + sed -i '/^bash.*/i sed -i "'"s/log4j.rootLogger=WARN, file, CONSOLE/log4j.rootLogger=${SEARCH_LOG_LEVEL}, file, CONSOLE/g"'" $${DIST_DIR}/logs/log4j.properties\n' \ + $${DIST_DIR}/solr/bin/search_config_setup.sh; \ +fi + VOLUME ["/opt/alfresco-search-services/keystore"] From 5b5fab47ad884b88ba2b6bcb9d1367d9b0c468d0 Mon Sep 17 00:00:00 2001 From: Alex Mukha Date: Mon, 20 Jul 2020 08:59:35 +0100 Subject: [PATCH 34/40] Update name of a job --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index b1b488104..38f535e8e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -125,7 +125,7 @@ jobs: - curl -LJO https://github.com/whitesource/unified-agent-distribution/releases/latest/download/wss-unified-agent.jar - java -jar wss-unified-agent.jar -apiKey ${WHITESOURCE_API_KEY} -project "InsightZeppelin" -projectVersion "2.0.0-SNAPSHOT" -d ./whitesource_scan/insight-zeppelin - stage: Deploy - name: Deploy to Nexus + name: Deploy snapshots to Nexus #TODO run only on release branches and master #if: fork = false AND (branch = master OR branch =~ /release\/.*/) AND type != pull_request install: From e4b65021031663fc3eb6bfd5bdaca6577f69c13f Mon Sep 17 00:00:00 2001 From: Alex Mukha Date: Mon, 27 Jul 2020 17:45:06 +0100 Subject: [PATCH 35/40] Add (internal) release stage --- .travis.yml | 55 ++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 54 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 38f535e8e..326ab4b0b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -29,7 +29,7 @@ stages: env: global: # Release version has to start with real version (2.0.0-....) for the docker image to build successfully. - - RELEASE_VERSION=2.0.0-A2 + - RELEASE_VERSION=2.0.0-A3 - DEVELOPMENT_VERSION=2.0.0-SNAPSHOT # This should not be required on community build @@ -158,3 +158,56 @@ jobs: - travis_retry docker push quay.io/alfresco/search-services:latest - travis_retry docker push quay.io/alfresco/insight-engine:latest - travis_retry docker push quay.io/alfresco/insight-zeppelin:latest + - name: "Release and Copy to S3 Staging Bucket" + stage: Release + # TODO release only on releasable branches +# if: fork = false AND (branch = master OR branch =~ /support\/.*/) AND type != pull_request AND commit_message =~ /\[release\]/ + if: commit_message =~ /\[release\]/ + install: + - travis_retry travis_wait 60 mvn --batch-mode -q clean package -DskipTests + script: + - travis_wait 60 mvn --batch-mode -DreleaseVersion=$RELEASE_VERSION -DdevelopmentVersion=$DEVELOPMENT_VERSION release:prepare release:perform -DskipTests "-Darguments=-DskipTests -Dadditionalparam=-Xdoclint:none" + deploy: + - provider: s3 + access_key_id: ${AWS_STAGING_ACCESS_KEY} + secret_access_key: ${AWS_STAGING_SECRET_KEY} + region: "eu-west-1" + bucket: "alfresco-artefacts-staging" + glob: "alfresco-search-services-${RELEASE_VERSION}.zip" + upload_dir: "enterprise/SearchServices/${RELEASE_VERSION}" + skip_cleanup: true + acl: private + local_dir: "search-services/packaging/target" + - provider: s3 + access_key_id: ${AWS_STAGING_ACCESS_KEY} + secret_access_key: ${AWS_STAGING_SECRET_KEY} + region: "eu-west-1" + bucket: "alfresco-artefacts-staging" + glob: "alfresco-insight-engine-distribution-${RELEASE_VERSION}.zip" + upload_dir: "enterprise/InsightEngine/${RELEASE_VERSION}" + skip_cleanup: true + acl: private + local_dir: "insight-engine/packaging/target" + - provider: s3 + access_key_id: ${AWS_STAGING_ACCESS_KEY} + secret_access_key: ${AWS_STAGING_SECRET_KEY} + region: "eu-west-1" + bucket: "alfresco-artefacts-staging" + glob: "alfresco-insight-zeppelin-${RELEASE_VERSION}.zip" + upload_dir: "enterprise/InsightZeppelin/${RELEASE_VERSION}" + skip_cleanup: true + acl: private + local_dir: "insight-engine/alfresco-insight-zeppelin/target" + - name: Deploy release version to quay.io + #TODO run only on release branches + #if: fork = false AND ( branch =~ /release\/.*/) AND type != pull_request + install: + - travis_retry travis_wait 60 mvn --batch-mode -q clean package -DskipTests + - docker login quay.io -u ${QUAY_USERNAME} -p ${QUAY_PASSWORD} + - travis_retry travis_wait 40 docker build -t quay.io/alfresco/search-services:$RELEASE_VERSION search-services/packaging/target/docker-resources + - travis_retry travis_wait 40 docker build -t quay.io/alfresco/insight-engine:$RELEASE_VERSION insight-engine/packaging/target/docker-resources + - travis_retry travis_wait 40 docker build -t quay.io/alfresco/insight-zeppelin:$RELEASE_VERSION insight-engine/alfresco-insight-zeppelin/target/docker-resources + script: + - travis_retry docker push quay.io/alfresco/search-services:$RELEASE_VERSION + - travis_retry docker push quay.io/alfresco/insight-engine:$RELEASE_VERSION + - travis_retry docker push quay.io/alfresco/insight-zeppelin:$RELEASE_VERSION \ No newline at end of file From d3bc0b4d8c24543850c078e3a7c19c20da3bd009 Mon Sep 17 00:00:00 2001 From: Alex Mukha Date: Wed, 29 Jul 2020 17:09:21 +0100 Subject: [PATCH 36/40] Add addional checks for the release stage --- .travis.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 326ab4b0b..97f36ab25 100644 --- a/.travis.yml +++ b/.travis.yml @@ -160,9 +160,9 @@ jobs: - travis_retry docker push quay.io/alfresco/insight-zeppelin:latest - name: "Release and Copy to S3 Staging Bucket" stage: Release - # TODO release only on releasable branches + # TODO release only on releasable branches and if the commit has "release" keyword # if: fork = false AND (branch = master OR branch =~ /support\/.*/) AND type != pull_request AND commit_message =~ /\[release\]/ - if: commit_message =~ /\[release\]/ +# if: commit_message =~ /\[release\]/ install: - travis_retry travis_wait 60 mvn --batch-mode -q clean package -DskipTests script: @@ -199,8 +199,9 @@ jobs: acl: private local_dir: "insight-engine/alfresco-insight-zeppelin/target" - name: Deploy release version to quay.io - #TODO run only on release branches + #TODO run only on release branches and if the commit has "release" keyword #if: fork = false AND ( branch =~ /release\/.*/) AND type != pull_request + #if: commit_message =~ /\[release\]/ install: - travis_retry travis_wait 60 mvn --batch-mode -q clean package -DskipTests - docker login quay.io -u ${QUAY_USERNAME} -p ${QUAY_PASSWORD} From 8e405dbb30c529832882d404d4cbbcd716292abd Mon Sep 17 00:00:00 2001 From: Alex Mukha Date: Thu, 30 Jul 2020 10:07:09 +0100 Subject: [PATCH 37/40] Modify release command --- .travis.yml | 6 +++++- search-services/packaging/pom.xml | 1 - 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 97f36ab25..e9c817c7c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -164,9 +164,13 @@ jobs: # if: fork = false AND (branch = master OR branch =~ /support\/.*/) AND type != pull_request AND commit_message =~ /\[release\]/ # if: commit_message =~ /\[release\]/ install: + # Use full history for release + - git checkout -B "${TRAVIS_BRANCH}" + # Add email to link commits to user + - git config user.email "${GIT_EMAIL}" - travis_retry travis_wait 60 mvn --batch-mode -q clean package -DskipTests script: - - travis_wait 60 mvn --batch-mode -DreleaseVersion=$RELEASE_VERSION -DdevelopmentVersion=$DEVELOPMENT_VERSION release:prepare release:perform -DskipTests "-Darguments=-DskipTests -Dadditionalparam=-Xdoclint:none" + - travis_wait 60 mvn --batch-mode -DreleaseVersion=$RELEASE_VERSION -DdevelopmentVersion=$DEVELOPMENT_VERSION -DscmCommentPrefix="[maven-release-plugin][skip ci]" -DskipTests "-Darguments=-DskipTests -Dadditionalparam=-Xdoclint:none" release:prepare release:perform deploy: - provider: s3 access_key_id: ${AWS_STAGING_ACCESS_KEY} diff --git a/search-services/packaging/pom.xml b/search-services/packaging/pom.xml index 8db3085d8..05c2767ad 100644 --- a/search-services/packaging/pom.xml +++ b/search-services/packaging/pom.xml @@ -8,7 +8,6 @@ UTF-8 UTF-8 ${maven.build.timestamp} - ${bamboo_repository_revision_number} org.alfresco From deef10f1eebedf94e96572f88df022c2405acc74 Mon Sep 17 00:00:00 2001 From: Alex Mukha Date: Thu, 30 Jul 2020 11:57:40 +0100 Subject: [PATCH 38/40] Modify release command --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index e9c817c7c..29fb6f269 100644 --- a/.travis.yml +++ b/.travis.yml @@ -170,7 +170,7 @@ jobs: - git config user.email "${GIT_EMAIL}" - travis_retry travis_wait 60 mvn --batch-mode -q clean package -DskipTests script: - - travis_wait 60 mvn --batch-mode -DreleaseVersion=$RELEASE_VERSION -DdevelopmentVersion=$DEVELOPMENT_VERSION -DscmCommentPrefix="[maven-release-plugin][skip ci]" -DskipTests "-Darguments=-DskipTests -Dadditionalparam=-Xdoclint:none" release:prepare release:perform + - travis_wait 60 mvn --batch-mode -DreleaseVersion="${RELEASE_VERSION}" -DdevelopmentVersion="${DEVELOPMENT_VERSION}" -Dusername="${GIT_USERNAME}" -Dpassword="${GIT_PASSWORD}" -DscmCommentPrefix="[maven-release-plugin][skip ci]" -DskipTests "-Darguments=-DskipTests -Dadditionalparam=-Xdoclint:none" release:prepare release:perform deploy: - provider: s3 access_key_id: ${AWS_STAGING_ACCESS_KEY} From 7d1a85dac4f5d393869246033fc52f65837eec03 Mon Sep 17 00:00:00 2001 From: Alex Mukha Date: Fri, 31 Jul 2020 12:04:21 +0100 Subject: [PATCH 39/40] Change scm tags to https --- pom.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index 9aaa80001..a218dd5f5 100644 --- a/pom.xml +++ b/pom.xml @@ -21,9 +21,9 @@ - scm:git:ssh://git@github.com/Alfresco/InsightEngine.git - scm:git:ssh://git@github.com/Alfresco/InsightEngine.git - scm:git:ssh://git@github.com/Alfresco/InsightEngine.git + scm:git:https://github.com/Alfresco/InsightEngine.git + scm:git:https://github.com/Alfresco//InsightEngine.git + https://github.com/Alfresco/InsightEngine HEAD From 09ba8918e19edc249fbda083b060c849f9fcc871 Mon Sep 17 00:00:00 2001 From: Alex Mukha Date: Fri, 31 Jul 2020 13:51:37 +0100 Subject: [PATCH 40/40] Fix a typo in the scm url --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index a218dd5f5..d4b1a9402 100644 --- a/pom.xml +++ b/pom.xml @@ -22,7 +22,7 @@ scm:git:https://github.com/Alfresco/InsightEngine.git - scm:git:https://github.com/Alfresco//InsightEngine.git + scm:git:https://github.com/Alfresco/InsightEngine.git https://github.com/Alfresco/InsightEngine HEAD