From 9c38c273167fcae55e58937328a83876496a1b5c Mon Sep 17 00:00:00 2001 From: Alex Mukha Date: Fri, 22 Jan 2021 16:48:36 +0000 Subject: [PATCH] SEARCH-2318: Add Travis configuration to Insight Engine (#873) The changes include the following: * The python scripts for e2e environment creation were moved to this code base * Mirror to community * Docker image check * Unit, integration and e2e tests * "Snapshots": "latest" docker image from master in quay.io, docker images for "release" branches to quay.io, "master" and * "release" branches snapshots to nexus * Releases: nexus, s3 staging bucket, quay.io and dockerhub * Fix for https://issues.alfresco.com/jira/browse/SEARCH-2389 The build is using branches only. --- e2e-test/helpers/start-alfresco.sh | 20 ++ e2e-test/helpers/wait-service-to-start.sh | 32 ++ e2e-test/pom.xml | 35 +- e2e-test/python-generator/README.md | 46 +++ e2e-test/python-generator/generator.py | 325 ++++++++++++++++++ .../keystores/alfresco/keystore | Bin 0 -> 327 bytes .../keystores/alfresco/ssl.keystore | Bin 0 -> 3288 bytes .../keystores/alfresco/ssl.truststore | Bin 0 -> 1854 bytes .../keystores/solr/ssl-repo-client.keystore | Bin 0 -> 3298 bytes .../keystores/solr/ssl-repo-client.truststore | Bin 0 -> 2883 bytes .../templates/alfresco/Dockerfile.template | 49 +++ .../templates/docker-compose.yml.template | 52 +++ .../templates/search-node.yml.template | 17 + .../templates/search/Dockerfile.template | 32 ++ .../functional/AbstractE2EFunctionalTest.java | 4 +- .../src/test/resources/default.properties | 3 +- pom.xml | 23 +- search-services/alfresco-search/pom.xml | 37 -- .../alfresco-solrclient-lib/pom.xml | 11 - search-services/packaging/pom.xml | 7 +- search-services/pom.xml | 48 +++ 21 files changed, 668 insertions(+), 73 deletions(-) create mode 100755 e2e-test/helpers/start-alfresco.sh create mode 100755 e2e-test/helpers/wait-service-to-start.sh create mode 100644 e2e-test/python-generator/README.md create mode 100644 e2e-test/python-generator/generator.py create mode 100644 e2e-test/python-generator/keystores/alfresco/keystore create mode 100644 e2e-test/python-generator/keystores/alfresco/ssl.keystore create mode 100644 e2e-test/python-generator/keystores/alfresco/ssl.truststore create mode 100644 e2e-test/python-generator/keystores/solr/ssl-repo-client.keystore create mode 100644 e2e-test/python-generator/keystores/solr/ssl-repo-client.truststore create mode 100755 e2e-test/python-generator/templates/alfresco/Dockerfile.template create mode 100644 e2e-test/python-generator/templates/docker-compose.yml.template create mode 100644 e2e-test/python-generator/templates/search-node.yml.template create mode 100644 e2e-test/python-generator/templates/search/Dockerfile.template diff --git a/e2e-test/helpers/start-alfresco.sh b/e2e-test/helpers/start-alfresco.sh new file mode 100755 index 000000000..7ad2c75ac --- /dev/null +++ b/e2e-test/helpers/start-alfresco.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +set -eux + +# Start Alfresco and Solr. + +# The location for the docker-compose files. +DOCKER_RESOURCE_FOLDER=$1 +# The search docker image. +SEARCH_IMAGE=$2 + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +export DOCKER_CLIENT_TIMEOUT=120 +export COMPOSE_HTTP_TIMEOUT=120 + +# Build the images and call docker-compose. +cd "$DOCKER_RESOURCE_FOLDER" +docker-compose up -d --build --force-recreate + +$SCRIPT_DIR/wait-service-to-start.sh diff --git a/e2e-test/helpers/wait-service-to-start.sh b/e2e-test/helpers/wait-service-to-start.sh new file mode 100755 index 000000000..2f599ba94 --- /dev/null +++ b/e2e-test/helpers/wait-service-to-start.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +set -e # exit if commands fails +set -x # trace what gets exe + +WAIT_INTERVAL=1 +COUNTER=0 +TIMEOUT=2000 +t0=`date +%s` + +declare -a endpoints=("${1:-http://localhost:8081/alfresco/}" "${1:-http://localhost:8083/solr/}") + +for endpoint in "${endpoints[@]}" +do + + echo "Waiting for Service to start using endpoint: ${endpoint}" + + until [[ "$(curl --output /dev/null -w ''%{http_code}'' --silent --head --fail ${endpoint})" == 200 ]] || [ "$COUNTER" -eq "$TIMEOUT" ]; do + printf '.' + sleep $WAIT_INTERVAL + COUNTER=$(($COUNTER+$WAIT_INTERVAL)) + done + + if (("$COUNTER" < "$TIMEOUT")) ; then + t1=`date +%s` + delta=$((($t1 - $t0)/60)) + echo "Service ${endpoint} Started in $delta minutes" + else + echo "Service ${endpoint} could not start in time." + echo "Waited $COUNTER seconds" + exit 1 + fi +done \ No newline at end of file diff --git a/e2e-test/pom.xml b/e2e-test/pom.xml index d13b617c3..650aeed3a 100644 --- a/e2e-test/pom.xml +++ b/e2e-test/pom.xml @@ -5,7 +5,7 @@ alfresco-search-and-insight-parent 2.0.2-SNAPSHOT - search-analytics-e2e-test + org.alfresco search-analytics-e2e-test Search Analytics E2E Tests Test Project to test Search Service and Analytics Features on a complete setup of Alfresco, Share @@ -128,4 +128,37 @@ 12-ea+10 + + + + alfresco-enterprise-releases + https://artifacts.alfresco.com/nexus/content/repositories/enterprise-releases + + true + + + false + + + + alfresco-enterprise-snapshots + https://artifacts.alfresco.com/nexus/content/repositories/enterprise-snapshots + + false + + + true + + + + alfresco-hotfix + https://artifacts.alfresco.com/nexus/content/groups/hotfix + + true + + + false + + + diff --git a/e2e-test/python-generator/README.md b/e2e-test/python-generator/README.md new file mode 100644 index 000000000..977e7dbb8 --- /dev/null +++ b/e2e-test/python-generator/README.md @@ -0,0 +1,46 @@ +# About +A python script to generator docker-compose files suitable for use with the automated tests in +the [Search and Insight E2E tests](https://git.alfresco.com/search_discovery/insightengine/tree/master/e2e-test). + +# Installation +The script uses Python 3, which can be installed from [python.org](https://www.python.org/downloads/) or using +a package manager. You will also need the [yaml library](https://pypi.org/project/PyYAML/), which can be installed +using Pip if it is not already present: +``` +pip3 install pyyaml +``` + +# Using the script +The script provides some help with the `-h` option: + +``` +python3 BuildScripts/generator/generator.py -h +usage: generator.py [-h] [-a ALFRESCO]... +... +``` + +For use with ACS 6.2.x the legacy transformers can be included: +``` +python3 BuildScripts/generator/generator.py --alfresco=alfresco/alfresco-content-repository:6.2.0 --transformer=AIOTransformers +``` + +For ACS 6.0.x and ACS 6.1.x no external transformer is needed: +``` +python3 BuildScripts/generator/generator.py --alfresco=alfresco/alfresco-content-repository:6.1.0 +``` + +For ACS 6.0.0.x and earlier ActiveMQ can be excluded: +``` +python3 BuildScripts/generator/generator.py --alfresco=alfresco/alfresco-content-repository:6.0.0.3 --excludeAMQ +``` + +For ACS 5.2.x the legacy LibreOffice transformer can be used: +``` +python3 BuildScripts/generator/generator.py --alfresco=quay.io/alfresco/alfresco-content-repository-52:5.2.5 --postgres=postgres:9.4 --excludeAMQ --transformer=LibreOffice +``` + +# Starting the containers +To start the containers you also need to build the images - for example: +``` +docker-compose up --build --force-recreate +``` diff --git a/e2e-test/python-generator/generator.py b/e2e-test/python-generator/generator.py new file mode 100644 index 000000000..c06fbb71b --- /dev/null +++ b/e2e-test/python-generator/generator.py @@ -0,0 +1,325 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +import argparse +from string import Template +import yaml +import os +from distutils.dir_util import copy_tree + +LIBRE_OFFICE = 'LibreOffice' +AIO_TRANSFORMERS = 'AIOTransformers' + +AMQ_OPTS = '-Dmessaging.broker.url="failover:(nio://activemq:61616)?timeout=3000&jms.useCompression=true"' +TRANSFORM_OPTS = ('-DlocalTransform.core-aio.url=http://transform-core-aio:8090/ ' + '-Dalfresco-pdf-renderer.url=http://transform-core-aio:8090/ ' + '-Djodconverter.url=http://transform-core-aio:8090/ ' + '-Dimg.url=http://transform-core-aio:8090/ ' + '-Dtika.url=http://transform-core-aio:8090/ ' + '-Dtransform.misc.url=http://transform-core-aio:8090/') +SHARE_TRANSFORM_OPTS = ('-DlocalTransform.pdfrenderer.url=http://alfresco-pdf-renderer:8090/ ' + '-Dalfresco-pdf-renderer.url=http://alfresco-pdf-renderer:8090/ ' + '-DlocalTransform.imagemagick.url=http://imagemagick:8090/ -Dimg.url=http://imagemagick:8090/') +SHARDING_OPTS = '-Dsolr.useDynamicShardRegistration=true' +JAVA_OPTS = ('-Ddb.driver=org.postgresql.Driver -Ddb.username=alfresco -Ddb.password=alfresco ' + '-Ddb.url=jdbc:postgresql://postgres:5432/alfresco -Dsolr.port=8983 ' + '-Dindex.subsystem.name=solr6 ' + '-Dalfresco.restApi.basicAuthScheme=true ' + # longer timeouts for CI + '-Dsolr.http.socket.timeout=30000 ' + '-Dsolr.http.connection.timeout=3000 ') +MTLS_OPTS = ('-Dsolr.port.ssl=8983 -Dsolr.secureComms=https ') +HTTP_OPTS = ('-Dsolr.secureComms=none') +JAVA_TOOL_OPTIONS = ('-Dencryption.keystore.type=JCEKS ' + '-Dencryption.cipherAlgorithm=DESede/CBC/PKCS5Padding ' + '-Dencryption.keyAlgorithm=DESede ' + '-Dencryption.keystore.location=/usr/local/tomcat/shared/classes/alfresco/extension/keystore/keystore ' + '-Dmetadata-keystore.password=mp6yc0UD9e ' + '-Dmetadata-keystore.aliases=metadata ' + '-Dmetadata-keystore.metadata.password=oKIWzVdEdA ' + '-Dmetadata-keystore.metadata.algorithm=DESede') +MTLS_JAVA_TOOL_OPTIONS = ('-Dencryption.keystore.type=pkcs12 -Dencryption.cipherAlgorithm=AES/CBC/PKCS5Padding ' + '-Dencryption.keyAlgorithm=DESede ' + '-Dssl-truststore.password=kT9X6oe68t -Dssl-keystore.password=kT9X6oe68t ' + '-Dssl-keystore.aliases=ssl-alfresco-ca,ssl-repo ' + '-Dssl-keystore.ssl-alfresco-ca.password=kT9X6oe68t ' + '-Dssl-keystore.ssl-repo.password=kT9X6oe68t ' + '-Dssl-truststore.aliases=alfresco-ca,ssl-repo-client ' + '-Dssl-truststore.alfresco-ca.password=kT9X6oe68t ' + '-Dssl-truststore.ssl-repo-client.password=kT9X6oe68t') + +def getJavaOpts(includeAMQ, includeTransform, includeShare, solrHost, solrBaseUrl, sharding, mtls): + + solrHost = '-Dsolr.host=' + solrHost + shardingOpts = (SHARDING_OPTS if sharding != None else '') + amqOpts = (AMQ_OPTS if includeAMQ else '') + transformOpts = (TRANSFORM_OPTS if includeTransform else '') + solrBaseUrlOpts = '-Dsolr.baseUrl=' + solrBaseUrl + shareTransformOpts = (SHARE_TRANSFORM_OPTS if includeShare and includeTransform else '') + mtlsOpts = (MTLS_OPTS if mtls else HTTP_OPTS) + + return ' '.join([JAVA_OPTS, amqOpts, transformOpts, shareTransformOpts, solrHost, solrBaseUrlOpts, shardingOpts, mtlsOpts]) + +def getJavaToolOptions(mtls): + + mtlsJavaToolOptions = (MTLS_JAVA_TOOL_OPTIONS if mtls else '') + + return ' '.join([JAVA_TOOL_OPTIONS, mtlsJavaToolOptions]) + +def deleteServices(dcYaml, *services): + for service in services: + if service in dcYaml['services'].keys(): + del(dcYaml['services'][service]) + +def getExtraEnvironmentVars(serviceName, replicationType): + """Return a dict of environment variables to add to the search container declaration in docker-compose.yml.""" + extraEnvironmentVars = {} + if replicationType == 'master': + extraEnvironmentVars['REPLICATION_TYPE'] = 'master' + elif replicationType == 'slave': + extraEnvironmentVars.update({'REPLICATION_TYPE': 'slave', + 'REPLICATION_MASTER_HOST': serviceName.replace('slave', 'master'), + 'REPLICATION_MASTER_PORT': '8983', + 'REPLICATION_POLL_INTERVAL': '00:00:10'}) + return extraEnvironmentVars + +def getSolrcoreConfig(sharding, shardId, shardCount, shardRange): + """Returns a list of properties to add to the end of the solrcore.properties file.""" + solrcoreConfig = [] + if sharding != None: + solrcoreConfig.append('solr.port.ssl=8983') + solrcoreConfig.append('shard.instance={}'.format(shardId)) + solrcoreConfig.append('alfresco.port=8080') + solrcoreConfig.append('alfresco.port.ssl=8443') + solrcoreConfig.append('alfresco.baseUrl=/alfresco') + if sharding not in ['DB_ID_RANGE', 'EXPLICIT_ID_FALLBACK_LRIS']: + solrcoreConfig.append('shard.count={}'.format(shardCount)) + if sharding == 'DB_ID_RANGE': + # The first shards each contain 800 nodes by default and the last continues the range to id 100000. + nodesPerShard = shardRange + rangeStart = shardId * nodesPerShard + rangeEnd = (rangeStart + nodesPerShard - 1 if shardId < shardCount - 1 else 100000) + solrcoreConfig.append('shard.range={}-{}'.format(rangeStart, rangeEnd)) + if sharding == 'DATE': + solrcoreConfig.append('shard.key=cm:created') + solrcoreConfig.append('shard.date.grouping={}'.format(12 // shardCount)) + if sharding in ['PROPERTY', 'EXPLICIT_ID', 'EXPLICIT_ID_FALLBACK_LRIS']: + solrcoreConfig.append('shard.key=shard:shardId') + print("SolrConfig for Shard: ", shardId, " : ", solrcoreConfig) + return solrcoreConfig + +def getSolrcoreReplacements(sharding, mtls): + """Returns a dict of replacements to make in the solrcore.properties file.""" + solrcoreReplacements = {} + if sharding != None: + solrcoreReplacements['shard.method=DB_ID'] = 'shard.method={}'.format(sharding) + if mtls: + solrcoreReplacements['alfresco.secureComms=none'] = 'alfresco.secureComms=https' + solrcoreReplacements['alfresco.encryption.ssl.keystore.location=.*'] = 'alfresco.encryption.ssl.keystore.location=\\\\\\/opt\\\\\\/alfresco-search-services\\\\\\/keystore\\\\\\/ssl-repo-client.keystore' + solrcoreReplacements['alfresco.encryption.ssl.keystore.type=.*'] = 'alfresco.encryption.ssl.keystore.type=JCEKS' + solrcoreReplacements['alfresco.encryption.ssl.truststore.location=.*'] = 'alfresco.encryption.ssl.truststore.location=\\\\\\/opt\\\\\\/alfresco-search-services\\\\\\/keystore\\\\\\/ssl-repo-client.truststore' + solrcoreReplacements['alfresco.encryption.ssl.truststore.type=.*'] = 'alfresco.encryption.ssl.truststore.type=JCEKS' + else: + solrcoreReplacements['alfresco.secureComms=https'] = 'alfresco.secureComms=none' + return solrcoreReplacements + +def addAlfrescoMtlsConfig(alfrescoArgsNode): + """Add a list of environment values in Docker Compose Alfresco Service for mTLS.""" + alfrescoArgsNode['TRUSTSTORE_TYPE'] = 'JCEKS' + alfrescoArgsNode['TRUSTSTORE_PASS'] = 'kT9X6oe68t' + alfrescoArgsNode['KEYSTORE_TYPE'] = 'JCEKS' + alfrescoArgsNode['KEYSTORE_PASS'] = 'kT9X6oe68t' + alfrescoArgsNode['SOLR_COMMS'] = 'https' + +def addAlfrescoVolumes(alfrescoNode): + """Add route to keystores folder""" + alfrescoNode['volumes'] = ['./keystores/alfresco:/usr/local/tomcat/alf_data/keystore'] + +def addSolrMtlsConfig(solrEnvNode): + """Add a list of environment values in Docker Compose SOLR Service for mTLS.""" + solrEnvNode['SOLR_SSL_TRUST_STORE'] = '/opt/alfresco-search-services/keystore/ssl-repo-client.truststore' + solrEnvNode['SOLR_SSL_TRUST_STORE_TYPE'] = 'JCEKS' + solrEnvNode['SOLR_SSL_KEY_STORE'] = '/opt/alfresco-search-services/keystore/ssl-repo-client.keystore' + solrEnvNode['SOLR_SSL_KEY_STORE_TYPE'] = 'JCEKS' + solrEnvNode['SOLR_SSL_NEED_CLIENT_AUTH'] = 'true' + +def addSolrOpts(solrEnvNode): + """Add a list of values to add in Docker Compose SOLR_OPTS property for mTLS.""" + solrOptions = ' '.join(['-Dsolr.ssl.checkPeerName=false', + '-Dsolr.allow.unsafe.resourceloading=true']) + solrEnvNode['SOLR_OPTS'] = solrOptions + +def addSolrJavaToolOptions(solrEnvNode): + """Add a list of values to add in Docker Compose JAVA_TOOL_OPTIONS property for mTLS.""" + solrOptions = ' '.join(['-Dsolr.jetty.truststore.password=kT9X6oe68t ', + '-Dsolr.jetty.keystore.password=kT9X6oe68t ', + '-Dssl-keystore.password=kT9X6oe68t', + '-Dssl-keystore.aliases=ssl-alfresco-ca,ssl-repo-client', + '-Dssl-keystore.ssl-alfresco-ca.password=kT9X6oe68t', + '-Dssl-keystore.ssl-repo-client.password=kT9X6oe68t', + '-Dssl-truststore.password=kT9X6oe68t', + '-Dssl-truststore.aliases=ssl-alfresco-ca,ssl-repo,ssl-repo-client', + '-Dssl-truststore.ssl-alfresco-ca.password=kT9X6oe68t', + '-Dssl-truststore.ssl-repo.password=kT9X6oe68t', + '-Dssl-truststore.ssl-repo-client.password=kT9X6oe68t']) + solrEnvNode['JAVA_TOOL_OPTIONS'] = solrOptions + +def addSolrVolumes(solrNode): + """Add route to keystores folder""" + solrNode['volumes'] = ['./keystores/solr:/opt/alfresco-search-services/keystore'] + +def makeSearchNode(outputDirectory, nodeName, externalPort, params, mtls, extraEnvironmentVars={}, solrcoreConfig=[], solrcoreReplacements={}): + # Create a dictionary for the template replacement. + allParams = dict(params) + allParams['SOLR_HOST'] = nodeName + allParams['ALFRESCO_PORT'] = 8443 if mtls else 8080 + allParams['EXTERNAL_PORT'] = externalPort + # Properties to add to solrcore.properties. + allParams['SOLRCORE_PROPERTIES'] = '\\n'.join(solrcoreConfig) + # Replacements to make in solrcore.properties (in an "abc/xyz" format suitable for sed). + allParams['SOLRCORE_REPLACEMENTS'] = ' '.join(map(lambda pair: '"{}/{}"'.format(*pair), solrcoreReplacements.items())) + + # mTLS settings + allParams['ALFRESCO_SECURE_COMMS'] = ('https' if mtls else 'none') + allParams['TRUSTSTORE_TYPE'] = 'JCEKS' + allParams['KEYSTORE_TYPE'] = 'JCEKS' + + # Create a Dockerfile with any extra configuration in. + with open(scriptDir + '/templates/search/Dockerfile.template') as f: + dockerfileTemplate = f.read() + dockerfileString = Template(dockerfileTemplate).substitute(allParams) + if not os.path.isdir('{}/{}'.format(outputDirectory, nodeName)): + os.mkdir('{}/{}'.format(outputDirectory, nodeName)) + with open('{}/{}/Dockerfile'.format(outputDirectory, nodeName), 'w') as f: + dockerfileTemplate = f.write(dockerfileString) + + # Load the search node template. + with open(scriptDir + '/templates/search-node.yml.template') as f: + searchNodeTemplate = f.read() + searchNodeString = Template(searchNodeTemplate).substitute(allParams) + # Read the result as yaml. + searchNodeYaml = yaml.safe_load(searchNodeString) + # Add any extra environment variables. + searchNodeYaml['environment'].update(extraEnvironmentVars) + + # Add mTLS configuration if required + if mtls: + addSolrMtlsConfig(searchNodeYaml['environment']) + addSolrOpts(searchNodeYaml['environment']) + addSolrJavaToolOptions(searchNodeYaml['environment']) + addSolrVolumes(searchNodeYaml) + + return searchNodeYaml + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Generate a docker-compose file for ACS.') + parser.add_argument('-a', '--alfresco', default='quay.io/alfresco/dev:acs-for-search', help='The Alfresco image') + parser.add_argument('-s', '--search', default='quay.io/alfresco/search-services:latest', help='The Search image') + parser.add_argument('-e', '--share', help='The Share image (or omit for no UI)') + parser.add_argument('-p', '--postgres', default='postgres:10.1', help='The Postgres image') + parser.add_argument('-q', '--excludeAMQ', action='store_true', help='Exclude ActiveMQ (i.e. pre-ACS 6.1)') + parser.add_argument('-t', '--transformer', choices=[LIBRE_OFFICE, AIO_TRANSFORMERS], help='Use external transformers. ' + + '"{}" for legacy LibreOffice (i.e. ACS 5.2.x). '.format(LIBRE_OFFICE) + + '"{}" for the all-in-one transformers for use with ACS 6.2.x and later.'.format(AIO_TRANSFORMERS)) + parser.add_argument('-c', '--spellcheck', action='store_true', help='Spellcheck Enabled') + parser.add_argument('-ms', '--masterslave', action='store_true', help='Master Slave Enabled') + parser.add_argument('-sh', '--sharding', help='Sharding method (or omit for no sharding). Note that sharding is not supported on SearchServices 1.2.x or earlier.', + choices=['DB_ID', 'DB_ID_RANGE', 'ACL_ID', 'MOD_ACL_ID', 'DATE', 'PROPERTY', 'LRIS', 'EXPLICIT_ID', 'EXPLICIT_ID_FALLBACK_LRIS']) + parser.add_argument('-sc', '--shardCount', type=int, help='Total number of shards to create (default 2)') + parser.add_argument('-sr', '--shardRange', type=int, help='Total number of nodes per shard with DB_ID_RANGE sharding (default 800)') + parser.add_argument('-ct', '--disableCascadeTracking', action='store_true', help='Cascade Tracking Disabled') + parser.add_argument('-sl', '--searchLogLevel', default='WARN', help='The log level for search (default WARN)', + choices=['TRACE', 'DEBUG', 'INFO', 'WARN', 'ERROR']) + parser.add_argument('-o', '--output', default='.', help='The path of the directory to output to') + parser.add_argument('-mtls', '--mtls', action='store_true', help='Use mTLS between SOLR and Alfresco Repository') + args = parser.parse_args() + + # If sharding is selected then the default number of shards is two. + if args.sharding != None and args.shardCount == None: + print('Using default shardCount of 2') + args.shardCount = 2 + elif args.sharding == None and args.shardCount != None: + print('ERROR: shardCount={} specified without sharding method'.format(args.shardCount)) + exit(1) + print('Arguments:', args) + + # If sharding is selected then the default number of nodes per shard is 800. + if args.sharding == "DB_ID_RANGE" and (args.shardRange == None or args.shardRange < 1): + print('Using default shardRange of 800') + args.shardRange = 800 + elif args.sharding != 'DB_ID_RANGE' and args.shardRange != None: + print('ERROR: shardRange={} is only supported for DB_ID_RANGE sharding.') + exit(1) + print('Arguments:', args) + + # Load the template and perform basic token substitution. + scriptDir = os.path.dirname(os.path.realpath(__file__)) + with open(scriptDir + '/templates/docker-compose.yml.template') as f: + template = f.read() + params = { + 'ALFRESCO_IMAGE': args.alfresco, + 'SHARE_IMAGE': args.share, + 'POSTGRES_IMAGE': args.postgres, + 'SEARCH_IMAGE': args.search, + 'SEARCH_LOG_LEVEL': args.searchLogLevel, + 'SEARCH_ENABLE_SPELLCHECK': str(args.spellcheck).lower(), + 'DISABLE_CASCADE_TRACKING': str(args.disableCascadeTracking).lower() + } + dcString = Template(template).substitute(params) + + # Edit the resulting yaml. + dcYaml = yaml.safe_load(dcString) + + # Insert the search node(s). + shardList = range(args.shardCount) if args.sharding != None else [0] + replicationTypes = ['master', 'slave'] if args.masterslave else ['standalone'] + for shardId in shardList: + for replicationType in replicationTypes: + serviceName = 'search_{}_{}'.format(shardId, replicationType) + # Workaround for ShardInfoTest.getShardInfoWithAdminAuthority. + if shardId == 0 and replicationType == 'standalone': + serviceName = 'search' + externalPort = 8083 + 100 * shardId + (1 if replicationType == 'slave' else 0) + dcYaml['services'][serviceName] = makeSearchNode(args.output, serviceName, externalPort, params, args.mtls, + extraEnvironmentVars=getExtraEnvironmentVars(serviceName, replicationType), + solrcoreConfig=getSolrcoreConfig(args.sharding, shardId, args.shardCount, args.shardRange), + solrcoreReplacements=getSolrcoreReplacements(args.sharding, args.mtls)) + + # Point Alfresco at whichever Solr node came last in the list. + solrHost = serviceName + solrBaseUrl = '/solr-slave' if args.masterslave else '/solr' + + javaOpts = getJavaOpts(not args.excludeAMQ, args.transformer == AIO_TRANSFORMERS, args.share != None, solrHost, solrBaseUrl, args.sharding, args.mtls) + dcYaml['services']['alfresco']['environment']['JAVA_OPTS'] = javaOpts + javaToolOpts = getJavaToolOptions(args.mtls) + dcYaml['services']['alfresco']['environment']['JAVA_TOOL_OPTIONS'] = javaToolOpts + if args.mtls: + addAlfrescoMtlsConfig(dcYaml['services']['alfresco']['build']['args']) + addAlfrescoVolumes(dcYaml['services']['alfresco']) + + if not args.share: + deleteServices(dcYaml, 'share', 'alfresco-pdf-renderer', 'imagemagick') + if args.excludeAMQ: + deleteServices(dcYaml, 'activemq') + if args.transformer != AIO_TRANSFORMERS: + deleteServices(dcYaml, 'transform-core-aio') + del(dcYaml['volumes']['shared-file-store-volume']) + if args.transformer == LIBRE_OFFICE: + dcYaml['services']['libreoffice'] = {'image': 'xcgd/libreoffice'} + + # Output the yaml. + with open(args.output + '/docker-compose.yml', 'w') as f: + f.write(yaml.safe_dump(dcYaml)) + + # Create an Alfresco Dockerfile with any extra configuration in. + with open(scriptDir + '/templates/alfresco/Dockerfile.template') as f: + dockerfileTemplate = f.read() + dockerfileString = Template(dockerfileTemplate).substitute(params) + if not os.path.isdir('{}/{}'.format(args.output, 'alfresco')): + os.mkdir('{}/{}'.format(args.output, 'alfresco')) + with open('{}/{}/Dockerfile'.format(args.output, 'alfresco'), 'w') as f: + dockerfileTemplate = f.write(dockerfileString) + + # Copy the keystores (when using mTLS) + if args.mtls: + copy_tree(scriptDir + '/keystores', args.output + '/keystores') diff --git a/e2e-test/python-generator/keystores/alfresco/keystore b/e2e-test/python-generator/keystores/alfresco/keystore new file mode 100644 index 0000000000000000000000000000000000000000..439bc1f861272ed97542fa9242f42ffd04e16988 GIT binary patch literal 327 zcmXqLVsvI=WHxC0%f_kI=F#?@myw-uLF0Rt#y19yFAW->p$Ok#X}oICc*&sgJR3Ju zH4hgf>w+qS3S92M!f1Tik zH9Q_)YD(SXXZ-Z$HgqwNhr5we#8601fFYM5m7#8E2#ZiiW^Ss2 zp{a$bg_(hwfuV_^p@E%&q5&rxt2Q4qlN2ihi%7tUi}~TpGSzd_qXZSXHa(ri<~xB! f#N|lal#HE$%&VpJjhV!L%WjK|cV%K`Y+C>T8B%IO literal 0 HcmV?d00001 diff --git a/e2e-test/python-generator/keystores/alfresco/ssl.keystore b/e2e-test/python-generator/keystores/alfresco/ssl.keystore new file mode 100644 index 0000000000000000000000000000000000000000..c0cff56f8b1b130dd2ce09fdfbf9175e17aedae3 GIT binary patch literal 3288 zcmeH}do+~$8pr3Icdll}Z5ZW#t7NbAPJ>Kp8>vtdhQu!0#K>)CG(&Ec88U~YLTqtd z3Po-0lx&v>gR&LbF4cAy*>0OfWvYWSlU=9PY3)DGT4(jo{^PgS`#kUSuJwC=&+q&B zKCP{-tuPo2{&C6ic!3)jfj(R&&x^CcivfcHN8KL$VhuwcFfxItlF~^i8a^gj4!Iy4 zfWwszYnREr+s#`T#k{T5Fr-}A>fq)C5l~n|p^b2X001~742mM-AS?=WL7<3m>+K%` z3?A;XhfIVNq;Jr8nY|1@ZzkK9$?zs?K^oGe44!29=U%kw9G~D&2A8RA7wm0BmWO1e z1vos~g&n}=gtEzM&}wN0#4B2b@Pauk?T=qtSwfmBgiR1+44IKPQH(c{-Bk$2(vb3J z`1Lm^$ocfl03-|)B*Gv;JX|1vO@>%jj_+@-npB|{k;_4ub;DVMvgm8r2L-R9=JtlD zFN$+)>qRM7en_7kq5R?Hq>$f~B%fY-`Gnb+m#%lXR=s#4!H}eQFzyE)0p&iESNf{* z;en9$g%?DmPSr92Td3?#C3RQaD<~O1T-Mk?khPO-Q=HyY7jqtaEe9Y8;2;14@;o7r z_f64|MUW6mSj<3u!=3)AoAkRDnJK5~Vk6f+NK4WH4L!svh{XSL0bW`_L?9s$0G8i( zP(x1ggg{8HlJL_5kX#y2^5^`R-^g7#~ZnGaWmGazA5TSa<49a3&lxf0l+X)NpP7# zoE0rJdrL%cMIwO9kU9$c4GKU2hHxSP$ssT?V$W`Uk^j$d5G2qL(QrQ8Db`y*apH)- z#cb(dZ%|b~+LFgg%E#9xx2cl(sojZuP>g)6>q1LRw@d0w{_ah$(dnDzI{S>bF0nc} z>A1?+B8xGH{rC5u#Vz-8Ot+N-3UnWvE#!Q+G+cyE?LXGI&ZI;QxJ~^;j(>9Sj=Jpd z<^1H;P5yPK2QYEC4q)_o=5)tZvctup$0A*&{IH~U>JjCz|28-(k6J(-p#nrWN@92{3Pd zbMlnnrGc?W5%C6w_FymjEO})7`L=y;f7W*g4$l=OHpOjrWzhpvj+4%3)}Lu{5tQHY zCxR9&O>Qm+e4fN~nfExI5MQrTs3+dF+O^>HG^jsyWNB;E*{*R#BZTK9Rkk1tslw&sG+Gr7XuB^B@zvVU-%n6Kew-5M5*r4H9`>2!zCH&3o6{62r%gX_n)|^!zF;K( zgJ!aQ-b-=Xgwew5%_7G9W3Ywz< zE))&0-&?E%uW$$^@l>p*&;}5Sw*!C_uP7Qs`A>>~SWD917sO=K>2})IOg59t2xfX~ z)0y03Os=*SlN;>kn_Inmg($CV}_JdW*%Y{xhCr&LCp&C1!a)n_Lrf5 zNpLD(Ao%K6EBecWigb$j=t0O_h$xNMAn=e33L{}V5{U)@NJ*MhKoB5YG&Ydq#R&A} z@Pa?nj?}Jp&cJ$qwD<5q9UZOSg}$`kYUB0%s)W!mb!H8&`9cX5Wp^N~r~+;CPsig5 zLLz2v=O*{4OGndPTPpgD@rX+&eu&&WME({DoMWbfPpna&(y#YsO+^t6Vr1b4p_j< z#K^=XlBBSW|F>81_op1ccFAfPPq7Jz3^L$l8#k_1$?-y_Ry6a3t>BaBQgjp4L|L5ggAsS^Vc5UbWr5j%+Z995|m(Pdq zywF?kgUu5-SIuV>Yh+?(WMEvJU=Rn66j?qNF%}U{j{Ap>Y`ynpL+~E^*^|41f9QqZ zHIN5JfUGi$gn?KCb_F~j1;Q+>2F#3%|G_aR%MWC+umF=%8*%^u6Ad%a=M6`{uVCic ztKo3v*0X|}&t`q@VlPh?%ROqn|L8Q`kjvTO!XGNKFL7ATJ-H%owt+^u+|QnRk+s=gH3oi(>rkHzOTL+F3T z-fwrzR{3#v-YS}a?TJx)*Zwl-0A~sQ;^G|LqSOLlcF4(0%_{+BiNeSeCr-h$ z1T!!@Fh2%jA4ry95@1C0%fP2E)C^z*%mBuq3}6P%07g*;m9R{SC5Muc8MO>FAem7S zJu?OY1F|@?B)_Oq!5JK!2KbWx<@U`^uX&p;On|L>xprUR3I_I72@!97|9z4dSiJqsJ~PQm)^&v+4V32CMQHnduDxD3 z*?DDa^ZLh^-}%^0YB;0p8+pX9({EL3#_Sn9GRse|stD*7Pg#3s3;)luO{{;Lu9`$b zl716okwFt<9=HetCSN8424I=Rnb79J*!IJTkr9-9*&Gav5kkByYGNw>1*v(#!9EJ^ zsd=eIi6yBi3c;yGWvN99&Z$KunQ58HKry5oWmb2Bzdr2P`RmK8%!CE2HYW%t#~L*5 zgJh}3Z3c~-v1h7uzN3ox1h!t~-pU zzZ^E(cR}Mt*tBAXllGStXK@AXf870bfg;<^|IL+bITwyFKAaf%%=HCFdsd3^dBc;& z|NMOg*UVg@Hp9?x5A%wPN|VJl?!Eo>JiEVj^y4-`lX-d3K4E+xle=G>%USV9kR|)i cHGauea=oD&UI|J%%dUKLv}lRl!@X5e07h+FmjD0& literal 0 HcmV?d00001 diff --git a/e2e-test/python-generator/keystores/solr/ssl-repo-client.keystore b/e2e-test/python-generator/keystores/solr/ssl-repo-client.keystore new file mode 100644 index 0000000000000000000000000000000000000000..f1474dd04f3cbe8fed7fe9e6f68daf658e75e181 GIT binary patch literal 3298 zcmeH}do+~$8pr3I8^g>nrbfhZOWPs#J4P;vjoBp`F2t(2z;g{ba zH~-@^0}v1xHyHwQ&qKLf$V{kd{d1>|h8byta^m+ev9HFj{E8K+K(&`FpINjDmD^m+ zx48cxy)-jx;WyG8%??-GmL{2XyD(aRil$7DRJgzTB1xSf<(81i#vonii*7I1cR7dt zv@(fDC^c+hP-Qz^4G6!~HJ8*p_o{7e!C)!a9;sQ&f5rY2Rgn)6xKJ1X0g609kL{+& zW8nn&4wUHA`z?2#yu4xG_)SiFqW!AY@ljAg&;WTA_%4tT^Z5d_umBH7fG_~q+U}sd zgy0Dd7hI+Ij}Jg_X<&jsk7->ggQ1($Onx4k3?7`!ew`>1P9p@g8aB6H(sbzaKaO9B z@P8y~aHXTnBL`HYU*8)ZAEDxIQnS`lFgzO z?hjCXF4f?7B~A-YnSnTN0m;9Q=vG*ZpC4jntIDpG52v{{RlNZd1d;$C5^Oe8lf?)Y z$bsq02;>NIAujE0V*)Ld3Up8i${|r2NB|C~L-7D80T*3&pVRLQ)Xjs!U>FlP4;o_( zfWzTGsMwJ;$;H-a%l&nx2XfsElbmrEBNA#{u?2%!=0B$PR#98D-5tb{+`=H^bI!&E znL}FAeRra&v!gWX59LKu>aYiTtIhJ%L!1&a%!B*vUf|FAMi$Mb+O9A!+B9_Kcww(K z@UoeSmNsMGCLiQwWV)z?cTAeK4)*fHY)?n+nm^8j{%{iYMtLNriGN|CBuM?x!oyhU`_!Ep?NLTU1{kB<~rP z^ot;m`dN*6dncJ05Rc$D?Q1vkI@}ya<|7WaJlv-}@$=Hm8De`J;@f9phljN9N` z@BM?1VeNSfvwm(%Ut3HPD_0ARJWH|}(&@DmS6zqpnI^AQP*wZF>c;yLzY6O~U%zqt z3IFX>_0fbVf08r>lA5aUb=%473Es)8=BjQD-qvzA^gCgjR);*BmmR{y-BYP8T?gyz zM=j1@R(V>C44Bzxs6bCNYD}J8rBCIh8I8ZfE68rtP~7hB7+FpnP|Hl&TB45bT$3Jb zt9S6;gSE^A(ERBE&nA_aIvmbD1%n+wAdy02_RmS$=``CvdpSCfrWH0{5UVDM1zIHiPRB4n^ZgwfFq%0p6W|pFNIUsB6iAKN*({WPjLUZ1fi(b-l zu0{K`l9R_8?|2ig?j|>wQ66rS{XM{80(iqF1dH+x;80266Xkt*8vZw~wFS5)3308v zjcaXJF!H^He8x}y2kTG=)!tc$(x=v8Cy+hcmlMQ_R3vYUAN(^=_no_CHiO|k%BVCX z4fHmq_#s_v4R%*XC+##!I(zDAR{QVknTrk8y$=Sf__k#bkqS@eS*y2;3X(;4U-ETj zA`#a^UV+NF#?BhnucHS-(#hpZ|YjwMOc zt43~1tcBe~Z1VbbT;7422e3dM5V&o)0@y-9h`?b{p7BS3_*F9i5W*H|0+Rld!oc?u z_Sgh70`2W7isp^*ZoJk#2Kfyo`LU9>*8^z zZ+YTr?jV={&H%WzAoteiE#NBmY%!Pn#cvqz^J9x}rug7UhyoBVj8)-iPz)(5Ks*8= z0t3Ju!Xyrk0mXSJW)O|a^a)~fK9P{ny!PMIsMLxL8TL6q79ZbQMqPjG+mvx8*_w zuJUr(-Yg=q5mxq-ayp@=X?VFu#K!Q%L_AI>FYpA#3HyqcG~LZC+rYv7H-3?j`D9f#yiwCM`2_QILf6({|3{#rV#)D literal 0 HcmV?d00001 diff --git a/e2e-test/python-generator/keystores/solr/ssl-repo-client.truststore b/e2e-test/python-generator/keystores/solr/ssl-repo-client.truststore new file mode 100644 index 0000000000000000000000000000000000000000..c09e6891a0e5e89969690e53b91828140b6e4e16 GIT binary patch literal 2883 zcmX?i?%X*B1_mY|W(L#z#l<Q3<2haEeAeR-9cuwd2Z1mWaZgT{TpV3bv6 zY20SexVb@uljHuOBU|si*$}+Pe)i<9;2(P7cNaFcENN^U@D$c0lBBSW|F>81_op1c zcFAfPPq7Jz3<4)oS$;;w|DZr(U0@&#@{cMDj{z4Ohc+7{D=RxQqk#xWP>_Y!fV+t^ zCqFqcCnLYO1UXRvvk)^dHJBdIUZdAscss+&nfuZIubk^{WbU21_3y4bjHkaGHrsbW z<3-rCVuq9UmlbDm1?_*_{dIvN+s^;Zm25c|jxau)82HTf1xI^Uit%~Flg9u2eFfLd zT%k6@&~Oj)ii=8<#WwD}{q;P%zjgHEHbIkldC@*$d>@m$U!2QX@kfv)`_DCgNknmw z2+fqb$%&BS;AkqmIAD5h(8L5R5|{!Y#R2w!8QcYgA+Ufj0TmF&-~z%hYQWPM-eTf) zspGDfVJCOL7Pnk!xPzHX{n6Y9eC(?@Pb~df|H-#h=I@XBo`()kUN(KkhbPAGl7j@7 z9GfUGW6Ppii|5HoDV6eve!S||5)+E)m|o1w7WIDdman_cM3i3q{!Exvarb{-&K06j zmSWd-?q9m`Wzx2zM|k;s_|6Nx^*-12k1V+Twzc5T-R6lWt*aZ)1n&7Cx5vbg zVe07vZlO}MuI8$~6xy9Nw^fhD=QTsHP^(5z%15difhoU;x>cjHfg&)Dfz`AW z?y3=QV!zwGb=6l&=8Y(r!_AZ!qSNZe9#k*#h zOIxhF9VhO&ZrX0oACDMR_^%rm?N+l+F3eiwBYe=RT&``gbH?W@ehe*#msKonm^p97 zzVBH-961gJ&uVONi1tzWP~Ojc?(ChbGK^-&zJ+|rb1FT2nT~a$ulmf^jBSeE(^5-! z7{5QmDO6;tIa(*eo0XtC(e*vUoi%=OX>5g0_qwq!40G0By<~THd4){sKJJrKHd?az zL{zNY$?h>Hutu;=nB$YHQB?iH!WrSGciuIOT`~7jz_idbjTQg?c}?a|JJ0B$b?oc| z1|D|P(uP;T{dNCpci7rlIJ`CV^vs)aM6kI$H#_g%fxITZ3+I#1@n~LQS^4{?#e~B$ UK6_e{>&hArKW8|-kn?aG0D2FY!vFvP literal 0 HcmV?d00001 diff --git a/e2e-test/python-generator/templates/alfresco/Dockerfile.template b/e2e-test/python-generator/templates/alfresco/Dockerfile.template new file mode 100755 index 000000000..3438f963e --- /dev/null +++ b/e2e-test/python-generator/templates/alfresco/Dockerfile.template @@ -0,0 +1,49 @@ +FROM ${ALFRESCO_IMAGE} + +ENV TOMCAT_DIR=/usr/local/tomcat +ENV ALF_DATA_DIR=$${TOMCAT_DIR}/alf_data + +# COMMS +ARG SOLR_COMMS +ENV SOLR_COMMS $$SOLR_COMMS + +# SSL +ARG TRUSTSTORE_TYPE +ARG TRUSTSTORE_PASS +ARG KEYSTORE_TYPE +ARG KEYSTORE_PASS + +ENV TRUSTSTORE_TYPE=$$TRUSTSTORE_TYPE \ + TRUSTSTORE_PASS=$$TRUSTSTORE_PASS \ + KEYSTORE_TYPE=$$KEYSTORE_TYPE \ + KEYSTORE_PASS=$$KEYSTORE_PASS + +USER root + +# Default value in 'repository.properties' is 'dir.keystore=classpath:alfresco/keystore' +RUN if [ "$$SOLR_COMMS" == "https" ] ; then \ + echo -e "\n\ + dir.keystore=$${ALF_DATA_DIR}/keystore\n\ + alfresco.encryption.ssl.keystore.type=$${TRUSTSTORE_TYPE}\n\ + alfresco.encryption.ssl.truststore.type=$${KEYSTORE_TYPE}\n\ + " >> $${TOMCAT_DIR}/shared/classes/alfresco-global.properties; \ + fi + +# Enable SSL by adding the proper Connector to server.xml +RUN if [ "$$SOLR_COMMS" == "https" ] ; then \ + sed -i "s/\ +[[:space:]]\+<\/Engine>/\n\ + <\/Engine>\n\ + \n\ + <\/Connector>/g" $${TOMCAT_DIR}/conf/server.xml; \ + fi + +# Expose keystore folder +# Useless for 'none'/'http' communications with SOLR +VOLUME ["$$ALF_DATA_DIR/keystore"] diff --git a/e2e-test/python-generator/templates/docker-compose.yml.template b/e2e-test/python-generator/templates/docker-compose.yml.template new file mode 100644 index 000000000..a5fd74c85 --- /dev/null +++ b/e2e-test/python-generator/templates/docker-compose.yml.template @@ -0,0 +1,52 @@ +version: '3' +services: + alfresco: + build: + context: ./alfresco + args: + SOLR_COMMS: none + environment: + CATALINA_OPTS : "-agentlib:jdwp=transport=dt_socket,address=*:8000,server=y,suspend=n" + JAVA_OPTS : "This will be populated by the generator script" + JAVA_TOOL_OPTIONS : "This will be populated by the generator script" + ports: + - "7203:7203" #JMX connect via service:jmx:rmi:///jndi/rmi://localhost:7203/jmxrmi + - "8000:8000" #Java debugging + - "8081:8080" #Browser port for Alfresco + share: + image: ${SHARE_IMAGE} + environment: + - REPO_HOST=alfresco + - REPO_PORT=8080 + ports: + - 8082:8080 #Browser port for Share + postgres: + image: ${POSTGRES_IMAGE} + environment: + - POSTGRES_PASSWORD=alfresco + - POSTGRES_USER=alfresco + - POSTGRES_DB=alfresco + ports: + - 5432:5432 + activemq: + image: alfresco/alfresco-activemq:5.15.6 + ports: + - 8161:8161 # Web Console + - 5672:5672 # AMQP + - 61616:61616 # OpenWire + - 61613:61613 # STOMP + transform-core-aio: + image: alfresco/alfresco-transform-core-aio:2.3.5 + environment: + JAVA_OPTS: " -Xms256m -Xmx512m" + ACTIVEMQ_URL: "nio://activemq:61616" + ACTIVEMQ_USER: "admin" + ACTIVEMQ_PASSWORD: "admin" + FILE_STORE_URL: "http://shared-file-store:8099/alfresco/api/-default-/private/sfs/versions/1/file" + ports: + - 8090:8090 +volumes: + shared-file-store-volume: + driver_opts: + type: tmpfs + device: tmpfs diff --git a/e2e-test/python-generator/templates/search-node.yml.template b/e2e-test/python-generator/templates/search-node.yml.template new file mode 100644 index 000000000..502be27f7 --- /dev/null +++ b/e2e-test/python-generator/templates/search-node.yml.template @@ -0,0 +1,17 @@ +build: + context: ./${SOLR_HOST} +environment: + #Solr needs to know how to register itself with Alfresco + SOLR_ALFRESCO_HOST: "alfresco" + SOLR_ALFRESCO_PORT: "${ALFRESCO_PORT}" + #Alfresco needs to know how to call solr + SOLR_SOLR_HOST: "${SOLR_HOST}" + SOLR_SOLR_PORT: "8983" + #Create the default alfresco and archive cores + SOLR_CREATE_ALFRESCO_DEFAULTS: "alfresco,archive" + #Enable Spellcheck by setting to true + ENABLE_SPELLCHECK: "${SEARCH_ENABLE_SPELLCHECK}" + #Disable Cascade Tracking + DISABLE_CASCADE_TRACKING: "${DISABLE_CASCADE_TRACKING}" +ports: + - ${EXTERNAL_PORT}:8983 #Browser port diff --git a/e2e-test/python-generator/templates/search/Dockerfile.template b/e2e-test/python-generator/templates/search/Dockerfile.template new file mode 100644 index 000000000..cd2db8ae2 --- /dev/null +++ b/e2e-test/python-generator/templates/search/Dockerfile.template @@ -0,0 +1,32 @@ +FROM ${SEARCH_IMAGE} + +# Create search_config_setup.sh if it does not exist (e.g. on SearchServices 1.2.x or earlier). +USER root +RUN touch $${DIST_DIR}/solr/bin/search_config_setup.sh \ + && chown solr:solr $${DIST_DIR}/solr/bin/search_config_setup.sh +USER solr + +RUN replacementPairs=(${SOLRCORE_REPLACEMENTS}); \ + for replacementPair in $${replacementPairs[@]}; \ + do \ + sed -i '/^bash.*/i sed -i "'"s/$$replacementPair/g"'" $${DIST_DIR}/solrhome/templates/rerank/conf/solrcore.properties\n' \ + $${DIST_DIR}/solr/bin/search_config_setup.sh; \ + done; \ + if [[ "${SOLRCORE_PROPERTIES}" != "" ]]; \ + then \ + sed -i '/^bash.*/i echo "\n${SOLRCORE_PROPERTIES}" >> $${DIST_DIR}/solrhome/templates/rerank/conf/solrcore.properties\n' \ + $${DIST_DIR}/solr/bin/search_config_setup.sh; \ + fi + +USER root +RUN mkdir -p /opt/alfresco-search-services/keystore \ + && chown -R solr:solr /opt/alfresco-search-services/keystore +USER solr + +# Set the search log level if requested. +RUN if [ "${SEARCH_LOG_LEVEL}" != "" ] ; then \ + sed -i '/^bash.*/i sed -i "'"s/log4j.rootLogger=WARN, file, CONSOLE/log4j.rootLogger=${SEARCH_LOG_LEVEL}, file, CONSOLE/g"'" $${DIST_DIR}/logs/log4j.properties\n' \ + $${DIST_DIR}/solr/bin/search_config_setup.sh; \ +fi + +VOLUME ["/opt/alfresco-search-services/keystore"] diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/AbstractE2EFunctionalTest.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/AbstractE2EFunctionalTest.java index 6f377f23a..81dd4430b 100644 --- a/e2e-test/src/test/java/org/alfresco/test/search/functional/AbstractE2EFunctionalTest.java +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/AbstractE2EFunctionalTest.java @@ -83,7 +83,7 @@ import com.fasterxml.jackson.core.JsonProcessingException; public abstract class AbstractE2EFunctionalTest extends AbstractTestNGSpringContextTests { /** The number of retries that a query will be tried before giving up. */ - protected static final int SEARCH_MAX_ATTEMPTS = 6; + protected static final int SEARCH_MAX_ATTEMPTS = 60; private static final Logger LOGGER = LogFactory.getLogger(); @@ -318,7 +318,7 @@ public abstract class AbstractE2EFunctionalTest extends AbstractTestNGSpringCont } else { - throw new RuntimeException("API returned status code:" + restClient.getStatusCode() + " Expected: " + expectedStatusCode); + throw new RuntimeException("API returned status code:" + restClient.getStatusCode() + " Expected: " + expectedStatusCode + "; Response body: " + response); } } diff --git a/e2e-test/src/test/resources/default.properties b/e2e-test/src/test/resources/default.properties index 72d142995..822a78906 100644 --- a/e2e-test/src/test/resources/default.properties +++ b/e2e-test/src/test/resources/default.properties @@ -21,7 +21,8 @@ solr.server=localhost solr.port=8083 #Solr Indexing Time -solrWaitTimeInSeconds=20 +# Use 1s and 60 attempts, see AbstractE2EFunctionalTest.SEARCH_MAX_ATTEMPTS +solrWaitTimeInSeconds=1 # credentials admin.user=admin diff --git a/pom.xml b/pom.xml index 918c52500..686d987c2 100644 --- a/pom.xml +++ b/pom.xml @@ -12,22 +12,22 @@ Alfresco Search And Insight Parent - alfresco-internal + alfresco-enterprise-releases https://artifacts.alfresco.com/nexus/content/repositories/enterprise-releases/ - alfresco-internal-snapshots + alfresco-enterprise-snapshots https://artifacts.alfresco.com/nexus/content/repositories/enterprise-snapshots/ - scm:git:ssh://git@github.com/Alfresco/InsightEngine.git - scm:git:ssh://git@github.com/Alfresco/InsightEngine.git - scm:git:ssh://git@github.com/Alfresco/InsightEngine.git + scm:git:https://github.com/Alfresco/InsightEngine.git + scm:git:https://github.com/Alfresco/InsightEngine.git + https://github.com/Alfresco/InsightEngine HEAD - 11 + 11 6.6.5 ${solr.base.version}-patched.2 @@ -46,17 +46,6 @@ - - org.apache.maven.plugins - maven-compiler-plugin - 3.8.1 - - ${java.version} - ${java.version} - true - true - - org.apache.maven.plugins maven-failsafe-plugin diff --git a/search-services/alfresco-search/pom.xml b/search-services/alfresco-search/pom.xml index 33be2c76a..25e1066d5 100644 --- a/search-services/alfresco-search/pom.xml +++ b/search-services/alfresco-search/pom.xml @@ -212,43 +212,6 @@ - - - alfresco-public - https://artifacts.alfresco.com/nexus/content/groups/public - - true - - - false - - - - alfresco-public-snapshots - https://artifacts.alfresco.com/nexus/content/groups/public-snapshots - - false - - - true - - - - central - Central Repository - https://repo.maven.apache.org/maven2 - default - - false - - - - maven-restlet - Public online Restlet repository - http://maven.restlet.talend.com - - - alfresco-solr diff --git a/search-services/alfresco-solrclient-lib/pom.xml b/search-services/alfresco-solrclient-lib/pom.xml index 00b6c975a..c876de216 100644 --- a/search-services/alfresco-solrclient-lib/pom.xml +++ b/search-services/alfresco-solrclient-lib/pom.xml @@ -10,17 +10,6 @@ 2.0.2-SNAPSHOT - - - alfresco-internal - https://artifacts.alfresco.com/nexus/content/repositories/releases - - - alfresco-internal-snapshots - https://artifacts.alfresco.com/nexus/content/repositories/snapshots - - - 8.319 2.11.3 diff --git a/search-services/packaging/pom.xml b/search-services/packaging/pom.xml index c36270cd3..261b273c8 100644 --- a/search-services/packaging/pom.xml +++ b/search-services/packaging/pom.xml @@ -8,7 +8,6 @@ UTF-8 UTF-8 ${maven.build.timestamp} - ${bamboo_repository_revision_number} org.alfresco @@ -82,7 +81,7 @@ unpack-solr-config - validate + prepare-package unpack @@ -99,7 +98,7 @@ unpack-solr-libs - validate + prepare-package unpack @@ -119,7 +118,7 @@ copy - validate + prepare-package copy diff --git a/search-services/pom.xml b/search-services/pom.xml index 2c277d1a3..a69ca506b 100644 --- a/search-services/pom.xml +++ b/search-services/pom.xml @@ -23,4 +23,52 @@ alfresco-search packaging + + + + alfresco-public-releases + https://artifacts.alfresco.com/nexus/content/repositories/releases + + + alfresco-public-snapshots + https://artifacts.alfresco.com/nexus/content/repositories/snapshots + + + + + + alfresco-public + https://artifacts.alfresco.com/nexus/content/groups/public + + true + + + false + + + + alfresco-public-snapshots + https://artifacts.alfresco.com/nexus/content/groups/public-snapshots + + false + + + true + + + + central + Central Repository + https://repo.maven.apache.org/maven2 + default + + false + + + + maven-restlet + Public online Restlet repository + http://maven.restlet.talend.com + +