diff --git a/e2e-test/helpers/start-alfresco.sh b/e2e-test/helpers/start-alfresco.sh new file mode 100755 index 000000000..7ad2c75ac --- /dev/null +++ b/e2e-test/helpers/start-alfresco.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +set -eux + +# Start Alfresco and Solr. + +# The location for the docker-compose files. +DOCKER_RESOURCE_FOLDER=$1 +# The search docker image. +SEARCH_IMAGE=$2 + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +export DOCKER_CLIENT_TIMEOUT=120 +export COMPOSE_HTTP_TIMEOUT=120 + +# Build the images and call docker-compose. +cd "$DOCKER_RESOURCE_FOLDER" +docker-compose up -d --build --force-recreate + +$SCRIPT_DIR/wait-service-to-start.sh diff --git a/e2e-test/helpers/wait-service-to-start.sh b/e2e-test/helpers/wait-service-to-start.sh new file mode 100755 index 000000000..2f599ba94 --- /dev/null +++ b/e2e-test/helpers/wait-service-to-start.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +set -e # exit if commands fails +set -x # trace what gets exe + +WAIT_INTERVAL=1 +COUNTER=0 +TIMEOUT=2000 +t0=`date +%s` + +declare -a endpoints=("${1:-http://localhost:8081/alfresco/}" "${1:-http://localhost:8083/solr/}") + +for endpoint in "${endpoints[@]}" +do + + echo "Waiting for Service to start using endpoint: ${endpoint}" + + until [[ "$(curl --output /dev/null -w ''%{http_code}'' --silent --head --fail ${endpoint})" == 200 ]] || [ "$COUNTER" -eq "$TIMEOUT" ]; do + printf '.' + sleep $WAIT_INTERVAL + COUNTER=$(($COUNTER+$WAIT_INTERVAL)) + done + + if (("$COUNTER" < "$TIMEOUT")) ; then + t1=`date +%s` + delta=$((($t1 - $t0)/60)) + echo "Service ${endpoint} Started in $delta minutes" + else + echo "Service ${endpoint} could not start in time." + echo "Waited $COUNTER seconds" + exit 1 + fi +done \ No newline at end of file diff --git a/e2e-test/pom.xml b/e2e-test/pom.xml index d13b617c3..650aeed3a 100644 --- a/e2e-test/pom.xml +++ b/e2e-test/pom.xml @@ -5,7 +5,7 @@ alfresco-search-and-insight-parent 2.0.2-SNAPSHOT - search-analytics-e2e-test + org.alfresco search-analytics-e2e-test Search Analytics E2E Tests Test Project to test Search Service and Analytics Features on a complete setup of Alfresco, Share @@ -128,4 +128,37 @@ 12-ea+10 + + + + alfresco-enterprise-releases + https://artifacts.alfresco.com/nexus/content/repositories/enterprise-releases + + true + + + false + + + + alfresco-enterprise-snapshots + https://artifacts.alfresco.com/nexus/content/repositories/enterprise-snapshots + + false + + + true + + + + alfresco-hotfix + https://artifacts.alfresco.com/nexus/content/groups/hotfix + + true + + + false + + + diff --git a/e2e-test/python-generator/README.md b/e2e-test/python-generator/README.md new file mode 100644 index 000000000..977e7dbb8 --- /dev/null +++ b/e2e-test/python-generator/README.md @@ -0,0 +1,46 @@ +# About +A python script to generator docker-compose files suitable for use with the automated tests in +the [Search and Insight E2E tests](https://git.alfresco.com/search_discovery/insightengine/tree/master/e2e-test). + +# Installation +The script uses Python 3, which can be installed from [python.org](https://www.python.org/downloads/) or using +a package manager. You will also need the [yaml library](https://pypi.org/project/PyYAML/), which can be installed +using Pip if it is not already present: +``` +pip3 install pyyaml +``` + +# Using the script +The script provides some help with the `-h` option: + +``` +python3 BuildScripts/generator/generator.py -h +usage: generator.py [-h] [-a ALFRESCO]... +... +``` + +For use with ACS 6.2.x the legacy transformers can be included: +``` +python3 BuildScripts/generator/generator.py --alfresco=alfresco/alfresco-content-repository:6.2.0 --transformer=AIOTransformers +``` + +For ACS 6.0.x and ACS 6.1.x no external transformer is needed: +``` +python3 BuildScripts/generator/generator.py --alfresco=alfresco/alfresco-content-repository:6.1.0 +``` + +For ACS 6.0.0.x and earlier ActiveMQ can be excluded: +``` +python3 BuildScripts/generator/generator.py --alfresco=alfresco/alfresco-content-repository:6.0.0.3 --excludeAMQ +``` + +For ACS 5.2.x the legacy LibreOffice transformer can be used: +``` +python3 BuildScripts/generator/generator.py --alfresco=quay.io/alfresco/alfresco-content-repository-52:5.2.5 --postgres=postgres:9.4 --excludeAMQ --transformer=LibreOffice +``` + +# Starting the containers +To start the containers you also need to build the images - for example: +``` +docker-compose up --build --force-recreate +``` diff --git a/e2e-test/python-generator/generator.py b/e2e-test/python-generator/generator.py new file mode 100644 index 000000000..c06fbb71b --- /dev/null +++ b/e2e-test/python-generator/generator.py @@ -0,0 +1,325 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +import argparse +from string import Template +import yaml +import os +from distutils.dir_util import copy_tree + +LIBRE_OFFICE = 'LibreOffice' +AIO_TRANSFORMERS = 'AIOTransformers' + +AMQ_OPTS = '-Dmessaging.broker.url="failover:(nio://activemq:61616)?timeout=3000&jms.useCompression=true"' +TRANSFORM_OPTS = ('-DlocalTransform.core-aio.url=http://transform-core-aio:8090/ ' + '-Dalfresco-pdf-renderer.url=http://transform-core-aio:8090/ ' + '-Djodconverter.url=http://transform-core-aio:8090/ ' + '-Dimg.url=http://transform-core-aio:8090/ ' + '-Dtika.url=http://transform-core-aio:8090/ ' + '-Dtransform.misc.url=http://transform-core-aio:8090/') +SHARE_TRANSFORM_OPTS = ('-DlocalTransform.pdfrenderer.url=http://alfresco-pdf-renderer:8090/ ' + '-Dalfresco-pdf-renderer.url=http://alfresco-pdf-renderer:8090/ ' + '-DlocalTransform.imagemagick.url=http://imagemagick:8090/ -Dimg.url=http://imagemagick:8090/') +SHARDING_OPTS = '-Dsolr.useDynamicShardRegistration=true' +JAVA_OPTS = ('-Ddb.driver=org.postgresql.Driver -Ddb.username=alfresco -Ddb.password=alfresco ' + '-Ddb.url=jdbc:postgresql://postgres:5432/alfresco -Dsolr.port=8983 ' + '-Dindex.subsystem.name=solr6 ' + '-Dalfresco.restApi.basicAuthScheme=true ' + # longer timeouts for CI + '-Dsolr.http.socket.timeout=30000 ' + '-Dsolr.http.connection.timeout=3000 ') +MTLS_OPTS = ('-Dsolr.port.ssl=8983 -Dsolr.secureComms=https ') +HTTP_OPTS = ('-Dsolr.secureComms=none') +JAVA_TOOL_OPTIONS = ('-Dencryption.keystore.type=JCEKS ' + '-Dencryption.cipherAlgorithm=DESede/CBC/PKCS5Padding ' + '-Dencryption.keyAlgorithm=DESede ' + '-Dencryption.keystore.location=/usr/local/tomcat/shared/classes/alfresco/extension/keystore/keystore ' + '-Dmetadata-keystore.password=mp6yc0UD9e ' + '-Dmetadata-keystore.aliases=metadata ' + '-Dmetadata-keystore.metadata.password=oKIWzVdEdA ' + '-Dmetadata-keystore.metadata.algorithm=DESede') +MTLS_JAVA_TOOL_OPTIONS = ('-Dencryption.keystore.type=pkcs12 -Dencryption.cipherAlgorithm=AES/CBC/PKCS5Padding ' + '-Dencryption.keyAlgorithm=DESede ' + '-Dssl-truststore.password=kT9X6oe68t -Dssl-keystore.password=kT9X6oe68t ' + '-Dssl-keystore.aliases=ssl-alfresco-ca,ssl-repo ' + '-Dssl-keystore.ssl-alfresco-ca.password=kT9X6oe68t ' + '-Dssl-keystore.ssl-repo.password=kT9X6oe68t ' + '-Dssl-truststore.aliases=alfresco-ca,ssl-repo-client ' + '-Dssl-truststore.alfresco-ca.password=kT9X6oe68t ' + '-Dssl-truststore.ssl-repo-client.password=kT9X6oe68t') + +def getJavaOpts(includeAMQ, includeTransform, includeShare, solrHost, solrBaseUrl, sharding, mtls): + + solrHost = '-Dsolr.host=' + solrHost + shardingOpts = (SHARDING_OPTS if sharding != None else '') + amqOpts = (AMQ_OPTS if includeAMQ else '') + transformOpts = (TRANSFORM_OPTS if includeTransform else '') + solrBaseUrlOpts = '-Dsolr.baseUrl=' + solrBaseUrl + shareTransformOpts = (SHARE_TRANSFORM_OPTS if includeShare and includeTransform else '') + mtlsOpts = (MTLS_OPTS if mtls else HTTP_OPTS) + + return ' '.join([JAVA_OPTS, amqOpts, transformOpts, shareTransformOpts, solrHost, solrBaseUrlOpts, shardingOpts, mtlsOpts]) + +def getJavaToolOptions(mtls): + + mtlsJavaToolOptions = (MTLS_JAVA_TOOL_OPTIONS if mtls else '') + + return ' '.join([JAVA_TOOL_OPTIONS, mtlsJavaToolOptions]) + +def deleteServices(dcYaml, *services): + for service in services: + if service in dcYaml['services'].keys(): + del(dcYaml['services'][service]) + +def getExtraEnvironmentVars(serviceName, replicationType): + """Return a dict of environment variables to add to the search container declaration in docker-compose.yml.""" + extraEnvironmentVars = {} + if replicationType == 'master': + extraEnvironmentVars['REPLICATION_TYPE'] = 'master' + elif replicationType == 'slave': + extraEnvironmentVars.update({'REPLICATION_TYPE': 'slave', + 'REPLICATION_MASTER_HOST': serviceName.replace('slave', 'master'), + 'REPLICATION_MASTER_PORT': '8983', + 'REPLICATION_POLL_INTERVAL': '00:00:10'}) + return extraEnvironmentVars + +def getSolrcoreConfig(sharding, shardId, shardCount, shardRange): + """Returns a list of properties to add to the end of the solrcore.properties file.""" + solrcoreConfig = [] + if sharding != None: + solrcoreConfig.append('solr.port.ssl=8983') + solrcoreConfig.append('shard.instance={}'.format(shardId)) + solrcoreConfig.append('alfresco.port=8080') + solrcoreConfig.append('alfresco.port.ssl=8443') + solrcoreConfig.append('alfresco.baseUrl=/alfresco') + if sharding not in ['DB_ID_RANGE', 'EXPLICIT_ID_FALLBACK_LRIS']: + solrcoreConfig.append('shard.count={}'.format(shardCount)) + if sharding == 'DB_ID_RANGE': + # The first shards each contain 800 nodes by default and the last continues the range to id 100000. + nodesPerShard = shardRange + rangeStart = shardId * nodesPerShard + rangeEnd = (rangeStart + nodesPerShard - 1 if shardId < shardCount - 1 else 100000) + solrcoreConfig.append('shard.range={}-{}'.format(rangeStart, rangeEnd)) + if sharding == 'DATE': + solrcoreConfig.append('shard.key=cm:created') + solrcoreConfig.append('shard.date.grouping={}'.format(12 // shardCount)) + if sharding in ['PROPERTY', 'EXPLICIT_ID', 'EXPLICIT_ID_FALLBACK_LRIS']: + solrcoreConfig.append('shard.key=shard:shardId') + print("SolrConfig for Shard: ", shardId, " : ", solrcoreConfig) + return solrcoreConfig + +def getSolrcoreReplacements(sharding, mtls): + """Returns a dict of replacements to make in the solrcore.properties file.""" + solrcoreReplacements = {} + if sharding != None: + solrcoreReplacements['shard.method=DB_ID'] = 'shard.method={}'.format(sharding) + if mtls: + solrcoreReplacements['alfresco.secureComms=none'] = 'alfresco.secureComms=https' + solrcoreReplacements['alfresco.encryption.ssl.keystore.location=.*'] = 'alfresco.encryption.ssl.keystore.location=\\\\\\/opt\\\\\\/alfresco-search-services\\\\\\/keystore\\\\\\/ssl-repo-client.keystore' + solrcoreReplacements['alfresco.encryption.ssl.keystore.type=.*'] = 'alfresco.encryption.ssl.keystore.type=JCEKS' + solrcoreReplacements['alfresco.encryption.ssl.truststore.location=.*'] = 'alfresco.encryption.ssl.truststore.location=\\\\\\/opt\\\\\\/alfresco-search-services\\\\\\/keystore\\\\\\/ssl-repo-client.truststore' + solrcoreReplacements['alfresco.encryption.ssl.truststore.type=.*'] = 'alfresco.encryption.ssl.truststore.type=JCEKS' + else: + solrcoreReplacements['alfresco.secureComms=https'] = 'alfresco.secureComms=none' + return solrcoreReplacements + +def addAlfrescoMtlsConfig(alfrescoArgsNode): + """Add a list of environment values in Docker Compose Alfresco Service for mTLS.""" + alfrescoArgsNode['TRUSTSTORE_TYPE'] = 'JCEKS' + alfrescoArgsNode['TRUSTSTORE_PASS'] = 'kT9X6oe68t' + alfrescoArgsNode['KEYSTORE_TYPE'] = 'JCEKS' + alfrescoArgsNode['KEYSTORE_PASS'] = 'kT9X6oe68t' + alfrescoArgsNode['SOLR_COMMS'] = 'https' + +def addAlfrescoVolumes(alfrescoNode): + """Add route to keystores folder""" + alfrescoNode['volumes'] = ['./keystores/alfresco:/usr/local/tomcat/alf_data/keystore'] + +def addSolrMtlsConfig(solrEnvNode): + """Add a list of environment values in Docker Compose SOLR Service for mTLS.""" + solrEnvNode['SOLR_SSL_TRUST_STORE'] = '/opt/alfresco-search-services/keystore/ssl-repo-client.truststore' + solrEnvNode['SOLR_SSL_TRUST_STORE_TYPE'] = 'JCEKS' + solrEnvNode['SOLR_SSL_KEY_STORE'] = '/opt/alfresco-search-services/keystore/ssl-repo-client.keystore' + solrEnvNode['SOLR_SSL_KEY_STORE_TYPE'] = 'JCEKS' + solrEnvNode['SOLR_SSL_NEED_CLIENT_AUTH'] = 'true' + +def addSolrOpts(solrEnvNode): + """Add a list of values to add in Docker Compose SOLR_OPTS property for mTLS.""" + solrOptions = ' '.join(['-Dsolr.ssl.checkPeerName=false', + '-Dsolr.allow.unsafe.resourceloading=true']) + solrEnvNode['SOLR_OPTS'] = solrOptions + +def addSolrJavaToolOptions(solrEnvNode): + """Add a list of values to add in Docker Compose JAVA_TOOL_OPTIONS property for mTLS.""" + solrOptions = ' '.join(['-Dsolr.jetty.truststore.password=kT9X6oe68t ', + '-Dsolr.jetty.keystore.password=kT9X6oe68t ', + '-Dssl-keystore.password=kT9X6oe68t', + '-Dssl-keystore.aliases=ssl-alfresco-ca,ssl-repo-client', + '-Dssl-keystore.ssl-alfresco-ca.password=kT9X6oe68t', + '-Dssl-keystore.ssl-repo-client.password=kT9X6oe68t', + '-Dssl-truststore.password=kT9X6oe68t', + '-Dssl-truststore.aliases=ssl-alfresco-ca,ssl-repo,ssl-repo-client', + '-Dssl-truststore.ssl-alfresco-ca.password=kT9X6oe68t', + '-Dssl-truststore.ssl-repo.password=kT9X6oe68t', + '-Dssl-truststore.ssl-repo-client.password=kT9X6oe68t']) + solrEnvNode['JAVA_TOOL_OPTIONS'] = solrOptions + +def addSolrVolumes(solrNode): + """Add route to keystores folder""" + solrNode['volumes'] = ['./keystores/solr:/opt/alfresco-search-services/keystore'] + +def makeSearchNode(outputDirectory, nodeName, externalPort, params, mtls, extraEnvironmentVars={}, solrcoreConfig=[], solrcoreReplacements={}): + # Create a dictionary for the template replacement. + allParams = dict(params) + allParams['SOLR_HOST'] = nodeName + allParams['ALFRESCO_PORT'] = 8443 if mtls else 8080 + allParams['EXTERNAL_PORT'] = externalPort + # Properties to add to solrcore.properties. + allParams['SOLRCORE_PROPERTIES'] = '\\n'.join(solrcoreConfig) + # Replacements to make in solrcore.properties (in an "abc/xyz" format suitable for sed). + allParams['SOLRCORE_REPLACEMENTS'] = ' '.join(map(lambda pair: '"{}/{}"'.format(*pair), solrcoreReplacements.items())) + + # mTLS settings + allParams['ALFRESCO_SECURE_COMMS'] = ('https' if mtls else 'none') + allParams['TRUSTSTORE_TYPE'] = 'JCEKS' + allParams['KEYSTORE_TYPE'] = 'JCEKS' + + # Create a Dockerfile with any extra configuration in. + with open(scriptDir + '/templates/search/Dockerfile.template') as f: + dockerfileTemplate = f.read() + dockerfileString = Template(dockerfileTemplate).substitute(allParams) + if not os.path.isdir('{}/{}'.format(outputDirectory, nodeName)): + os.mkdir('{}/{}'.format(outputDirectory, nodeName)) + with open('{}/{}/Dockerfile'.format(outputDirectory, nodeName), 'w') as f: + dockerfileTemplate = f.write(dockerfileString) + + # Load the search node template. + with open(scriptDir + '/templates/search-node.yml.template') as f: + searchNodeTemplate = f.read() + searchNodeString = Template(searchNodeTemplate).substitute(allParams) + # Read the result as yaml. + searchNodeYaml = yaml.safe_load(searchNodeString) + # Add any extra environment variables. + searchNodeYaml['environment'].update(extraEnvironmentVars) + + # Add mTLS configuration if required + if mtls: + addSolrMtlsConfig(searchNodeYaml['environment']) + addSolrOpts(searchNodeYaml['environment']) + addSolrJavaToolOptions(searchNodeYaml['environment']) + addSolrVolumes(searchNodeYaml) + + return searchNodeYaml + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Generate a docker-compose file for ACS.') + parser.add_argument('-a', '--alfresco', default='quay.io/alfresco/dev:acs-for-search', help='The Alfresco image') + parser.add_argument('-s', '--search', default='quay.io/alfresco/search-services:latest', help='The Search image') + parser.add_argument('-e', '--share', help='The Share image (or omit for no UI)') + parser.add_argument('-p', '--postgres', default='postgres:10.1', help='The Postgres image') + parser.add_argument('-q', '--excludeAMQ', action='store_true', help='Exclude ActiveMQ (i.e. pre-ACS 6.1)') + parser.add_argument('-t', '--transformer', choices=[LIBRE_OFFICE, AIO_TRANSFORMERS], help='Use external transformers. ' + + '"{}" for legacy LibreOffice (i.e. ACS 5.2.x). '.format(LIBRE_OFFICE) + + '"{}" for the all-in-one transformers for use with ACS 6.2.x and later.'.format(AIO_TRANSFORMERS)) + parser.add_argument('-c', '--spellcheck', action='store_true', help='Spellcheck Enabled') + parser.add_argument('-ms', '--masterslave', action='store_true', help='Master Slave Enabled') + parser.add_argument('-sh', '--sharding', help='Sharding method (or omit for no sharding). Note that sharding is not supported on SearchServices 1.2.x or earlier.', + choices=['DB_ID', 'DB_ID_RANGE', 'ACL_ID', 'MOD_ACL_ID', 'DATE', 'PROPERTY', 'LRIS', 'EXPLICIT_ID', 'EXPLICIT_ID_FALLBACK_LRIS']) + parser.add_argument('-sc', '--shardCount', type=int, help='Total number of shards to create (default 2)') + parser.add_argument('-sr', '--shardRange', type=int, help='Total number of nodes per shard with DB_ID_RANGE sharding (default 800)') + parser.add_argument('-ct', '--disableCascadeTracking', action='store_true', help='Cascade Tracking Disabled') + parser.add_argument('-sl', '--searchLogLevel', default='WARN', help='The log level for search (default WARN)', + choices=['TRACE', 'DEBUG', 'INFO', 'WARN', 'ERROR']) + parser.add_argument('-o', '--output', default='.', help='The path of the directory to output to') + parser.add_argument('-mtls', '--mtls', action='store_true', help='Use mTLS between SOLR and Alfresco Repository') + args = parser.parse_args() + + # If sharding is selected then the default number of shards is two. + if args.sharding != None and args.shardCount == None: + print('Using default shardCount of 2') + args.shardCount = 2 + elif args.sharding == None and args.shardCount != None: + print('ERROR: shardCount={} specified without sharding method'.format(args.shardCount)) + exit(1) + print('Arguments:', args) + + # If sharding is selected then the default number of nodes per shard is 800. + if args.sharding == "DB_ID_RANGE" and (args.shardRange == None or args.shardRange < 1): + print('Using default shardRange of 800') + args.shardRange = 800 + elif args.sharding != 'DB_ID_RANGE' and args.shardRange != None: + print('ERROR: shardRange={} is only supported for DB_ID_RANGE sharding.') + exit(1) + print('Arguments:', args) + + # Load the template and perform basic token substitution. + scriptDir = os.path.dirname(os.path.realpath(__file__)) + with open(scriptDir + '/templates/docker-compose.yml.template') as f: + template = f.read() + params = { + 'ALFRESCO_IMAGE': args.alfresco, + 'SHARE_IMAGE': args.share, + 'POSTGRES_IMAGE': args.postgres, + 'SEARCH_IMAGE': args.search, + 'SEARCH_LOG_LEVEL': args.searchLogLevel, + 'SEARCH_ENABLE_SPELLCHECK': str(args.spellcheck).lower(), + 'DISABLE_CASCADE_TRACKING': str(args.disableCascadeTracking).lower() + } + dcString = Template(template).substitute(params) + + # Edit the resulting yaml. + dcYaml = yaml.safe_load(dcString) + + # Insert the search node(s). + shardList = range(args.shardCount) if args.sharding != None else [0] + replicationTypes = ['master', 'slave'] if args.masterslave else ['standalone'] + for shardId in shardList: + for replicationType in replicationTypes: + serviceName = 'search_{}_{}'.format(shardId, replicationType) + # Workaround for ShardInfoTest.getShardInfoWithAdminAuthority. + if shardId == 0 and replicationType == 'standalone': + serviceName = 'search' + externalPort = 8083 + 100 * shardId + (1 if replicationType == 'slave' else 0) + dcYaml['services'][serviceName] = makeSearchNode(args.output, serviceName, externalPort, params, args.mtls, + extraEnvironmentVars=getExtraEnvironmentVars(serviceName, replicationType), + solrcoreConfig=getSolrcoreConfig(args.sharding, shardId, args.shardCount, args.shardRange), + solrcoreReplacements=getSolrcoreReplacements(args.sharding, args.mtls)) + + # Point Alfresco at whichever Solr node came last in the list. + solrHost = serviceName + solrBaseUrl = '/solr-slave' if args.masterslave else '/solr' + + javaOpts = getJavaOpts(not args.excludeAMQ, args.transformer == AIO_TRANSFORMERS, args.share != None, solrHost, solrBaseUrl, args.sharding, args.mtls) + dcYaml['services']['alfresco']['environment']['JAVA_OPTS'] = javaOpts + javaToolOpts = getJavaToolOptions(args.mtls) + dcYaml['services']['alfresco']['environment']['JAVA_TOOL_OPTIONS'] = javaToolOpts + if args.mtls: + addAlfrescoMtlsConfig(dcYaml['services']['alfresco']['build']['args']) + addAlfrescoVolumes(dcYaml['services']['alfresco']) + + if not args.share: + deleteServices(dcYaml, 'share', 'alfresco-pdf-renderer', 'imagemagick') + if args.excludeAMQ: + deleteServices(dcYaml, 'activemq') + if args.transformer != AIO_TRANSFORMERS: + deleteServices(dcYaml, 'transform-core-aio') + del(dcYaml['volumes']['shared-file-store-volume']) + if args.transformer == LIBRE_OFFICE: + dcYaml['services']['libreoffice'] = {'image': 'xcgd/libreoffice'} + + # Output the yaml. + with open(args.output + '/docker-compose.yml', 'w') as f: + f.write(yaml.safe_dump(dcYaml)) + + # Create an Alfresco Dockerfile with any extra configuration in. + with open(scriptDir + '/templates/alfresco/Dockerfile.template') as f: + dockerfileTemplate = f.read() + dockerfileString = Template(dockerfileTemplate).substitute(params) + if not os.path.isdir('{}/{}'.format(args.output, 'alfresco')): + os.mkdir('{}/{}'.format(args.output, 'alfresco')) + with open('{}/{}/Dockerfile'.format(args.output, 'alfresco'), 'w') as f: + dockerfileTemplate = f.write(dockerfileString) + + # Copy the keystores (when using mTLS) + if args.mtls: + copy_tree(scriptDir + '/keystores', args.output + '/keystores') diff --git a/e2e-test/python-generator/keystores/alfresco/keystore b/e2e-test/python-generator/keystores/alfresco/keystore new file mode 100644 index 000000000..439bc1f86 Binary files /dev/null and b/e2e-test/python-generator/keystores/alfresco/keystore differ diff --git a/e2e-test/python-generator/keystores/alfresco/ssl.keystore b/e2e-test/python-generator/keystores/alfresco/ssl.keystore new file mode 100644 index 000000000..c0cff56f8 Binary files /dev/null and b/e2e-test/python-generator/keystores/alfresco/ssl.keystore differ diff --git a/e2e-test/python-generator/keystores/alfresco/ssl.truststore b/e2e-test/python-generator/keystores/alfresco/ssl.truststore new file mode 100644 index 000000000..33da2c362 Binary files /dev/null and b/e2e-test/python-generator/keystores/alfresco/ssl.truststore differ diff --git a/e2e-test/python-generator/keystores/solr/ssl-repo-client.keystore b/e2e-test/python-generator/keystores/solr/ssl-repo-client.keystore new file mode 100644 index 000000000..f1474dd04 Binary files /dev/null and b/e2e-test/python-generator/keystores/solr/ssl-repo-client.keystore differ diff --git a/e2e-test/python-generator/keystores/solr/ssl-repo-client.truststore b/e2e-test/python-generator/keystores/solr/ssl-repo-client.truststore new file mode 100644 index 000000000..c09e6891a Binary files /dev/null and b/e2e-test/python-generator/keystores/solr/ssl-repo-client.truststore differ diff --git a/e2e-test/python-generator/templates/alfresco/Dockerfile.template b/e2e-test/python-generator/templates/alfresco/Dockerfile.template new file mode 100755 index 000000000..3438f963e --- /dev/null +++ b/e2e-test/python-generator/templates/alfresco/Dockerfile.template @@ -0,0 +1,49 @@ +FROM ${ALFRESCO_IMAGE} + +ENV TOMCAT_DIR=/usr/local/tomcat +ENV ALF_DATA_DIR=$${TOMCAT_DIR}/alf_data + +# COMMS +ARG SOLR_COMMS +ENV SOLR_COMMS $$SOLR_COMMS + +# SSL +ARG TRUSTSTORE_TYPE +ARG TRUSTSTORE_PASS +ARG KEYSTORE_TYPE +ARG KEYSTORE_PASS + +ENV TRUSTSTORE_TYPE=$$TRUSTSTORE_TYPE \ + TRUSTSTORE_PASS=$$TRUSTSTORE_PASS \ + KEYSTORE_TYPE=$$KEYSTORE_TYPE \ + KEYSTORE_PASS=$$KEYSTORE_PASS + +USER root + +# Default value in 'repository.properties' is 'dir.keystore=classpath:alfresco/keystore' +RUN if [ "$$SOLR_COMMS" == "https" ] ; then \ + echo -e "\n\ + dir.keystore=$${ALF_DATA_DIR}/keystore\n\ + alfresco.encryption.ssl.keystore.type=$${TRUSTSTORE_TYPE}\n\ + alfresco.encryption.ssl.truststore.type=$${KEYSTORE_TYPE}\n\ + " >> $${TOMCAT_DIR}/shared/classes/alfresco-global.properties; \ + fi + +# Enable SSL by adding the proper Connector to server.xml +RUN if [ "$$SOLR_COMMS" == "https" ] ; then \ + sed -i "s/\ +[[:space:]]\+<\/Engine>/\n\ + <\/Engine>\n\ + \n\ + <\/Connector>/g" $${TOMCAT_DIR}/conf/server.xml; \ + fi + +# Expose keystore folder +# Useless for 'none'/'http' communications with SOLR +VOLUME ["$$ALF_DATA_DIR/keystore"] diff --git a/e2e-test/python-generator/templates/docker-compose.yml.template b/e2e-test/python-generator/templates/docker-compose.yml.template new file mode 100644 index 000000000..a5fd74c85 --- /dev/null +++ b/e2e-test/python-generator/templates/docker-compose.yml.template @@ -0,0 +1,52 @@ +version: '3' +services: + alfresco: + build: + context: ./alfresco + args: + SOLR_COMMS: none + environment: + CATALINA_OPTS : "-agentlib:jdwp=transport=dt_socket,address=*:8000,server=y,suspend=n" + JAVA_OPTS : "This will be populated by the generator script" + JAVA_TOOL_OPTIONS : "This will be populated by the generator script" + ports: + - "7203:7203" #JMX connect via service:jmx:rmi:///jndi/rmi://localhost:7203/jmxrmi + - "8000:8000" #Java debugging + - "8081:8080" #Browser port for Alfresco + share: + image: ${SHARE_IMAGE} + environment: + - REPO_HOST=alfresco + - REPO_PORT=8080 + ports: + - 8082:8080 #Browser port for Share + postgres: + image: ${POSTGRES_IMAGE} + environment: + - POSTGRES_PASSWORD=alfresco + - POSTGRES_USER=alfresco + - POSTGRES_DB=alfresco + ports: + - 5432:5432 + activemq: + image: alfresco/alfresco-activemq:5.15.6 + ports: + - 8161:8161 # Web Console + - 5672:5672 # AMQP + - 61616:61616 # OpenWire + - 61613:61613 # STOMP + transform-core-aio: + image: alfresco/alfresco-transform-core-aio:2.3.5 + environment: + JAVA_OPTS: " -Xms256m -Xmx512m" + ACTIVEMQ_URL: "nio://activemq:61616" + ACTIVEMQ_USER: "admin" + ACTIVEMQ_PASSWORD: "admin" + FILE_STORE_URL: "http://shared-file-store:8099/alfresco/api/-default-/private/sfs/versions/1/file" + ports: + - 8090:8090 +volumes: + shared-file-store-volume: + driver_opts: + type: tmpfs + device: tmpfs diff --git a/e2e-test/python-generator/templates/search-node.yml.template b/e2e-test/python-generator/templates/search-node.yml.template new file mode 100644 index 000000000..502be27f7 --- /dev/null +++ b/e2e-test/python-generator/templates/search-node.yml.template @@ -0,0 +1,17 @@ +build: + context: ./${SOLR_HOST} +environment: + #Solr needs to know how to register itself with Alfresco + SOLR_ALFRESCO_HOST: "alfresco" + SOLR_ALFRESCO_PORT: "${ALFRESCO_PORT}" + #Alfresco needs to know how to call solr + SOLR_SOLR_HOST: "${SOLR_HOST}" + SOLR_SOLR_PORT: "8983" + #Create the default alfresco and archive cores + SOLR_CREATE_ALFRESCO_DEFAULTS: "alfresco,archive" + #Enable Spellcheck by setting to true + ENABLE_SPELLCHECK: "${SEARCH_ENABLE_SPELLCHECK}" + #Disable Cascade Tracking + DISABLE_CASCADE_TRACKING: "${DISABLE_CASCADE_TRACKING}" +ports: + - ${EXTERNAL_PORT}:8983 #Browser port diff --git a/e2e-test/python-generator/templates/search/Dockerfile.template b/e2e-test/python-generator/templates/search/Dockerfile.template new file mode 100644 index 000000000..cd2db8ae2 --- /dev/null +++ b/e2e-test/python-generator/templates/search/Dockerfile.template @@ -0,0 +1,32 @@ +FROM ${SEARCH_IMAGE} + +# Create search_config_setup.sh if it does not exist (e.g. on SearchServices 1.2.x or earlier). +USER root +RUN touch $${DIST_DIR}/solr/bin/search_config_setup.sh \ + && chown solr:solr $${DIST_DIR}/solr/bin/search_config_setup.sh +USER solr + +RUN replacementPairs=(${SOLRCORE_REPLACEMENTS}); \ + for replacementPair in $${replacementPairs[@]}; \ + do \ + sed -i '/^bash.*/i sed -i "'"s/$$replacementPair/g"'" $${DIST_DIR}/solrhome/templates/rerank/conf/solrcore.properties\n' \ + $${DIST_DIR}/solr/bin/search_config_setup.sh; \ + done; \ + if [[ "${SOLRCORE_PROPERTIES}" != "" ]]; \ + then \ + sed -i '/^bash.*/i echo "\n${SOLRCORE_PROPERTIES}" >> $${DIST_DIR}/solrhome/templates/rerank/conf/solrcore.properties\n' \ + $${DIST_DIR}/solr/bin/search_config_setup.sh; \ + fi + +USER root +RUN mkdir -p /opt/alfresco-search-services/keystore \ + && chown -R solr:solr /opt/alfresco-search-services/keystore +USER solr + +# Set the search log level if requested. +RUN if [ "${SEARCH_LOG_LEVEL}" != "" ] ; then \ + sed -i '/^bash.*/i sed -i "'"s/log4j.rootLogger=WARN, file, CONSOLE/log4j.rootLogger=${SEARCH_LOG_LEVEL}, file, CONSOLE/g"'" $${DIST_DIR}/logs/log4j.properties\n' \ + $${DIST_DIR}/solr/bin/search_config_setup.sh; \ +fi + +VOLUME ["/opt/alfresco-search-services/keystore"] diff --git a/e2e-test/src/test/java/org/alfresco/test/search/functional/AbstractE2EFunctionalTest.java b/e2e-test/src/test/java/org/alfresco/test/search/functional/AbstractE2EFunctionalTest.java index 6f377f23a..81dd4430b 100644 --- a/e2e-test/src/test/java/org/alfresco/test/search/functional/AbstractE2EFunctionalTest.java +++ b/e2e-test/src/test/java/org/alfresco/test/search/functional/AbstractE2EFunctionalTest.java @@ -83,7 +83,7 @@ import com.fasterxml.jackson.core.JsonProcessingException; public abstract class AbstractE2EFunctionalTest extends AbstractTestNGSpringContextTests { /** The number of retries that a query will be tried before giving up. */ - protected static final int SEARCH_MAX_ATTEMPTS = 6; + protected static final int SEARCH_MAX_ATTEMPTS = 60; private static final Logger LOGGER = LogFactory.getLogger(); @@ -318,7 +318,7 @@ public abstract class AbstractE2EFunctionalTest extends AbstractTestNGSpringCont } else { - throw new RuntimeException("API returned status code:" + restClient.getStatusCode() + " Expected: " + expectedStatusCode); + throw new RuntimeException("API returned status code:" + restClient.getStatusCode() + " Expected: " + expectedStatusCode + "; Response body: " + response); } } diff --git a/e2e-test/src/test/resources/default.properties b/e2e-test/src/test/resources/default.properties index 72d142995..822a78906 100644 --- a/e2e-test/src/test/resources/default.properties +++ b/e2e-test/src/test/resources/default.properties @@ -21,7 +21,8 @@ solr.server=localhost solr.port=8083 #Solr Indexing Time -solrWaitTimeInSeconds=20 +# Use 1s and 60 attempts, see AbstractE2EFunctionalTest.SEARCH_MAX_ATTEMPTS +solrWaitTimeInSeconds=1 # credentials admin.user=admin diff --git a/pom.xml b/pom.xml index 918c52500..686d987c2 100644 --- a/pom.xml +++ b/pom.xml @@ -12,22 +12,22 @@ Alfresco Search And Insight Parent - alfresco-internal + alfresco-enterprise-releases https://artifacts.alfresco.com/nexus/content/repositories/enterprise-releases/ - alfresco-internal-snapshots + alfresco-enterprise-snapshots https://artifacts.alfresco.com/nexus/content/repositories/enterprise-snapshots/ - scm:git:ssh://git@github.com/Alfresco/InsightEngine.git - scm:git:ssh://git@github.com/Alfresco/InsightEngine.git - scm:git:ssh://git@github.com/Alfresco/InsightEngine.git + scm:git:https://github.com/Alfresco/InsightEngine.git + scm:git:https://github.com/Alfresco/InsightEngine.git + https://github.com/Alfresco/InsightEngine HEAD - 11 + 11 6.6.5 ${solr.base.version}-patched.2 @@ -46,17 +46,6 @@ - - org.apache.maven.plugins - maven-compiler-plugin - 3.8.1 - - ${java.version} - ${java.version} - true - true - - org.apache.maven.plugins maven-failsafe-plugin diff --git a/search-services/alfresco-search/pom.xml b/search-services/alfresco-search/pom.xml index 33be2c76a..25e1066d5 100644 --- a/search-services/alfresco-search/pom.xml +++ b/search-services/alfresco-search/pom.xml @@ -212,43 +212,6 @@ - - - alfresco-public - https://artifacts.alfresco.com/nexus/content/groups/public - - true - - - false - - - - alfresco-public-snapshots - https://artifacts.alfresco.com/nexus/content/groups/public-snapshots - - false - - - true - - - - central - Central Repository - https://repo.maven.apache.org/maven2 - default - - false - - - - maven-restlet - Public online Restlet repository - http://maven.restlet.talend.com - - - alfresco-solr diff --git a/search-services/alfresco-solrclient-lib/pom.xml b/search-services/alfresco-solrclient-lib/pom.xml index 00b6c975a..c876de216 100644 --- a/search-services/alfresco-solrclient-lib/pom.xml +++ b/search-services/alfresco-solrclient-lib/pom.xml @@ -10,17 +10,6 @@ 2.0.2-SNAPSHOT - - - alfresco-internal - https://artifacts.alfresco.com/nexus/content/repositories/releases - - - alfresco-internal-snapshots - https://artifacts.alfresco.com/nexus/content/repositories/snapshots - - - 8.319 2.11.3 diff --git a/search-services/packaging/pom.xml b/search-services/packaging/pom.xml index c36270cd3..261b273c8 100644 --- a/search-services/packaging/pom.xml +++ b/search-services/packaging/pom.xml @@ -8,7 +8,6 @@ UTF-8 UTF-8 ${maven.build.timestamp} - ${bamboo_repository_revision_number} org.alfresco @@ -82,7 +81,7 @@ unpack-solr-config - validate + prepare-package unpack @@ -99,7 +98,7 @@ unpack-solr-libs - validate + prepare-package unpack @@ -119,7 +118,7 @@ copy - validate + prepare-package copy diff --git a/search-services/pom.xml b/search-services/pom.xml index 2c277d1a3..a69ca506b 100644 --- a/search-services/pom.xml +++ b/search-services/pom.xml @@ -23,4 +23,52 @@ alfresco-search packaging + + + + alfresco-public-releases + https://artifacts.alfresco.com/nexus/content/repositories/releases + + + alfresco-public-snapshots + https://artifacts.alfresco.com/nexus/content/repositories/snapshots + + + + + + alfresco-public + https://artifacts.alfresco.com/nexus/content/groups/public + + true + + + false + + + + alfresco-public-snapshots + https://artifacts.alfresco.com/nexus/content/groups/public-snapshots + + false + + + true + + + + central + Central Repository + https://repo.maven.apache.org/maven2 + default + + false + + + + maven-restlet + Public online Restlet repository + http://maven.restlet.talend.com + +