SEARCH-2318: Add Travis configuration to Insight Engine (#873)

The changes include the following:
* The python scripts for e2e environment creation were moved to this code base
* Mirror to community
* Docker image check
* Unit, integration and e2e tests
* "Snapshots": "latest" docker image from master in quay.io, docker images for "release" branches to quay.io, "master" and * "release" branches snapshots to nexus
* Releases: nexus, s3 staging bucket, quay.io and dockerhub
* Fix for https://issues.alfresco.com/jira/browse/SEARCH-2389

The build is using branches only.

e2e-test/helpers/start-alfresco.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+set -eux
+
+# Start Alfresco and Solr.
+
+# The location for the docker-compose files.
+DOCKER_RESOURCE_FOLDER=$1
+# The search docker image.
+SEARCH_IMAGE=$2
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+export DOCKER_CLIENT_TIMEOUT=120
+export COMPOSE_HTTP_TIMEOUT=120
+
+# Build the images and call docker-compose.
+cd "$DOCKER_RESOURCE_FOLDER"
+docker-compose up -d --build --force-recreate
+
+$SCRIPT_DIR/wait-service-to-start.sh

e2e-test/helpers/wait-service-to-start.sh

@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+set -e  # exit if commands fails
+set -x  # trace what gets exe
+
+WAIT_INTERVAL=1
+COUNTER=0
+TIMEOUT=2000
+t0=`date +%s`
+
+declare -a endpoints=("${1:-http://localhost:8081/alfresco/}" "${1:-http://localhost:8083/solr/}")
+
+for endpoint in "${endpoints[@]}"
+do
+
+   echo "Waiting for Service to start using endpoint: ${endpoint}"
+
+   until [[ "$(curl --output /dev/null -w ''%{http_code}'' --silent --head --fail ${endpoint})" == 200 ]] || [ "$COUNTER" -eq "$TIMEOUT" ]; do
+      printf '.'
+      sleep $WAIT_INTERVAL
+      COUNTER=$(($COUNTER+$WAIT_INTERVAL))
+   done
+
+   if (("$COUNTER" < "$TIMEOUT")) ; then
+      t1=`date +%s`
+      delta=$((($t1 - $t0)/60))
+      echo "Service ${endpoint} Started in $delta minutes"
+   else
+      echo "Service ${endpoint} could not start in time."
+      echo "Waited $COUNTER seconds"
+      exit 1
+   fi
+done

e2e-test/pom.xml

@@ -5,7 +5,7 @@
       <artifactId>alfresco-search-and-insight-parent</artifactId>
       <version>2.0.2-SNAPSHOT</version>
    </parent>
-   <groupId>search-analytics-e2e-test</groupId>
+   <groupId>org.alfresco</groupId>
    <artifactId>search-analytics-e2e-test</artifactId>
    <name>Search Analytics E2E Tests</name>
    <description>Test Project to test Search Service and Analytics Features on a complete setup of Alfresco, Share</description>
@@ -128,4 +128,37 @@
          <version>12-ea+10</version>
       </dependency>
    </dependencies>
+
+    <repositories>
+        <repository>
+            <id>alfresco-enterprise-releases</id>
+            <url>https://artifacts.alfresco.com/nexus/content/repositories/enterprise-releases</url>
+            <releases>
+                <enabled>true</enabled>
+            </releases>
+            <snapshots>
+                <enabled>false</enabled>
+            </snapshots>
+        </repository>
+        <repository>
+            <id>alfresco-enterprise-snapshots</id>
+            <url>https://artifacts.alfresco.com/nexus/content/repositories/enterprise-snapshots</url>
+            <releases>
+                <enabled>false</enabled>
+            </releases>
+            <snapshots>
+                <enabled>true</enabled>
+            </snapshots>
+        </repository>
+        <repository>
+            <id>alfresco-hotfix</id>
+            <url>https://artifacts.alfresco.com/nexus/content/groups/hotfix</url>
+            <releases>
+                <enabled>true</enabled>
+            </releases>
+            <snapshots>
+                <enabled>false</enabled>
+            </snapshots>
+        </repository>
+    </repositories>
 </project>

e2e-test/python-generator/README.md

@@ -0,0 +1,46 @@
+# About
+A python script to generator docker-compose files suitable for use with the automated tests in
+the [Search and Insight E2E tests](https://git.alfresco.com/search_discovery/insightengine/tree/master/e2e-test).
+
+# Installation
+The script uses Python 3, which can be installed from [python.org](https://www.python.org/downloads/) or using
+a package manager. You will also need the [yaml library](https://pypi.org/project/PyYAML/), which can be installed
+using Pip if it is not already present:
+```
+pip3 install pyyaml
+```
+
+# Using the script
+The script provides some help with the `-h` option:
+
+```
+python3 BuildScripts/generator/generator.py -h
+usage: generator.py [-h] [-a ALFRESCO]...
+...
+```
+
+For use with ACS 6.2.x the legacy transformers can be included:
+```
+python3 BuildScripts/generator/generator.py --alfresco=alfresco/alfresco-content-repository:6.2.0 --transformer=AIOTransformers
+```
+
+For ACS 6.0.x and ACS 6.1.x no external transformer is needed:
+```
+python3 BuildScripts/generator/generator.py --alfresco=alfresco/alfresco-content-repository:6.1.0
+```
+
+For ACS 6.0.0.x and earlier ActiveMQ can be excluded:
+```
+python3 BuildScripts/generator/generator.py --alfresco=alfresco/alfresco-content-repository:6.0.0.3 --excludeAMQ
+```
+
+For ACS 5.2.x the legacy LibreOffice transformer can be used:
+```
+python3 BuildScripts/generator/generator.py --alfresco=quay.io/alfresco/alfresco-content-repository-52:5.2.5 --postgres=postgres:9.4  --excludeAMQ --transformer=LibreOffice
+```
+
+# Starting the containers
+To start the containers you also need to build the images - for example:
+```
+docker-compose up --build --force-recreate
+```

e2e-test/python-generator/generator.py

@@ -0,0 +1,325 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+import argparse
+from string import Template
+import yaml
+import os
+from distutils.dir_util import copy_tree
+
+LIBRE_OFFICE = 'LibreOffice'
+AIO_TRANSFORMERS = 'AIOTransformers'
+
+AMQ_OPTS = '-Dmessaging.broker.url="failover:(nio://activemq:61616)?timeout=3000&jms.useCompression=true"'
+TRANSFORM_OPTS = ('-DlocalTransform.core-aio.url=http://transform-core-aio:8090/ '
+        '-Dalfresco-pdf-renderer.url=http://transform-core-aio:8090/ '
+        '-Djodconverter.url=http://transform-core-aio:8090/ '
+        '-Dimg.url=http://transform-core-aio:8090/ '
+        '-Dtika.url=http://transform-core-aio:8090/ '
+        '-Dtransform.misc.url=http://transform-core-aio:8090/')
+SHARE_TRANSFORM_OPTS = ('-DlocalTransform.pdfrenderer.url=http://alfresco-pdf-renderer:8090/ '
+        '-Dalfresco-pdf-renderer.url=http://alfresco-pdf-renderer:8090/ '
+        '-DlocalTransform.imagemagick.url=http://imagemagick:8090/ -Dimg.url=http://imagemagick:8090/')
+SHARDING_OPTS = '-Dsolr.useDynamicShardRegistration=true'
+JAVA_OPTS = ('-Ddb.driver=org.postgresql.Driver -Ddb.username=alfresco -Ddb.password=alfresco '
+        '-Ddb.url=jdbc:postgresql://postgres:5432/alfresco -Dsolr.port=8983 '
+        '-Dindex.subsystem.name=solr6 '
+        '-Dalfresco.restApi.basicAuthScheme=true '
+        # longer timeouts for CI
+        '-Dsolr.http.socket.timeout=30000 '
+        '-Dsolr.http.connection.timeout=3000 ')
+MTLS_OPTS = ('-Dsolr.port.ssl=8983 -Dsolr.secureComms=https ')
+HTTP_OPTS = ('-Dsolr.secureComms=none')
+JAVA_TOOL_OPTIONS = ('-Dencryption.keystore.type=JCEKS '
+        '-Dencryption.cipherAlgorithm=DESede/CBC/PKCS5Padding '
+        '-Dencryption.keyAlgorithm=DESede '
+        '-Dencryption.keystore.location=/usr/local/tomcat/shared/classes/alfresco/extension/keystore/keystore '
+        '-Dmetadata-keystore.password=mp6yc0UD9e '
+        '-Dmetadata-keystore.aliases=metadata '
+        '-Dmetadata-keystore.metadata.password=oKIWzVdEdA '
+        '-Dmetadata-keystore.metadata.algorithm=DESede')
+MTLS_JAVA_TOOL_OPTIONS = ('-Dencryption.keystore.type=pkcs12 -Dencryption.cipherAlgorithm=AES/CBC/PKCS5Padding '
+        '-Dencryption.keyAlgorithm=DESede '
+        '-Dssl-truststore.password=kT9X6oe68t -Dssl-keystore.password=kT9X6oe68t '
+        '-Dssl-keystore.aliases=ssl-alfresco-ca,ssl-repo '
+        '-Dssl-keystore.ssl-alfresco-ca.password=kT9X6oe68t '
+        '-Dssl-keystore.ssl-repo.password=kT9X6oe68t '
+        '-Dssl-truststore.aliases=alfresco-ca,ssl-repo-client '
+        '-Dssl-truststore.alfresco-ca.password=kT9X6oe68t '
+        '-Dssl-truststore.ssl-repo-client.password=kT9X6oe68t')
+
+def getJavaOpts(includeAMQ, includeTransform, includeShare, solrHost, solrBaseUrl, sharding, mtls):
+
+    solrHost = '-Dsolr.host=' + solrHost
+    shardingOpts = (SHARDING_OPTS if sharding != None else '')
+    amqOpts = (AMQ_OPTS if includeAMQ else '')
+    transformOpts = (TRANSFORM_OPTS if includeTransform else '')
+    solrBaseUrlOpts = '-Dsolr.baseUrl=' + solrBaseUrl
+    shareTransformOpts = (SHARE_TRANSFORM_OPTS if includeShare and includeTransform else '')
+    mtlsOpts = (MTLS_OPTS if mtls else HTTP_OPTS)
+
+    return ' '.join([JAVA_OPTS, amqOpts, transformOpts, shareTransformOpts, solrHost, solrBaseUrlOpts, shardingOpts, mtlsOpts])
+
+def getJavaToolOptions(mtls):
+
+    mtlsJavaToolOptions = (MTLS_JAVA_TOOL_OPTIONS if mtls else '')
+
+    return ' '.join([JAVA_TOOL_OPTIONS, mtlsJavaToolOptions])
+
+def deleteServices(dcYaml, *services):
+    for service in services:
+        if service in dcYaml['services'].keys():
+            del(dcYaml['services'][service])
+
+def getExtraEnvironmentVars(serviceName, replicationType):
+    """Return a dict of environment variables to add to the search container declaration in docker-compose.yml."""
+    extraEnvironmentVars = {}
+    if replicationType == 'master':
+        extraEnvironmentVars['REPLICATION_TYPE'] = 'master'
+    elif replicationType == 'slave':
+        extraEnvironmentVars.update({'REPLICATION_TYPE': 'slave',
+                                 'REPLICATION_MASTER_HOST': serviceName.replace('slave', 'master'),
+                                 'REPLICATION_MASTER_PORT': '8983',
+                                 'REPLICATION_POLL_INTERVAL': '00:00:10'})
+    return extraEnvironmentVars
+
+def getSolrcoreConfig(sharding, shardId, shardCount, shardRange):
+    """Returns a list of properties to add to the end of the solrcore.properties file."""
+    solrcoreConfig = []
+    if sharding != None:
+        solrcoreConfig.append('solr.port.ssl=8983')
+        solrcoreConfig.append('shard.instance={}'.format(shardId))
+        solrcoreConfig.append('alfresco.port=8080')
+        solrcoreConfig.append('alfresco.port.ssl=8443')
+        solrcoreConfig.append('alfresco.baseUrl=/alfresco')
+        if sharding not in ['DB_ID_RANGE', 'EXPLICIT_ID_FALLBACK_LRIS']:
+            solrcoreConfig.append('shard.count={}'.format(shardCount))
+        if sharding == 'DB_ID_RANGE':
+            # The first shards each contain 800 nodes by default and the last continues the range to id 100000.
+            nodesPerShard = shardRange
+            rangeStart = shardId * nodesPerShard
+            rangeEnd = (rangeStart + nodesPerShard - 1 if shardId < shardCount - 1 else 100000)
+            solrcoreConfig.append('shard.range={}-{}'.format(rangeStart, rangeEnd))
+        if sharding == 'DATE':
+            solrcoreConfig.append('shard.key=cm:created')
+            solrcoreConfig.append('shard.date.grouping={}'.format(12 // shardCount))
+        if sharding in ['PROPERTY', 'EXPLICIT_ID', 'EXPLICIT_ID_FALLBACK_LRIS']:
+            solrcoreConfig.append('shard.key=shard:shardId')
+        print("SolrConfig for Shard: ", shardId, " : ", solrcoreConfig)
+    return solrcoreConfig
+
+def getSolrcoreReplacements(sharding, mtls):
+    """Returns a dict of replacements to make in the solrcore.properties file."""
+    solrcoreReplacements = {}
+    if sharding != None:
+        solrcoreReplacements['shard.method=DB_ID'] = 'shard.method={}'.format(sharding)
+    if mtls:
+        solrcoreReplacements['alfresco.secureComms=none'] = 'alfresco.secureComms=https'
+        solrcoreReplacements['alfresco.encryption.ssl.keystore.location=.*'] = 'alfresco.encryption.ssl.keystore.location=\\\\\\/opt\\\\\\/alfresco-search-services\\\\\\/keystore\\\\\\/ssl-repo-client.keystore'
+        solrcoreReplacements['alfresco.encryption.ssl.keystore.type=.*'] = 'alfresco.encryption.ssl.keystore.type=JCEKS'
+        solrcoreReplacements['alfresco.encryption.ssl.truststore.location=.*'] = 'alfresco.encryption.ssl.truststore.location=\\\\\\/opt\\\\\\/alfresco-search-services\\\\\\/keystore\\\\\\/ssl-repo-client.truststore'
+        solrcoreReplacements['alfresco.encryption.ssl.truststore.type=.*'] = 'alfresco.encryption.ssl.truststore.type=JCEKS'
+    else:
+        solrcoreReplacements['alfresco.secureComms=https'] = 'alfresco.secureComms=none'
+    return solrcoreReplacements
+
+def addAlfrescoMtlsConfig(alfrescoArgsNode):
+    """Add a list of environment values in Docker Compose Alfresco Service for mTLS."""
+    alfrescoArgsNode['TRUSTSTORE_TYPE'] = 'JCEKS'
+    alfrescoArgsNode['TRUSTSTORE_PASS'] = 'kT9X6oe68t'
+    alfrescoArgsNode['KEYSTORE_TYPE'] = 'JCEKS'
+    alfrescoArgsNode['KEYSTORE_PASS'] = 'kT9X6oe68t'
+    alfrescoArgsNode['SOLR_COMMS'] = 'https'
+
+def addAlfrescoVolumes(alfrescoNode):
+    """Add route to keystores folder"""
+    alfrescoNode['volumes'] = ['./keystores/alfresco:/usr/local/tomcat/alf_data/keystore']
+
+def addSolrMtlsConfig(solrEnvNode):
+    """Add a list of environment values in Docker Compose SOLR Service for mTLS."""
+    solrEnvNode['SOLR_SSL_TRUST_STORE'] = '/opt/alfresco-search-services/keystore/ssl-repo-client.truststore'
+    solrEnvNode['SOLR_SSL_TRUST_STORE_TYPE'] = 'JCEKS'
+    solrEnvNode['SOLR_SSL_KEY_STORE'] = '/opt/alfresco-search-services/keystore/ssl-repo-client.keystore'
+    solrEnvNode['SOLR_SSL_KEY_STORE_TYPE'] = 'JCEKS'
+    solrEnvNode['SOLR_SSL_NEED_CLIENT_AUTH'] = 'true'
+
+def addSolrOpts(solrEnvNode):
+    """Add a list of values to add in Docker Compose SOLR_OPTS property for mTLS."""
+    solrOptions = ' '.join(['-Dsolr.ssl.checkPeerName=false',
+          '-Dsolr.allow.unsafe.resourceloading=true'])
+    solrEnvNode['SOLR_OPTS'] = solrOptions
+
+def addSolrJavaToolOptions(solrEnvNode):
+    """Add a list of values to add in Docker Compose JAVA_TOOL_OPTIONS property for mTLS."""
+    solrOptions = ' '.join(['-Dsolr.jetty.truststore.password=kT9X6oe68t ',
+          '-Dsolr.jetty.keystore.password=kT9X6oe68t ',
+          '-Dssl-keystore.password=kT9X6oe68t',
+          '-Dssl-keystore.aliases=ssl-alfresco-ca,ssl-repo-client',
+          '-Dssl-keystore.ssl-alfresco-ca.password=kT9X6oe68t',
+          '-Dssl-keystore.ssl-repo-client.password=kT9X6oe68t',
+          '-Dssl-truststore.password=kT9X6oe68t',
+          '-Dssl-truststore.aliases=ssl-alfresco-ca,ssl-repo,ssl-repo-client',
+          '-Dssl-truststore.ssl-alfresco-ca.password=kT9X6oe68t',
+          '-Dssl-truststore.ssl-repo.password=kT9X6oe68t',
+          '-Dssl-truststore.ssl-repo-client.password=kT9X6oe68t'])
+    solrEnvNode['JAVA_TOOL_OPTIONS'] = solrOptions
+
+def addSolrVolumes(solrNode):
+    """Add route to keystores folder"""
+    solrNode['volumes'] = ['./keystores/solr:/opt/alfresco-search-services/keystore']
+
+def makeSearchNode(outputDirectory, nodeName, externalPort, params, mtls, extraEnvironmentVars={}, solrcoreConfig=[], solrcoreReplacements={}):
+    # Create a dictionary for the template replacement.
+    allParams = dict(params)
+    allParams['SOLR_HOST'] = nodeName
+    allParams['ALFRESCO_PORT'] = 8443 if mtls else 8080
+    allParams['EXTERNAL_PORT'] = externalPort
+    # Properties to add to solrcore.properties.
+    allParams['SOLRCORE_PROPERTIES'] = '\\n'.join(solrcoreConfig)
+    # Replacements to make in solrcore.properties (in an "abc/xyz" format suitable for sed).
+    allParams['SOLRCORE_REPLACEMENTS'] = ' '.join(map(lambda pair: '"{}/{}"'.format(*pair), solrcoreReplacements.items()))
+
+    # mTLS settings
+    allParams['ALFRESCO_SECURE_COMMS'] = ('https' if mtls else 'none')
+    allParams['TRUSTSTORE_TYPE'] = 'JCEKS'
+    allParams['KEYSTORE_TYPE'] = 'JCEKS'
+
+    # Create a Dockerfile with any extra configuration in.
+    with open(scriptDir + '/templates/search/Dockerfile.template') as f:
+        dockerfileTemplate = f.read()
+    dockerfileString = Template(dockerfileTemplate).substitute(allParams)
+    if not os.path.isdir('{}/{}'.format(outputDirectory, nodeName)):
+        os.mkdir('{}/{}'.format(outputDirectory, nodeName))
+    with open('{}/{}/Dockerfile'.format(outputDirectory, nodeName), 'w') as f:
+        dockerfileTemplate = f.write(dockerfileString)
+
+    # Load the search node template.
+    with open(scriptDir + '/templates/search-node.yml.template') as f:
+        searchNodeTemplate = f.read()
+    searchNodeString = Template(searchNodeTemplate).substitute(allParams)
+    # Read the result as yaml.
+    searchNodeYaml = yaml.safe_load(searchNodeString)
+    # Add any extra environment variables.
+    searchNodeYaml['environment'].update(extraEnvironmentVars)
+
+    # Add mTLS configuration if required
+    if mtls:
+        addSolrMtlsConfig(searchNodeYaml['environment'])
+        addSolrOpts(searchNodeYaml['environment'])
+        addSolrJavaToolOptions(searchNodeYaml['environment'])
+        addSolrVolumes(searchNodeYaml)
+
+    return searchNodeYaml
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='Generate a docker-compose file for ACS.')
+    parser.add_argument('-a', '--alfresco', default='quay.io/alfresco/dev:acs-for-search', help='The Alfresco image')
+    parser.add_argument('-s', '--search', default='quay.io/alfresco/search-services:latest', help='The Search image')
+    parser.add_argument('-e', '--share', help='The Share image (or omit for no UI)')
+    parser.add_argument('-p', '--postgres', default='postgres:10.1', help='The Postgres image')
+    parser.add_argument('-q', '--excludeAMQ', action='store_true', help='Exclude ActiveMQ (i.e. pre-ACS 6.1)')
+    parser.add_argument('-t', '--transformer', choices=[LIBRE_OFFICE, AIO_TRANSFORMERS], help='Use external transformers. '
+                        + '"{}" for legacy LibreOffice (i.e. ACS 5.2.x). '.format(LIBRE_OFFICE)
+                        + '"{}" for the all-in-one transformers for use with ACS 6.2.x and later.'.format(AIO_TRANSFORMERS))
+    parser.add_argument('-c', '--spellcheck', action='store_true', help='Spellcheck Enabled')
+    parser.add_argument('-ms', '--masterslave', action='store_true', help='Master Slave Enabled')
+    parser.add_argument('-sh', '--sharding', help='Sharding method (or omit for no sharding). Note that sharding is not supported on SearchServices 1.2.x or earlier.',
+                        choices=['DB_ID', 'DB_ID_RANGE', 'ACL_ID', 'MOD_ACL_ID', 'DATE', 'PROPERTY', 'LRIS', 'EXPLICIT_ID', 'EXPLICIT_ID_FALLBACK_LRIS'])
+    parser.add_argument('-sc', '--shardCount', type=int, help='Total number of shards to create (default 2)')
+    parser.add_argument('-sr', '--shardRange', type=int, help='Total number of nodes per shard with DB_ID_RANGE sharding (default 800)')
+    parser.add_argument('-ct', '--disableCascadeTracking', action='store_true', help='Cascade Tracking Disabled')
+    parser.add_argument('-sl', '--searchLogLevel', default='WARN', help='The log level for search (default WARN)',
+                        choices=['TRACE', 'DEBUG', 'INFO', 'WARN', 'ERROR'])
+    parser.add_argument('-o', '--output', default='.', help='The path of the directory to output to')
+    parser.add_argument('-mtls', '--mtls', action='store_true', help='Use mTLS between SOLR and Alfresco Repository')
+    args = parser.parse_args()
+
+    # If sharding is selected then the default number of shards is two.
+    if args.sharding != None and args.shardCount == None:
+        print('Using default shardCount of 2')
+        args.shardCount = 2
+    elif args.sharding == None and args.shardCount != None:
+        print('ERROR: shardCount={} specified without sharding method'.format(args.shardCount))
+        exit(1)
+    print('Arguments:', args)
+
+    # If sharding is selected then the default number of nodes per shard is 800.
+    if args.sharding == "DB_ID_RANGE" and (args.shardRange == None or args.shardRange < 1):
+        print('Using default shardRange of 800')
+        args.shardRange = 800
+    elif args.sharding != 'DB_ID_RANGE' and args.shardRange != None:
+        print('ERROR: shardRange={} is only supported for DB_ID_RANGE sharding.')
+        exit(1)
+    print('Arguments:', args)
+
+    # Load the template and perform basic token substitution.
+    scriptDir = os.path.dirname(os.path.realpath(__file__))
+    with open(scriptDir + '/templates/docker-compose.yml.template') as f:
+        template = f.read()
+    params = {
+            'ALFRESCO_IMAGE': args.alfresco,
+            'SHARE_IMAGE': args.share,
+            'POSTGRES_IMAGE': args.postgres,
+            'SEARCH_IMAGE': args.search,
+            'SEARCH_LOG_LEVEL': args.searchLogLevel,
+            'SEARCH_ENABLE_SPELLCHECK': str(args.spellcheck).lower(),
+            'DISABLE_CASCADE_TRACKING': str(args.disableCascadeTracking).lower()
+            }
+    dcString = Template(template).substitute(params)
+
+    # Edit the resulting yaml.
+    dcYaml = yaml.safe_load(dcString)
+
+    # Insert the search node(s).
+    shardList = range(args.shardCount) if args.sharding != None else [0]
+    replicationTypes = ['master', 'slave'] if args.masterslave else ['standalone']
+    for shardId in shardList:
+        for replicationType in replicationTypes:
+            serviceName = 'search_{}_{}'.format(shardId, replicationType)
+            # Workaround for ShardInfoTest.getShardInfoWithAdminAuthority.
+            if shardId == 0 and replicationType == 'standalone':
+                serviceName = 'search'
+            externalPort = 8083 + 100 * shardId + (1 if replicationType == 'slave' else 0)
+            dcYaml['services'][serviceName] = makeSearchNode(args.output, serviceName, externalPort, params, args.mtls,
+                      extraEnvironmentVars=getExtraEnvironmentVars(serviceName, replicationType),
+                      solrcoreConfig=getSolrcoreConfig(args.sharding, shardId, args.shardCount, args.shardRange),
+                      solrcoreReplacements=getSolrcoreReplacements(args.sharding, args.mtls))
+
+    # Point Alfresco at whichever Solr node came last in the list.
+    solrHost = serviceName
+    solrBaseUrl = '/solr-slave' if args.masterslave else '/solr'
+
+    javaOpts = getJavaOpts(not args.excludeAMQ, args.transformer == AIO_TRANSFORMERS, args.share != None, solrHost, solrBaseUrl, args.sharding, args.mtls)
+    dcYaml['services']['alfresco']['environment']['JAVA_OPTS'] = javaOpts
+    javaToolOpts = getJavaToolOptions(args.mtls)
+    dcYaml['services']['alfresco']['environment']['JAVA_TOOL_OPTIONS'] = javaToolOpts
+    if args.mtls:
+        addAlfrescoMtlsConfig(dcYaml['services']['alfresco']['build']['args'])
+        addAlfrescoVolumes(dcYaml['services']['alfresco'])
+
+    if not args.share:
+        deleteServices(dcYaml, 'share', 'alfresco-pdf-renderer', 'imagemagick')
+    if args.excludeAMQ:
+        deleteServices(dcYaml, 'activemq')
+    if args.transformer != AIO_TRANSFORMERS:
+        deleteServices(dcYaml, 'transform-core-aio')
+        del(dcYaml['volumes']['shared-file-store-volume'])
+    if args.transformer == LIBRE_OFFICE:
+        dcYaml['services']['libreoffice'] = {'image': 'xcgd/libreoffice'}
+
+    # Output the yaml.
+    with open(args.output + '/docker-compose.yml', 'w') as f:
+        f.write(yaml.safe_dump(dcYaml))
+
+    # Create an Alfresco Dockerfile with any extra configuration in.
+    with open(scriptDir + '/templates/alfresco/Dockerfile.template') as f:
+        dockerfileTemplate = f.read()
+    dockerfileString = Template(dockerfileTemplate).substitute(params)
+    if not os.path.isdir('{}/{}'.format(args.output, 'alfresco')):
+        os.mkdir('{}/{}'.format(args.output, 'alfresco'))
+    with open('{}/{}/Dockerfile'.format(args.output, 'alfresco'), 'w') as f:
+        dockerfileTemplate = f.write(dockerfileString)
+
+    # Copy the keystores (when using mTLS)
+    if args.mtls:
+        copy_tree(scriptDir + '/keystores', args.output + '/keystores')

e2e-test/python-generator/templates/alfresco/Dockerfile.template

@@ -0,0 +1,49 @@
+FROM ${ALFRESCO_IMAGE}
+
+ENV TOMCAT_DIR=/usr/local/tomcat
+ENV ALF_DATA_DIR=$${TOMCAT_DIR}/alf_data
+
+# COMMS
+ARG SOLR_COMMS
+ENV SOLR_COMMS $$SOLR_COMMS
+
+# SSL
+ARG TRUSTSTORE_TYPE
+ARG TRUSTSTORE_PASS
+ARG KEYSTORE_TYPE
+ARG KEYSTORE_PASS
+
+ENV TRUSTSTORE_TYPE=$$TRUSTSTORE_TYPE \
+    TRUSTSTORE_PASS=$$TRUSTSTORE_PASS \
+    KEYSTORE_TYPE=$$KEYSTORE_TYPE \
+    KEYSTORE_PASS=$$KEYSTORE_PASS
+
+USER root
+
+# Default value in 'repository.properties' is 'dir.keystore=classpath:alfresco/keystore'
+RUN if [ "$$SOLR_COMMS" == "https" ] ; then \
+        echo -e "\n\
+        dir.keystore=$${ALF_DATA_DIR}/keystore\n\
+        alfresco.encryption.ssl.keystore.type=$${TRUSTSTORE_TYPE}\n\
+        alfresco.encryption.ssl.truststore.type=$${KEYSTORE_TYPE}\n\
+        " >> $${TOMCAT_DIR}/shared/classes/alfresco-global.properties; \
+    fi
+
+# Enable SSL by adding the proper Connector to server.xml
+RUN if [ "$$SOLR_COMMS" == "https" ] ; then \
+      sed -i "s/\
+[[:space:]]\+<\/Engine>/\n\
+        <\/Engine>\n\
+        <Connector port=\"8443\" protocol=\"org.apache.coyote.http11.Http11Protocol\"\n\
+            connectionTimeout=\"20000\"\n\
+            SSLEnabled=\"true\" maxThreads=\"150\" scheme=\"https\"\n\
+            keystoreFile=\"\/usr\/local\/tomcat\/alf_data\/keystore\/ssl.keystore\"\n\
+            keystorePass=\"$${KEYSTORE_PASS}\" keystoreType=\"$${KEYSTORE_TYPE}\" secure=\"true\"\n\
+            truststoreFile=\"\/usr\/local\/tomcat\/alf_data\/keystore\/ssl.truststore\"\n\
+            truststorePass=\"$${TRUSTSTORE_PASS}\" truststoreType=\"$${TRUSTSTORE_TYPE}\" clientAuth=\"want\" sslProtocol=\"TLS\">\n\
+        <\/Connector>/g" $${TOMCAT_DIR}/conf/server.xml; \
+    fi
+
+# Expose keystore folder
+# Useless for 'none'/'http' communications with SOLR
+VOLUME ["$$ALF_DATA_DIR/keystore"]

e2e-test/python-generator/templates/docker-compose.yml.template

@@ -0,0 +1,52 @@
+version: '3'
+services:
+  alfresco:
+    build:
+      context: ./alfresco
+      args:
+        SOLR_COMMS: none
+    environment:
+      CATALINA_OPTS : "-agentlib:jdwp=transport=dt_socket,address=*:8000,server=y,suspend=n"
+      JAVA_OPTS : "This will be populated by the generator script"
+      JAVA_TOOL_OPTIONS : "This will be populated by the generator script"
+    ports:
+      - "7203:7203" #JMX connect via service:jmx:rmi:///jndi/rmi://localhost:7203/jmxrmi
+      - "8000:8000" #Java debugging
+      - "8081:8080" #Browser port for Alfresco
+  share:
+    image: ${SHARE_IMAGE}
+    environment:
+      - REPO_HOST=alfresco
+      - REPO_PORT=8080
+    ports:
+      - 8082:8080 #Browser port for Share
+  postgres:
+    image: ${POSTGRES_IMAGE}
+    environment:
+      - POSTGRES_PASSWORD=alfresco
+      - POSTGRES_USER=alfresco
+      - POSTGRES_DB=alfresco
+    ports:
+      - 5432:5432
+  activemq:
+    image: alfresco/alfresco-activemq:5.15.6
+    ports:
+      - 8161:8161 # Web Console
+      - 5672:5672 # AMQP
+      - 61616:61616 # OpenWire
+      - 61613:61613 # STOMP
+  transform-core-aio:
+    image: alfresco/alfresco-transform-core-aio:2.3.5
+    environment:
+      JAVA_OPTS: " -Xms256m -Xmx512m"
+      ACTIVEMQ_URL: "nio://activemq:61616"
+      ACTIVEMQ_USER: "admin"
+      ACTIVEMQ_PASSWORD: "admin"
+      FILE_STORE_URL: "http://shared-file-store:8099/alfresco/api/-default-/private/sfs/versions/1/file"
+    ports:
+      - 8090:8090
+volumes:
+    shared-file-store-volume:
+        driver_opts:
+            type: tmpfs
+            device: tmpfs

e2e-test/python-generator/templates/search-node.yml.template

@@ -0,0 +1,17 @@
+build:
+  context: ./${SOLR_HOST}
+environment:
+  #Solr needs to know how to register itself with Alfresco
+  SOLR_ALFRESCO_HOST: "alfresco"
+  SOLR_ALFRESCO_PORT: "${ALFRESCO_PORT}"
+  #Alfresco needs to know how to call solr
+  SOLR_SOLR_HOST: "${SOLR_HOST}"
+  SOLR_SOLR_PORT: "8983"
+  #Create the default alfresco and archive cores
+  SOLR_CREATE_ALFRESCO_DEFAULTS: "alfresco,archive"
+  #Enable Spellcheck by setting to true
+  ENABLE_SPELLCHECK: "${SEARCH_ENABLE_SPELLCHECK}"
+  #Disable Cascade Tracking
+  DISABLE_CASCADE_TRACKING: "${DISABLE_CASCADE_TRACKING}"
+ports:
+  - ${EXTERNAL_PORT}:8983 #Browser port

e2e-test/python-generator/templates/search/Dockerfile.template

@@ -0,0 +1,32 @@
+FROM ${SEARCH_IMAGE}
+
+# Create search_config_setup.sh if it does not exist (e.g. on SearchServices 1.2.x or earlier).
+USER root
+RUN touch $${DIST_DIR}/solr/bin/search_config_setup.sh \
+    && chown solr:solr $${DIST_DIR}/solr/bin/search_config_setup.sh
+USER solr
+
+RUN replacementPairs=(${SOLRCORE_REPLACEMENTS}); \
+    for replacementPair in $${replacementPairs[@]}; \
+    do \
+        sed -i '/^bash.*/i sed -i "'"s/$$replacementPair/g"'" $${DIST_DIR}/solrhome/templates/rerank/conf/solrcore.properties\n' \
+        $${DIST_DIR}/solr/bin/search_config_setup.sh; \
+    done; \
+    if [[ "${SOLRCORE_PROPERTIES}" != "" ]]; \
+    then \
+        sed -i '/^bash.*/i echo "\n${SOLRCORE_PROPERTIES}" >> $${DIST_DIR}/solrhome/templates/rerank/conf/solrcore.properties\n' \
+        $${DIST_DIR}/solr/bin/search_config_setup.sh; \
+    fi
+
+USER root
+RUN mkdir -p /opt/alfresco-search-services/keystore \
+    && chown -R solr:solr /opt/alfresco-search-services/keystore
+USER solr
+
+# Set the search log level if requested.
+RUN if [ "${SEARCH_LOG_LEVEL}" != "" ] ; then \
+  sed -i '/^bash.*/i sed -i "'"s/log4j.rootLogger=WARN, file, CONSOLE/log4j.rootLogger=${SEARCH_LOG_LEVEL}, file, CONSOLE/g"'" $${DIST_DIR}/logs/log4j.properties\n' \
+  $${DIST_DIR}/solr/bin/search_config_setup.sh; \
+fi
+
+VOLUME ["/opt/alfresco-search-services/keystore"]

e2e-test/src/test/java/org/alfresco/test/search/functional/AbstractE2EFunctionalTest.java

@@ -83,7 +83,7 @@ import com.fasterxml.jackson.core.JsonProcessingException;
 public abstract class AbstractE2EFunctionalTest extends AbstractTestNGSpringContextTests
 {
     /** The number of retries that a query will be tried before giving up. */
-    protected static final int SEARCH_MAX_ATTEMPTS = 6;
+    protected static final int SEARCH_MAX_ATTEMPTS = 60;
 
     private static final Logger LOGGER = LogFactory.getLogger();
 
@@ -318,7 +318,7 @@ public abstract class AbstractE2EFunctionalTest extends AbstractTestNGSpringCont
             }
             else
             {
-                throw new RuntimeException("API returned status code:" + restClient.getStatusCode() + " Expected: " + expectedStatusCode);
+                throw new RuntimeException("API returned status code:" + restClient.getStatusCode() + " Expected: " + expectedStatusCode + "; Response body: " + response);
             }
         }
 

e2e-test/src/test/resources/default.properties

@@ -21,7 +21,8 @@ solr.server=localhost
 solr.port=8083
 
 #Solr Indexing Time
-solrWaitTimeInSeconds=20
+# Use 1s and 60 attempts, see AbstractE2EFunctionalTest.SEARCH_MAX_ATTEMPTS
+solrWaitTimeInSeconds=1
 
 # credentials
 admin.user=admin

pom.xml

@@ -12,22 +12,22 @@
    <name>Alfresco Search And Insight Parent</name>
    <distributionManagement>
       <repository>
-         <id>alfresco-internal</id>
+         <id>alfresco-enterprise-releases</id>
          <url>https://artifacts.alfresco.com/nexus/content/repositories/enterprise-releases/</url>
       </repository>
       <snapshotRepository>
-         <id>alfresco-internal-snapshots</id>
+         <id>alfresco-enterprise-snapshots</id>
          <url>https://artifacts.alfresco.com/nexus/content/repositories/enterprise-snapshots/</url>
       </snapshotRepository>
    </distributionManagement>
    <scm>
-      <connection>scm:git:ssh://git@github.com/Alfresco/InsightEngine.git</connection>
+      <connection>scm:git:https://github.com/Alfresco/InsightEngine.git</connection>
-      <developerConnection>scm:git:ssh://git@github.com/Alfresco/InsightEngine.git</developerConnection>
+      <developerConnection>scm:git:https://github.com/Alfresco/InsightEngine.git</developerConnection>
-      <url>scm:git:ssh://git@github.com/Alfresco/InsightEngine.git</url>
+      <url>https://github.com/Alfresco/InsightEngine</url>
       <tag>HEAD</tag>
    </scm>
    <properties>
-      <java.version>11</java.version>
+      <maven.build.sourceVersion>11</maven.build.sourceVersion>
       <solr.base.version>6.6.5</solr.base.version>
       <solr.version>${solr.base.version}-patched.2</solr.version>
       <!-- The location to download the solr zip file from. -->
@@ -46,17 +46,6 @@
    </modules>
    <build>
       <plugins>
-         <plugin>
-            <groupId>org.apache.maven.plugins</groupId>
-            <artifactId>maven-compiler-plugin</artifactId>
-            <version>3.8.1</version>
-            <configuration>
-               <release>${java.version}</release>
-               <target>${java.version}</target>
-               <showWarnings>true</showWarnings>
-               <showDeprecation>true</showDeprecation>
-            </configuration>
-         </plugin>
          <plugin>
             <groupId>org.apache.maven.plugins</groupId>
             <artifactId>maven-failsafe-plugin</artifactId>

search-services/alfresco-search/pom.xml

@@ -212,43 +212,6 @@
         </dependency>
     </dependencies>
 
-    <repositories>
-        <repository>
-            <id>alfresco-public</id>
-            <url>https://artifacts.alfresco.com/nexus/content/groups/public</url>
-            <releases>
-                <enabled>true</enabled>
-            </releases>
-            <snapshots>
-                <enabled>false</enabled>
-            </snapshots>
-        </repository>
-        <repository>
-            <id>alfresco-public-snapshots</id>
-            <url>https://artifacts.alfresco.com/nexus/content/groups/public-snapshots</url>
-            <releases>
-                <enabled>false</enabled>
-            </releases>
-            <snapshots>
-                <enabled>true</enabled>
-            </snapshots>
-        </repository>
-        <repository>
-            <id>central</id>
-            <name>Central Repository</name>
-            <url>https://repo.maven.apache.org/maven2</url>
-            <layout>default</layout>
-            <snapshots>
-                <enabled>false</enabled>
-            </snapshots>
-        </repository>
-        <repository>
-            <id>maven-restlet</id>
-            <name>Public online Restlet repository</name>
-            <url>http://maven.restlet.talend.com</url>
-        </repository>
-    </repositories>
-
     <build>
         <finalName>alfresco-solr</finalName>
         <plugins>

search-services/alfresco-solrclient-lib/pom.xml

@@ -10,17 +10,6 @@
         <version>2.0.2-SNAPSHOT</version>
     </parent>
 
-    <distributionManagement>
-        <repository>
-            <id>alfresco-internal</id>
-            <url>https://artifacts.alfresco.com/nexus/content/repositories/releases</url>
-        </repository>
-        <snapshotRepository>
-            <id>alfresco-internal-snapshots</id>
-            <url>https://artifacts.alfresco.com/nexus/content/repositories/snapshots</url>
-        </snapshotRepository>
-    </distributionManagement>
-
     <properties>
         <dependency.alfresco-data-model.version>8.319</dependency.alfresco-data-model.version>
         <dependency.jackson.version>2.11.3</dependency.jackson.version>

search-services/packaging/pom.xml

@@ -8,7 +8,6 @@
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
         <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
         <project.timestamp>${maven.build.timestamp}</project.timestamp>
-        <project.revision.key>${bamboo_repository_revision_number}</project.revision.key>
     </properties>
     <parent>
         <groupId>org.alfresco</groupId>
@@ -82,7 +81,7 @@
                 <executions>
                     <execution>
                         <id>unpack-solr-config</id>
-                        <phase>validate</phase>
+                        <phase>prepare-package</phase>
                         <goals>
                             <goal>unpack</goal>
                         </goals>
@@ -99,7 +98,7 @@
                     </execution>
                     <execution>
                         <id>unpack-solr-libs</id>
-                        <phase>validate</phase>
+                        <phase>prepare-package</phase>
                         <goals>
                             <goal>unpack</goal>
                         </goals>
@@ -119,7 +118,7 @@
                     </execution>
                     <execution>
                         <id>copy</id>
-                        <phase>validate</phase>
+                        <phase>prepare-package</phase>
                         <goals>
                             <goal>copy</goal>
                         </goals>

search-services/pom.xml

@@ -23,4 +23,52 @@
         <module>alfresco-search</module>
         <module>packaging</module>
     </modules>
+
+    <distributionManagement>
+        <repository>
+            <id>alfresco-public-releases</id>
+            <url>https://artifacts.alfresco.com/nexus/content/repositories/releases</url>
+        </repository>
+        <snapshotRepository>
+            <id>alfresco-public-snapshots</id>
+            <url>https://artifacts.alfresco.com/nexus/content/repositories/snapshots</url>
+        </snapshotRepository>
+    </distributionManagement>
+
+    <repositories>
+        <repository>
+            <id>alfresco-public</id>
+            <url>https://artifacts.alfresco.com/nexus/content/groups/public</url>
+            <releases>
+                <enabled>true</enabled>
+            </releases>
+            <snapshots>
+                <enabled>false</enabled>
+            </snapshots>
+        </repository>
+        <repository>
+            <id>alfresco-public-snapshots</id>
+            <url>https://artifacts.alfresco.com/nexus/content/groups/public-snapshots</url>
+            <releases>
+                <enabled>false</enabled>
+            </releases>
+            <snapshots>
+                <enabled>true</enabled>
+            </snapshots>
+        </repository>
+        <repository>
+            <id>central</id>
+            <name>Central Repository</name>
+            <url>https://repo.maven.apache.org/maven2</url>
+            <layout>default</layout>
+            <snapshots>
+                <enabled>false</enabled>
+            </snapshots>
+        </repository>
+        <repository>
+            <id>maven-restlet</id>
+            <name>Public online Restlet repository</name>
+            <url>http://maven.restlet.talend.com</url>
+        </repository>
+    </repositories>
 </project>