diff --git a/.travis.yml b/.travis.yml index 1ede9fe6aa..4714b39982 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,14 +1,22 @@ +--- import: - source: travis-env-vars.yml -os: linux dist: xenial +sudo: required language: java -jdk: - - openjdk11 +jdk: openjdk11 + +addons: + firefox: "43.0.1" services: + - xvfb - docker +git: + depth: false + quiet: true + branches: only: - master @@ -18,14 +26,16 @@ branches: cache: directories: - - $HOME/.m2 + - ${HOME}/.m2/repository # the cache can grow constantly -before_cache: - - rm -rf $HOME/.m2/repository/org/alfresco/alfresco-governance-services* +before_cache: rm -rf ${HOME}/.m2/repository/org/alfresco/alfresco-governance-services* before_install: - - "cp .travis.settings.xml $HOME/.m2/settings.xml" + - mkdir -p "${HOME}/.m2" && cp -f .travis.settings.xml "${HOME}/.m2/settings.xml" + - find "${HOME}/.m2/repository/" -type d -name "*-SNAPSHOT*" | xargs -r -l rm -rf + - docker login quay.io -u ${QUAY_USERNAME} -p ${QUAY_PASSWORD} + - docker login -u ${DOCKERHUB_USERNAME} -p ${DOCKERHUB_PASSWORD} install: skip stages: @@ -46,18 +56,18 @@ jobs: - name: "Build AGS Enterprise" stage: Build AGS - before_script: source scripts/setUpMavenPhase.sh install: - travis_retry travis_wait 30 mvn -B deploy -N - travis_retry travis_wait 60 mvn -B -q clean install $MVN_SKIP -f rm-community/pom.xml -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn + before_script: source scripts/setUpMavenPhase.sh script: - travis_retry travis_wait 80 mvn -B -q ${MAVEN_PHASE} -P${BUILD_PROFILE} -Dimage.tag=${IMAGE_TAG} -Dskip.integrationtests=false -f rm-enterprise/pom.xml -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn - name: "Build AGS Benchmark" stage: Build AGS - before_script: source scripts/setUpMavenPhase.sh install: - travis_retry travis_wait 80 mvn -B -q clean install $MVN_SKIP -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn + before_script: source scripts/setUpMavenPhase.sh script: - travis_retry travis_wait 35 mvn -B -q ${MAVEN_PHASE} -Dskip.integrationtests=false -f rm-benchmark/pom.xml -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn @@ -71,41 +81,68 @@ jobs: - echo "Enterprise Integrations Tests on MySQL" - name: "Community Rest API Tests" stage: Tests - before_install: - - travis_retry travis_wait 90 mvn -B -q install $MVN_SKIP -PbuildDockerImage -pl :alfresco-governance-services-community-repo -am install: + - travis_retry travis_wait 90 mvn -B -q install $MVN_SKIP -PbuildDockerImage -pl :alfresco-governance-services-community-repo -am + before_script: - bash scripts/startAlfresco.sh $COMMUNITY_REPO_PATH - bash scripts/waitForAlfrescoToStart.sh script: - echo "Community Rest API Tests" - name: "Enterprise Rest API Tests" stage: Tests - before_install: - - travis_retry travis_wait 90 mvn -B -q install $MVN_SKIP -PbuildDockerImage -pl :alfresco-governance-services-enterprise-repo -am install: + - travis_retry travis_wait 90 mvn -B -q install $MVN_SKIP -PbuildDockerImage -pl :alfresco-governance-services-enterprise-repo -am + before_script: - bash scripts/startAlfresco.sh $ENTERPRISE_REPO_PATH - bash scripts/waitForAlfrescoToStart.sh script: - echo "Enterprise Rest API Tests" + + - name: "Enterprise Rest API WORM Tests" + stage: Tests + install: + - travis_retry travis_wait 90 mvn -B -U -q clean install ${MVN_SKIP} -PbuildDockerImage -pl :alfresco-governance-services-enterprise-repo,:alfresco-governance-services-enterprise-share -am + - travis_retry travis_wait 30 mvn -B -U -q clean install ${MVN_SKIP} -pl :alfresco-governance-services-automation-enterprise-rest-api -am + before_script: + - bash scripts/create-worm-bucket.sh + - bash scripts/start-compose.sh "${ENTERPRISE_SHARE_PATH}/docker-compose-worm-support-rest.yml" + - bash scripts/waitForAlfrescoToStart.sh + script: mvn -B test -pl :alfresco-governance-services-automation-enterprise-rest-api -DsuiteXmlFile=wormTestSuite.xml -Dskip.automationtests=false + after_script: bash scripts/cleanup.sh + after_failure: docker ps -a | grep '_alfresco_1' | awk '{print $1}' | xargs docker logs | tail -5000 + - name: "Community UI Tests ..." stage: Tests - before_install: - - travis_retry travis_wait 90 mvn -B -q install $MVN_SKIP -PbuildDockerImage -pl :alfresco-governance-services-community-repo,:alfresco-governance-services-community-share -am install: + - travis_retry travis_wait 90 mvn -B -q install $MVN_SKIP -PbuildDockerImage -pl :alfresco-governance-services-community-repo,:alfresco-governance-services-community-share -am + before_script: - bash scripts/startAlfresco.sh $COMMUNITY_SHARE_PATH - bash scripts/waitForAlfrescoToStart.sh script: - echo "Community UI Tests ..." - name: "Enterprise UI Tests ..." stage: Tests - before_install: - - travis_retry travis_wait 90 mvn -B -q install $MVN_SKIP -PbuildDockerImage -pl :alfresco-governance-services-enterprise-repo,:alfresco-governance-services-enterprise-share -am install: + - travis_retry travis_wait 90 mvn -B -q install $MVN_SKIP -PbuildDockerImage -pl :alfresco-governance-services-enterprise-repo,:alfresco-governance-services-enterprise-share -am + before_script: - bash scripts/startAlfresco.sh $ENTERPRISE_SHARE_PATH - bash scripts/waitForAlfrescoToStart.sh script: - echo "Enterprise UI Tests ..." + - name: "Enterprise UI WORM Tests" + stage: Tests + install: + - travis_retry travis_wait 90 mvn -B -U -q clean install ${MVN_SKIP} -PbuildDockerImage -pl :alfresco-governance-services-enterprise-repo,:alfresco-governance-services-enterprise-share -am + - travis_retry travis_wait 30 mvn -B -U -q clean install ${MVN_SKIP} -pl :alfresco-governance-services-automation-ui -am + before_script: + - bash scripts/create-worm-bucket.sh + - bash scripts/start-compose.sh "${ENTERPRISE_SHARE_PATH}/docker-compose-worm-support-ui.yml" + - bash scripts/waitForAlfrescoToStart.sh + script: mvn -B test -pl :alfresco-governance-services-automation-ui -DsuiteXmlFile=wormTestSuite.xml -Dskip.automationtests=false -Dshare.url=${SHARE_URL} -Dalfresco.url=${ALFRESCO_URL} -Dwebdriver.local.grid=true -Dwebdriver.browser=RemoteFireFox -Dwebdriver.localGrid=false -Dwebdriver.element.wait.time=20000 -Dwebdriver.page.render.wait.time=60000 + after_script: bash scripts/cleanup.sh + after_failure: docker ps -a | grep '_alfresco_1' | awk '{print $1}' | xargs docker logs | tail -5000 + - name: "Source Clear Scan (SCA)" stage: Security Scans script: diff --git a/pom.xml b/pom.xml index b534bc1222..b5e6c207c0 100644 --- a/pom.xml +++ b/pom.xml @@ -983,7 +983,7 @@ maven-failsafe-plugin - 2.22.1 + 3.0.0-M5 --illegal-access=permit diff --git a/scripts/cleanup.sh b/scripts/cleanup.sh new file mode 100644 index 0000000000..d7993ad92f --- /dev/null +++ b/scripts/cleanup.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +echo "=========================== Starting Cleanup Script ===========================" +PS4="\[\e[35m\]+ \[\e[m\]" +set -vx +pushd "$(dirname "${BASH_SOURCE[0]}")/../" + + +# Stop and remove the containers +docker ps -a -q | xargs -l -r docker stop +docker ps -a -q | xargs -l -r docker rm + +pip install awscli +printf "${CREATE_BUCKET_AWS_ACCESS_KEY}\n${CREATE_BUCKET_AWS_SECRET_KEY}\n\n\n" | aws configure + +aws s3 ls | awk '{print $3}' | grep "^${S3_BUCKET_NAME}" | xargs -l -r -I{} aws s3 rb "s3://{}" --force +aws s3 ls | awk '{print $3}' | grep "^${S3_BUCKET2_NAME}" | xargs -l -r -I{} aws s3 rb "s3://{}" --force + +popd +set +vx +echo "=========================== Finishing Cleanup Script ==========================" \ No newline at end of file diff --git a/scripts/create-worm-bucket.sh b/scripts/create-worm-bucket.sh new file mode 100644 index 0000000000..a3ff254ad6 --- /dev/null +++ b/scripts/create-worm-bucket.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash + +echo "=========================== Create Worm Bucket ===========================" +PS4="\[\e[35m\]+ \[\e[m\]" +set -vex +pushd "$(dirname "${BASH_SOURCE[0]}")/../" + +pip install awscli +printf "${CREATE_BUCKET_AWS_ACCESS_KEY}\n${CREATE_BUCKET_AWS_SECRET_KEY}\n\n\n" | aws configure + +if aws s3 ls | awk '{print $3}' | grep -q "^${S3_BUCKET2_NAME}$" ; then + echo "Bucket ${S3_BUCKET2_NAME} already exists" + exit 0 +fi + +aws s3api create-bucket --bucket "${S3_BUCKET2_NAME}" --region ${S3_BUCKET_REGION} --object-lock-enabled-for-bucket +aws s3api put-object-lock-configuration \ + --bucket "${S3_BUCKET2_NAME}" \ + --object-lock-configuration 'ObjectLockEnabled=Enabled,Rule={DefaultRetention={Mode=COMPLIANCE,Days=1}}' + +aws s3api put-bucket-tagging --bucket "${S3_BUCKET2_NAME}" \ + --tagging="TagSet=[{Key=toDeleteAfterTests,Value=true}]" + +popd +set +vex +echo "=========================== Finishing Create Worm Bucket Script ==========================" diff --git a/scripts/delete-test-buckets-lambda.py b/scripts/delete-test-buckets-lambda.py new file mode 100644 index 0000000000..974f1bb220 --- /dev/null +++ b/scripts/delete-test-buckets-lambda.py @@ -0,0 +1,105 @@ +import json +import boto3 +from datetime import datetime, timedelta, timezone +from botocore.exceptions import ClientError + + +# This python module is intended for use as a Python 3 AWS lambda function +# Tested in python 3.6 environment +# The AWS role used with this lambda function will need AmazonS3FullAccess and CloudWatchLogsFullAccess permissions +# Tested with role lambda_s3_execution_role in engineering account + +# Retrieve bucket's tag set +def get_tagset(bucket): + try: + return bucket.Tagging().tag_set + except ClientError as e: + return [] + +# Check if a bucket should be deleted +def tag_matches(bucket): + for tag in get_tagset(bucket): + if tag["Key"] == "toDeleteAfterTests" and tag["Value"] == "true" : + return True + return False + +def age_matches(bucket): + delta = datetime.now(timezone.utc) - bucket.creation_date + return delta.days > 0 + +def prefix_matches(bucket, prefix): + if not prefix: + return True + if bucket.name.startswith(prefix): + return True + return False + +# Get a list of buckets to delete +def get_buckets_to_delete(prefix): + s3 = boto3.resource('s3') + + # Get all buckets matching bucket name prefix + prefixed_buckets = [bucket for bucket in s3.buckets.all() if prefix_matches(bucket, prefix)] + + # Filter buckets on tag + # tagged_buckets = [bucket for bucket in prefixed_buckets if tag_matches(bucket)] + + # Filter buckets on age + old_buckets = [bucket for bucket in prefixed_buckets if age_matches(bucket)] + + return old_buckets + +# Delete bucket +def delete_bucket(bucket): + try: + [object.delete for object in bucket.objects.all()] + except ClientError as e: + print("Failed to delete objects in bucket: " + bucket.name) + print(e) + try: + bucket.objects.all().delete() + except ClientError as e: + print("Failed to delete objects in bucket: " + bucket.name) + print(e) + + try: + [version.delete() for version in bucket.object_versions.all()] + except ClientError as e: + print("Failed to delete object_versions in bucket: " + bucket.name) + print(e) + try: + bucket.object_versions.delete() + except ClientError as e: + print("Failed to delete object_versions in bucket: " + bucket.name) + print(e) + + try: + bucket.delete() + print("Bucket " + bucket.name + " was deleted") + except ClientError as e: + print("Failed to delete bucket: " + bucket.name) + print(e) + + +# Non-empty buckets are deleted (recursively); failed attempts will be logged. +# The buckets are filtered on the name prefix: "travis-ags-worm-" +def lambda_handler(event, context): + + # Retrieve bucket name prefix option + prefix = "travis-ags-" + + # Get a list of buckets to delete + buckets_to_delete = get_buckets_to_delete(prefix) + + # Delete buckets + print ("Deleting buckets:") + for bucket in buckets_to_delete : + print (bucket.name) + delete_bucket(bucket) + + return { + 'statusCode': 200, + 'body': json.dumps('Done!') + } + +#lambda_handler(None, None) diff --git a/scripts/setUpMavenPhase.sh b/scripts/setUpMavenPhase.sh index 688f019df8..384131f1ee 100644 --- a/scripts/setUpMavenPhase.sh +++ b/scripts/setUpMavenPhase.sh @@ -2,27 +2,22 @@ echo "Branch name: ${TRAVIS_BRANCH}" echo "Pull request: ${TRAVIS_PULL_REQUEST}" echo "Travis job name: ${TRAVIS_JOB_NAME}" -branchName=${TRAVIS_BRANCH} -imageTag=${branchName:8} -echo "Image tag: ${imageTag}" +echo "Image tag: ${TRAVIS_BRANCH:8}" - if [[ ${TRAVIS_JOB_NAME} == "Build AGS Enterprise" ]]; - then - export BUILD_PROFILE="internal" - else - export BUILD_PROFILE="master" - fi +if [[ ${TRAVIS_JOB_NAME} == "Build AGS Enterprise" ]] ; then + export BUILD_PROFILE="internal" +else + export BUILD_PROFILE="master" +fi - if [[ "${TRAVIS_BRANCH}" == "master" && "${TRAVIS_PULL_REQUEST}" == "false" ]]; - then - export MAVEN_PHASE="deploy" - export IMAGE_TAG="latest" - elif [[ ${TRAVIS_BRANCH} = release* && "${TRAVIS_PULL_REQUEST}" == "false" ]]; - then - export MAVEN_PHASE="deploy" - export IMAGE_TAG="${imageTag}-latest" - else - export MAVEN_PHASE="verify" - export BUILD_PROFILE="buildDockerImage" - export IMAGE_TAG="latest" - fi +if [[ "${TRAVIS_BRANCH}" == "master" && "${TRAVIS_PULL_REQUEST}" == "false" ]] ; then + export MAVEN_PHASE="deploy" + export IMAGE_TAG="latest" +elif [[ ${TRAVIS_BRANCH} = release* && "${TRAVIS_PULL_REQUEST}" == "false" ]] ; then + export MAVEN_PHASE="deploy" + export IMAGE_TAG="${TRAVIS_BRANCH:8}-latest" +else + export MAVEN_PHASE="verify" + export BUILD_PROFILE="buildDockerImage" + export IMAGE_TAG="latest" +fi diff --git a/scripts/start-compose.sh b/scripts/start-compose.sh new file mode 100755 index 0000000000..a3b47dcb2e --- /dev/null +++ b/scripts/start-compose.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +export DOCKER_COMPOSE_PATH="${1}" + +if [ -z "$DOCKER_COMPOSE_PATH" ] ; then + echo "Please provide path to docker-compose.yml: \"${0##*/} /path/to/docker-compose.yml\"" + exit 1 +fi + +echo "Starting AGS stack in ${DOCKER_COMPOSE_PATH}" + +# .env files are picked up from project directory correctly on docker-compose 1.23.0+ +docker-compose --file "${DOCKER_COMPOSE_PATH}" --project-directory $(dirname "${DOCKER_COMPOSE_PATH}") up -d + +if [ $? -eq 0 ] ; then + echo "Docker Compose started ok" +else + echo "Docker Compose failed to start" >&2 + exit 1 +fi \ No newline at end of file diff --git a/scripts/startAlfresco.sh b/scripts/startAlfresco.sh index b161dbde05..4143c742d2 100644 --- a/scripts/startAlfresco.sh +++ b/scripts/startAlfresco.sh @@ -3,5 +3,4 @@ set -ev cd $1 -docker login quay.io -u ${QUAY_USERNAME} -p ${QUAY_PASSWORD} docker-compose up -d diff --git a/scripts/waitForAlfrescoToStart.sh b/scripts/waitForAlfrescoToStart.sh index 59c7e40733..065ae39544 100755 --- a/scripts/waitForAlfrescoToStart.sh +++ b/scripts/waitForAlfrescoToStart.sh @@ -3,21 +3,27 @@ WAIT_INTERVAL=1 COUNTER=0 TIMEOUT=300 -t0=`date +%s` +t0=$(date +%s) echo "Waiting for alfresco to start" -until $(curl --output /dev/null --silent --head --fail http://localhost:8080/alfresco) || [ "$COUNTER" -eq "$TIMEOUT" ]; do +until $(curl --output /dev/null --silent --head --fail ${ALFRESCO_URL}) || [ "$COUNTER" -eq "$TIMEOUT" ]; do printf '.' sleep $WAIT_INTERVAL COUNTER=$(($COUNTER+$WAIT_INTERVAL)) done if (("$COUNTER" < "$TIMEOUT")) ; then - t1=`date +%s` + t1=$(date +%s) delta=$((($t1 - $t0)/60)) echo "Alfresco Started in $delta minutes" else echo "Waited $COUNTER seconds" echo "Alfresco Could not start in time." + echo "All started containers:" + docker ps -a + ALFCONTAINER=$(docker ps -a | grep _alfresco_1 | awk '{ print $1 }') + echo "Last 200 lines from alfresco.log on container $ALFCONTAINER:" + docker logs --tail=200 ${ALFCONTAINER} exit 1 -fi \ No newline at end of file +fi + diff --git a/travis-env-vars.yml b/travis-env-vars.yml index aa0e1f1190..fe0aed8112 100644 --- a/travis-env-vars.yml +++ b/travis-env-vars.yml @@ -8,3 +8,11 @@ env: - ENTERPRISE_REPO_PATH=rm-enterprise/rm-enterprise-repo - ENTERPRISE_SHARE_PATH=rm-enterprise/rm-enterprise-share - MVN_SKIP='-Dmaven.javadoc.skip=true -Dmaven.source.skip=true -DskipTests' + - AWS_ACCESS_KEY_ID=${CREATE_BUCKET_AWS_ACCESS_KEY} + - AWS_SECRET_ACCESS_KEY=${CREATE_BUCKET_AWS_SECRET_KEY} + - S3_BUCKET_REGION="us-east-1" + - S3_BUCKET_NAME="travis-ags-${TRAVIS_JOB_NUMBER}" + - S3_BUCKET2_NAME="travis-ags-worm-b2" + - ALFRESCO_URL="http://localhost:8080/alfresco" + - SHARE_URL="http://localhost:8181/share" + - DISPLAY=:99.0