From 231963f5467875bba350d1a76fdd5b606e1b9a67 Mon Sep 17 00:00:00 2001 From: Claudia Agache Date: Tue, 3 Nov 2020 08:11:40 +0200 Subject: [PATCH 01/12] Merge remote-tracking branch 'remotes/origin/release/V3.1' into merge-3.2/APPS-238_APPS-104_no --- travis-env-vars.yml => .travis.env-vars.yml | 4 + .travis.publish-stage.yml | 43 ++++++ .travis.release-stage.yml | 52 ++++++++ .travis.tests-stage.yml | 140 ++++++++++++++++++++ .travis.yml | 81 +---------- pom.xml | 6 +- scripts/check-existing-tags.sh | 18 +++ scripts/dockerLimitMemory.sh | 7 +- scripts/release.sh | 44 ++++++ scripts/set-release-variables.sh | 16 +++ scripts/zip-artifacts-release.sh | 24 ++++ scripts/zip-artifacts-staging.sh | 15 +++ 12 files changed, 371 insertions(+), 79 deletions(-) rename travis-env-vars.yml => .travis.env-vars.yml (66%) create mode 100644 .travis.publish-stage.yml create mode 100644 .travis.release-stage.yml create mode 100644 .travis.tests-stage.yml create mode 100644 scripts/check-existing-tags.sh create mode 100755 scripts/release.sh create mode 100755 scripts/set-release-variables.sh create mode 100644 scripts/zip-artifacts-release.sh create mode 100755 scripts/zip-artifacts-staging.sh diff --git a/travis-env-vars.yml b/.travis.env-vars.yml similarity index 66% rename from travis-env-vars.yml rename to .travis.env-vars.yml index aa0e1f1190..016c5bc889 100644 --- a/travis-env-vars.yml +++ b/.travis.env-vars.yml @@ -8,3 +8,7 @@ env: - ENTERPRISE_REPO_PATH=rm-enterprise/rm-enterprise-repo - ENTERPRISE_SHARE_PATH=rm-enterprise/rm-enterprise-share - MVN_SKIP='-Dmaven.javadoc.skip=true -Dmaven.source.skip=true -DskipTests' + - ALFRESCO_URL="http://localhost:8080/alfresco" + - SHARE_URL="http://localhost:8181/share" + - DISPLAY=:99.0 + - WEBDRIVER_ARGUMENTS='-Dwebdriver.local.grid=true -Dwebdriver.browser=RemoteFireFox -Dwebdriver.localGrid=false -Dorg.alfresco.rm.timeout_seconds.MEDIUM=30' diff --git a/.travis.publish-stage.yml b/.travis.publish-stage.yml new file mode 100644 index 0000000000..fdb32962c4 --- /dev/null +++ b/.travis.publish-stage.yml @@ -0,0 +1,43 @@ +jobs: + include: + - name: "Community Publish to S3 Release Bucket" + stage: Publish + if: commit_message =~ /\[community release .*\]/ AND commit_message =~ /\[publish\]/ + before_script: source scripts/set-release-variables.sh + script: skip + before_deploy: bash scripts/zip-artifacts-release.sh "community" + deploy: + - provider: s3 + access_key_id: ${RELEASE_AWS_ACCESS_KEY} + secret_access_key: $RELEASE_AWS_ACCESS_SECRET} + region: "eu-west-1" + bucket: "eu.dl.alfresco.com" + upload_dir: "release/community/RM/${RELEASE_VERSION}" + skip_cleanup: true + acl: private + local_dir: artifacts_dir + on: + all_branches: true + after_deploy: + - echo "Finished release and deployed to https://s3.console.aws.amazon.com/s3/buckets/eu.dl.alfresco.com/release/community/RM/${RELEASE_VERSION}" + + - name: "Enterprise Publish to S3 Release Bucket" + stage: Publish + if: commit_message =~ /\[enterprise release .*\]/ AND commit_message =~ /\[publish\]/ + before_script: source scripts/set-release-variables.sh + script: skip + before_deploy: bash scripts/zip-artifacts-release.sh "enterprise" + deploy: + - provider: s3 + access_key_id: ${RELEASE_AWS_ACCESS_KEY} + secret_access_key: ${RELEASE_AWS_ACCESS_SECRET} + region: "eu-west-1" + bucket: "eu.dl.alfresco.com" + upload_dir: "release/enterprise/RM/${RELEASE_VERSION}" + skip_cleanup: true + acl: private + local_dir: artifacts_dir + on: + all_branches: true + after_deploy: + - echo "Finished release and deployed to https://s3.console.aws.amazon.com/s3/buckets/eu.dl.alfresco.com/release/enterprise/RM/${RELEASE_VERSION}" diff --git a/.travis.release-stage.yml b/.travis.release-stage.yml new file mode 100644 index 0000000000..acdf93a64a --- /dev/null +++ b/.travis.release-stage.yml @@ -0,0 +1,52 @@ +jobs: + include: + - name: "Community Release and Publish to S3 Staging Bucket" + stage: Release + if: commit_message =~ /\[community release .*\]/ + before_script: + - source scripts/set-release-variables.sh + - bash scripts/check-existing-tags.sh "alfresco/alfresco-governance-share-community" + script: + - bash scripts/release.sh "community" + before_deploy: + - bash scripts/zip-artifacts-staging.sh "community" + - bash scripts/pushDockerDigestTag.sh -i quay.io/alfresco/alfresco-governance-repository-community -i quay.io/alfresco/alfresco-governance-share-community -r quay.io -t ${RELEASE_VERSION} + deploy: + - provider: s3 + access_key_id: ${STAGING_AWS_ACCESS_KEY} + secret_access_key: ${STAGING_AWS_SECRET_KEY} + region: "eu-west-1" + bucket: "alfresco-artefacts-staging" + upload_dir: "community/RM/${RELEASE_VERSION}" + skip_cleanup: true + acl: private + local_dir: artifacts_dir + on: + all_branches: true + after_deploy: + - echo "Finished release and deployed to https://s3.console.aws.amazon.com/s3/buckets/alfresco-artefacts-staging/community/RM/${RELEASE_VERSION}" + + - name: "Enterprise Release and Publish to S3 Staging Bucket" + stage: Release + if: commit_message =~ /\[enterprise release .*\]/ + before_script: + - source scripts/set-release-variables.sh + script: + - bash scripts/release.sh "enterprise" + before_deploy: + - bash scripts/zip-artifacts-staging.sh "enterprise" + - bash scripts/pushDockerDigestTag.sh -i quay.io/alfresco/alfresco-governance-repository-enterprise -i quay.io/alfresco/alfresco-governance-share-enterprise -r quay.io -t ${RELEASE_VERSION} + deploy: + - provider: s3 + access_key_id: ${STAGING_AWS_ACCESS_KEY} + secret_access_key: ${STAGING_AWS_SECRET_KEY} + region: "eu-west-1" + bucket: "alfresco-artefacts-staging" + upload_dir: "enterprise/RM/${RELEASE_VERSION}" + skip_cleanup: true + acl: private + local_dir: artifacts_dir + on: + all_branches: true + after_deploy: + - echo "Finished release and deployed to https://s3.console.aws.amazon.com/s3/buckets/alfresco-artefacts-staging/enterprise/RM/${RELEASE_VERSION}" diff --git a/.travis.tests-stage.yml b/.travis.tests-stage.yml new file mode 100644 index 0000000000..b893fb6332 --- /dev/null +++ b/.travis.tests-stage.yml @@ -0,0 +1,140 @@ +jobs: + include: + - name: "Community Integrations Tests on MySQL" + stage: Tests + script: + - echo "Community Integrations Tests on MySQL" + - name: "Enterprise Integrations Tests on MySQL" + stage: Tests + script: + - echo "Enterprise Integrations Tests on MySQL" + + - name: "Community Rest API Tests" + stage: Tests + install: + - | + if [[ ${TRAVIS_BRANCH} == *community* ]]; then + travis_retry travis_wait 90 mvn -B -q install $MVN_SKIP -PbuildDockerImage -pl :alfresco-governance-services-community-repo -am + else + travis_retry travis_wait 90 mvn -B -q install $MVN_SKIP -PbuildDockerImage -pl :alfresco-governance-services-enterprise-repo -am + fi + - travis_retry travis_wait 30 mvn -B -U -q clean install ${MVN_SKIP} -pl :alfresco-governance-services-automation-community-rest-api -am + before_script: + - | + if [[ ${TRAVIS_BRANCH} == *community* ]]; then + bash scripts/startAlfresco.sh $COMMUNITY_REPO_PATH + else + bash scripts/startAlfresco.sh $ENTERPRISE_REPO_PATH + fi + - bash scripts/waitForAlfrescoToStart.sh + script: mvn -B test -pl :alfresco-governance-services-automation-community-rest-api -Dskip.automationtests=false + after_failure: docker ps -a | grep '_alfresco_1' | awk '{print $1}' | xargs docker logs | tail -5000 + + - name: "Enterprise Rest API Tests" + stage: Tests + install: + - travis_retry travis_wait 90 mvn -B -q install $MVN_SKIP -PbuildDockerImage -pl :alfresco-governance-services-enterprise-repo -am + - travis_retry travis_wait 30 mvn -B -U -q clean install ${MVN_SKIP} -pl :alfresco-governance-services-automation-enterprise-rest-api -am + before_script: + - bash scripts/startAlfresco.sh $ENTERPRISE_REPO_PATH + - bash scripts/waitForAlfrescoToStart.sh + - bash scripts/dockerLimitMemory.sh + - sudo free -m -t + script: + - mvn -B test -pl :alfresco-governance-services-automation-enterprise-rest-api -Dskip.automationtests=false + after_failure: docker ps -a | grep '_alfresco_1' | awk '{print $1}' | xargs docker logs | tail -5000 + + - &community_shared_UI_configuration + name: "Community Smoke UI Tests for Records" + stage: Tests + addons: + firefox: "43.0.1" + artifacts: + paths: + - ./rm-automation/rm-automation-ui/target/surefire-reports + - ./rm-automation/rm-automation-ui/target/reports + - ./rm-automation/rm-automation-ui/target/screenshots + target_paths: $TRAVIS_BUILD_NUMBER + before_install: + - | + if [[ ${TRAVIS_COMMIT_MESSAGE} == *"[tests on community]"* ]]; then + travis_retry travis_wait 90 mvn -B -q install $MVN_SKIP -PbuildDockerImage -pl :alfresco-governance-services-community-repo,:alfresco-governance-services-community-share -am + else + travis_retry travis_wait 90 mvn -B -q install $MVN_SKIP -PbuildDockerImage -pl :alfresco-governance-services-enterprise-repo,:alfresco-governance-services-enterprise-share -am + fi + - travis_retry travis_wait 30 mvn -B -U -q clean install ${MVN_SKIP} -pl :alfresco-governance-services-automation-ui -am + install: + - | + if [[ ${TRAVIS_COMMIT_MESSAGE} == *"[tests on community]"* ]]; then + bash scripts/startAlfresco.sh $COMMUNITY_SHARE_PATH + else + bash scripts/startAlfresco.sh $ENTERPRISE_SHARE_PATH + fi + - bash scripts/waitForAlfrescoToStart.sh + - bash scripts/dockerLimitMemory.sh + - sudo free -m -t + script: + - travis_wait 90 xvfb-run --server-args="-screen 0 1920x1080x24" mvn -B test -pl :alfresco-governance-services-automation-ui -Dskip.automationtests=false -DsuiteXmlFile=communitySmokeRecords.xml -Dshare.url=${SHARE_URL} -Dalfresco.url=${ALFRESCO_URL} ${WEBDRIVER_ARGUMENTS} + + - <<: *community_shared_UI_configuration + name: "Community Smoke UI Tests for actions in RM site" + script: + - travis_wait 90 xvfb-run --server-args="-screen 0 1920x1080x24" mvn -B test -pl :alfresco-governance-services-automation-ui -Dskip.automationtests=false -DsuiteXmlFile=communitySmokeRMSite.xml -Dshare.url=${SHARE_URL} -Dalfresco.url=${ALFRESCO_URL} ${WEBDRIVER_ARGUMENTS} + + - <<: *community_shared_UI_configuration + name: "Community Level 2 UI Tests" + script: + - travis_wait 90 xvfb-run --server-args="-screen 0 1920x1080x24" mvn -B test -pl :alfresco-governance-services-automation-ui -Dskip.automationtests=false -DsuiteXmlFile=communityLevel2Tests.xml -Dshare.url=${SHARE_URL} -Dalfresco.url=${ALFRESCO_URL} ${WEBDRIVER_ARGUMENTS} + + - &enterprise_shared_UI_configuration + name: "Enterprise Level 2 UI Tests" + stage: Tests + if: commit_message !~ /\[tests on community\]/ + addons: + firefox: "43.0.1" + artifacts: + paths: + - ./rm-automation/rm-automation-ui/target/surefire-reports + - ./rm-automation/rm-automation-ui/target/reports + - ./rm-automation/rm-automation-ui/target/screenshots + target_paths: $TRAVIS_BUILD_NUMBER + before_install: + - travis_retry travis_wait 90 mvn -B -q install $MVN_SKIP -PbuildDockerImage -pl :alfresco-governance-services-enterprise-repo,:alfresco-governance-services-enterprise-share -am + - travis_retry travis_wait 30 mvn -B -U -q clean install ${MVN_SKIP} -pl :alfresco-governance-services-automation-ui -am + install: + - bash scripts/startAlfresco.sh $ENTERPRISE_SHARE_PATH + - bash scripts/waitForAlfrescoToStart.sh + - bash scripts/dockerLimitMemory.sh + - sudo free -m -t + script: + - travis_wait 90 xvfb-run --server-args="-screen 0 1920x1080x24" mvn -B test -pl :alfresco-governance-services-automation-ui -Dskip.automationtests=false -DsuiteXmlFile=enterpriseLevel2Tests.xml -Dshare.url=${SHARE_URL} -Dalfresco.url=${ALFRESCO_URL} ${WEBDRIVER_ARGUMENTS} + + - <<: *enterprise_shared_UI_configuration + name: "Enterprise Classification Level 2 UI Tests" + script: + - travis_wait 90 xvfb-run --server-args="-screen 0 1920x1080x24" mvn -B test -pl :alfresco-governance-services-automation-ui -Dskip.automationtests=false -DsuiteXmlFile=enterpriseLevel2ClassificationTests.xml -Dshare.url=${SHARE_URL} -Dalfresco.url=${ALFRESCO_URL} ${WEBDRIVER_ARGUMENTS} + + - <<: *enterprise_shared_UI_configuration + name: "Enterprise Records Classification UI Tests" + script: + - travis_wait 90 xvfb-run --server-args="-screen 0 1920x1080x24" mvn -B test -pl :alfresco-governance-services-automation-ui -Dskip.automationtests=false -DsuiteXmlFile=enterpriseRecordsClassificationTests.xml -Dshare.url=${SHARE_URL} -Dalfresco.url=${ALFRESCO_URL} ${WEBDRIVER_ARGUMENTS} + + - <<: *enterprise_shared_UI_configuration + name: "Enterprise Folders Classification UI Tests" + script: + - travis_wait 90 xvfb-run --server-args="-screen 0 1920x1080x24" mvn -B test -pl :alfresco-governance-services-automation-ui -Dskip.automationtests=false -DsuiteXmlFile=enterpriseFoldersClassificationTests.xml -Dshare.url=${SHARE_URL} -Dalfresco.url=${ALFRESCO_URL} ${WEBDRIVER_ARGUMENTS} + + - <<: *enterprise_shared_UI_configuration + name: "Enterprise Documents Classification UI Tests" + script: + - travis_wait 90 xvfb-run --server-args="-screen 0 1920x1080x24" mvn -B test -pl :alfresco-governance-services-automation-ui -Dskip.automationtests=false -DsuiteXmlFile=enterpriseDocumentsClassificationTests.xml -Dshare.url=${SHARE_URL} -Dalfresco.url=${ALFRESCO_URL} ${WEBDRIVER_ARGUMENTS} + + - <<: *enterprise_shared_UI_configuration + name: "Enterprise Security Marks UI Tests" + script: + - travis_wait 90 xvfb-run --server-args="-screen 0 1920x1080x24" mvn -B test -pl :alfresco-governance-services-automation-ui -Dskip.automationtests=false -DsuiteXmlFile=enterpriseSecurityMarksTests.xml -Dshare.url=${SHARE_URL} -Dalfresco.url=${ALFRESCO_URL} ${WEBDRIVER_ARGUMENTS} + + - <<: *enterprise_shared_UI_configuration + name: "Enterprise Console, Guides and Configuration UI Tests" + script: + - travis_wait 90 xvfb-run --server-args="-screen 0 1920x1080x24" mvn -B test -pl :alfresco-governance-services-automation-ui -Dskip.automationtests=false -DsuiteXmlFile=enterpriseConsoleGuidesAndConfiguration.xml -Dshare.url=${SHARE_URL} -Dalfresco.url=${ALFRESCO_URL} ${WEBDRIVER_ARGUMENTS} diff --git a/.travis.yml b/.travis.yml index a102bf4757..ca9f205318 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,8 @@ import: - - source: travis-env-vars.yml + - source: .travis.env-vars.yml + - source: .travis.tests-stage.yml + - source: .travis.release-stage.yml + - source: .travis.publish-stage.yml dist: xenial sudo: required language: java @@ -34,7 +37,9 @@ stages: if: commit_message !~ /\[skip tests\]/ - name: Security Scans - name: Release + if: fork = false AND (branch = master OR branch =~ /release\/.*/) AND type != pull_request AND commit_message =~ /\[(community|enterprise) release .*\]/ - name: Publish + if: fork = false AND (branch = master OR branch =~ /release\/.*/) AND type != pull_request AND commit_message =~ /\[publish\]/ jobs: include: @@ -60,66 +65,6 @@ jobs: script: - travis_retry travis_wait 35 mvn -B -q ${MAVEN_PHASE} -Dskip.integrationtests=false -f rm-benchmark/pom.xml -Dorg.slf4j.simpleLogger.log.org.apache.maven.cli.transfer.Slf4jMavenTransferListener=warn - - name: "Community Integrations Tests on MySQL" - stage: Tests - script: - - echo "Community Integrations Tests on MySQL" - - name: "Enterprise Integrations Tests on MySQL" - stage: Tests - script: - - echo "Enterprise Integrations Tests on MySQL" - - name: "Community Rest API Tests" - stage: Tests - install: - - | - if [[ ${TRAVIS_BRANCH} == *community* ]]; then - travis_retry travis_wait 90 mvn -B -q install $MVN_SKIP -PbuildDockerImage -pl :alfresco-governance-services-community-repo -am - else - travis_retry travis_wait 90 mvn -B -q install $MVN_SKIP -PbuildDockerImage -pl :alfresco-governance-services-enterprise-repo -am - fi - - travis_retry travis_wait 30 mvn -B -U -q clean install ${MVN_SKIP} -pl :alfresco-governance-services-automation-community-rest-api -am - before_script: - - | - if [[ ${TRAVIS_BRANCH} == *community* ]]; then - bash scripts/startAlfresco.sh $COMMUNITY_REPO_PATH - else - bash scripts/startAlfresco.sh $ENTERPRISE_REPO_PATH - fi - - bash scripts/waitForAlfrescoToStart.sh - script: mvn -B test -pl :alfresco-governance-services-automation-community-rest-api -Dskip.automationtests=false - after_failure: docker ps -a | grep '_alfresco_1' | awk '{print $1}' | xargs docker logs | tail -5000 - - name: "Enterprise Rest API Tests" - stage: Tests - install: - - travis_retry travis_wait 90 mvn -B -q install $MVN_SKIP -PbuildDockerImage -pl :alfresco-governance-services-enterprise-repo -am - - travis_retry travis_wait 30 mvn -B -U -q clean install ${MVN_SKIP} -pl :alfresco-governance-services-automation-enterprise-rest-api -am - before_script: - - bash scripts/startAlfresco.sh $ENTERPRISE_REPO_PATH - - bash scripts/waitForAlfrescoToStart.sh - - bash scripts/dockerLimitMemory.sh - - sudo free -m -t - script: - - mvn -B test -pl :alfresco-governance-services-automation-enterprise-rest-api -Dskip.automationtests=false - after_failure: docker ps -a | grep '_alfresco_1' | awk '{print $1}' | xargs docker logs | tail -5000 - - name: "Community UI Tests ..." - stage: Tests - before_install: - - travis_retry travis_wait 90 mvn -B -q install $MVN_SKIP -PbuildDockerImage -pl :alfresco-governance-services-community-repo,:alfresco-governance-services-community-share -am - install: - - bash scripts/startAlfresco.sh $COMMUNITY_SHARE_PATH - - bash scripts/waitForAlfrescoToStart.sh - script: - - echo "Community UI Tests ..." - - name: "Enterprise UI Tests ..." - stage: Tests - before_install: - - travis_retry travis_wait 90 mvn -B -q install $MVN_SKIP -PbuildDockerImage -pl :alfresco-governance-services-enterprise-repo,:alfresco-governance-services-enterprise-share -am - install: - - bash scripts/startAlfresco.sh $ENTERPRISE_SHARE_PATH - - bash scripts/waitForAlfrescoToStart.sh - script: - - echo "Enterprise UI Tests ..." - - name: "Source Clear Scan (SCA)" stage: Security Scans script: @@ -128,17 +73,3 @@ jobs: stage: Security Scans script: - echo "Static Analysis (SAST)" - - - name: "Community Release" - stage: Release - script: - - echo "Community Release" - - name: "Enterprise Release" - stage: Release - script: - - echo "Enterprise Release" - - - name: "Copy to S3 Release Bucket" - stage: Publish - script: - - echo "Copy to S3 Release Bucket" diff --git a/pom.xml b/pom.xml index 765af0fccd..27d0090e3e 100644 --- a/pom.xml +++ b/pom.xml @@ -15,9 +15,9 @@ - scm:git:ssh://git@github.com/Alfresco/governance-services.git - scm:git:ssh://git@github.com/Alfresco/governance-services.git - scm:git:ssh://git@github.com/Alfresco/governance-services.git + scm:git:https://github.com/Alfresco/governance-services.git + scm:git:https://github.com/Alfresco/governance-services.git + https://github.com/Alfresco/governance-services HEAD diff --git a/scripts/check-existing-tags.sh b/scripts/check-existing-tags.sh new file mode 100644 index 0000000000..dee1edabc3 --- /dev/null +++ b/scripts/check-existing-tags.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +alfresco_docker_image=$1 +# Verify release tags +get_tags="$(curl https://hub.docker.com/r/$alfresco_docker_image/tags/ | grep -o '\"result\".*\"]')" +arrayTags=($get_tags) + +echo "Existing Tags: $get_tags" + +for tag in "${arrayTags[@]}" +do + if [[ $tag = ${RELEASE_VERSION} ]]; then + echo "Tag ${RELEASE_VERSION} already pushed, release process will interrupt." + exit 0 + fi +done + +echo "The ${RELEASE_VERSION} tag was not found" diff --git a/scripts/dockerLimitMemory.sh b/scripts/dockerLimitMemory.sh index cdb1dbdd5d..713e4ed294 100644 --- a/scripts/dockerLimitMemory.sh +++ b/scripts/dockerLimitMemory.sh @@ -6,7 +6,7 @@ docker stats --no-stream #limit memory used by each container docker update --memory=2Gb --memory-swap -1 --restart on-failure $(docker ps -a | grep '_alfresco_' | awk '{print $1}') -docker update --memory=1200Mb --memory-swap -1 --restart on-failure $(docker ps -a | grep '_search_' | awk '{print $1}') +docker update --memory=1Gb --memory-swap -1 --restart on-failure $(docker ps -a | grep '_search_' | awk '{print $1}') #docker update --memory=1Gb --memory-swap -1 $(docker ps -a | grep '_zeppelin_' | awk '{print $1}') docker update --memory=512Mb --memory-swap -1 --restart on-failure $(docker ps -a | grep '_postgres_' | awk '{print $1}') docker update --memory=512Mb --memory-swap -1 --restart on-failure $(docker ps -a | grep '_transform-router_' | awk '{print $1}') @@ -17,6 +17,11 @@ docker update --memory=512Mb --memory-swap -1 --restart on-failure $(docker ps - docker update --memory=512Mb --memory-swap -1 --restart on-failure $(docker ps -a | grep '_libreoffice_' | awk '{print $1}') docker update --memory=512Mb --memory-swap -1 --restart on-failure $(docker ps -a | grep '_activemq_' | awk '{print $1}') +shareContainerId=$(docker ps -a | grep '_share_' | awk '{print $1}') +if [ -n "$shareContainerId" ]; then + docker update --memory=1Gb --memory-swap -1 --restart on-failure $shareContainerId +fi + #stop not needed container docker stop $(docker ps -a | grep '_zeppelin_' | awk '{print $1}') diff --git a/scripts/release.sh b/scripts/release.sh new file mode 100755 index 0000000000..4435342437 --- /dev/null +++ b/scripts/release.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +set -e + +# Use full history for release +git checkout -B "${TRAVIS_BRANCH}" + +git config user.email "build@alfresco.com" + +release_type=$1 +echo Release type: "$release_type" + +if [ -z $release_type ]; then + echo "Please provide a release type." + exit 1 +elif [ $release_type != "community" -a $release_type != "enterprise" ]; then + echo "The provided release type is not valid." + exit 1 +fi + +if [ -z ${RELEASE_VERSION} ] || [ -z ${DEVELOPMENT_VERSION} ]; then + echo "Please provide a Release and Development verison" + exit 1 +fi + +# Docker Logins +echo "${DOCKERHUB_PASSWORD}" | docker login -u="${DOCKERHUB_USERNAME}" --password-stdin +echo "${QUAY_PASSWORD}" | docker login -u="${QUAY_USERNAME}" --password-stdin quay.io + +# Check if it's a hotfix version by counting the number of dots in the version number. +if [ $(echo "${RELEASE_VERSION}" | grep -o "\." | wc -l) == 3 ] && [ ${release_type} == "enterprise" ]; +then + deployment_repository="hotfix-release" +else + deployment_repository="${release_type}-release" +fi + +mvn --batch-mode \ + -Dusername="${GIT_USERNAME}" \ + -Dpassword="${GIT_PASSWORD}" \ + -DreleaseVersion=${RELEASE_VERSION} \ + -DdevelopmentVersion=${DEVELOPMENT_VERSION} \ + -DscmCommentPrefix="[maven-release-plugin][skip ci] " \ + "-Darguments=-DskipTests -D${release_type} -P${deployment_repository},release-${release_type}" \ + release:clean release:prepare release:perform \ No newline at end of file diff --git a/scripts/set-release-variables.sh b/scripts/set-release-variables.sh new file mode 100755 index 0000000000..35013b7053 --- /dev/null +++ b/scripts/set-release-variables.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +set -e + +echo "Travis commit message: $TRAVIS_COMMIT_MESSAGE" +release_message=$(echo $TRAVIS_COMMIT_MESSAGE | grep -Po '(\[(community|enterprise)\srelease\s(\d\.)+(\d|[a-z])(-[A-Z]\d){0,1}\s(\d\.)+\d-SNAPSHOT\])') + +if [ ! -n "$release_message" ]; then + echo "The commit message is in the wrong format or it does not contain all the required properties." + exit 1 +fi + +export RELEASE_VERSION=$(echo $release_message | grep -Po '(\d\.)+(\d|[a-z])(-[A-Z]\d){0,1}' | head -1) +export DEVELOPMENT_VERSION=$(echo $release_message | grep -Po '(\d\.)+\d-SNAPSHOT') + +echo "Release version is set to $RELEASE_VERSION" +echo "Development version is set to $DEVELOPMENT_VERSION" \ No newline at end of file diff --git a/scripts/zip-artifacts-release.sh b/scripts/zip-artifacts-release.sh new file mode 100644 index 0000000000..b62faef5f0 --- /dev/null +++ b/scripts/zip-artifacts-release.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +set -e + +release_type=$1 + +mkdir "artifacts_dir" + +mvn -B org.apache.maven.plugins:maven-dependency-plugin:3.1.1:copy \ + -Dartifact=org.alfresco:alfresco-governance-services-${release_type}-repo:${RELEASE_VERSION}:amp \ + -DoutputDirectory=artifacts_dir + +mvn -B org.apache.maven.plugins:maven-dependency-plugin:3.1.1:copy \ + -Dartifact=org.alfresco:alfresco-governance-services-${release_type}-share:${RELEASE_VERSION}:amp \ + -DoutputDirectory=artifacts_dir + +mvn -B org.apache.maven.plugins:maven-dependency-plugin:3.1.1:copy \ + -Dartifact=org.alfresco:alfresco-governance-services-${release_type}-rest-api-explorer:${RELEASE_VERSION}:war \ + -DoutputDirectory=artifacts_dir + +cd artifacts_dir +zip alfresco-governance-services-${release_type}-${RELEASE_VERSION}.zip * + +# rm *.amp *.war -f +ls \ No newline at end of file diff --git a/scripts/zip-artifacts-staging.sh b/scripts/zip-artifacts-staging.sh new file mode 100755 index 0000000000..439d9619c7 --- /dev/null +++ b/scripts/zip-artifacts-staging.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +set -e + +release_type=$1 + +mkdir "artifacts_dir" +cp rm-${release_type}/rm-${release_type}-repo/target/alfresco-governance-services-${release_type}-repo-*.amp artifacts_dir +cp rm-${release_type}/rm-${release_type}-share/target/alfresco-governance-services-${release_type}-share-*.amp artifacts_dir +cp rm-${release_type}/rm-${release_type}-rest-api-explorer/target/alfresco-governance-services-${release_type}-rest-api-explorer-*.war artifacts_dir + +cd artifacts_dir +zip alfresco-governance-services-${release_type}-${RELEASE_VERSION}.zip * + +# rm *.amp *.war -f +ls \ No newline at end of file From 803c4a13e01825d31be7eaaf57847f755f38d8ed Mon Sep 17 00:00:00 2001 From: Claudia Agache Date: Tue, 3 Nov 2020 15:00:12 +0200 Subject: [PATCH 02/12] limit memory for shared-file-store container to 300Mb --- scripts/dockerLimitMemory.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/dockerLimitMemory.sh b/scripts/dockerLimitMemory.sh index 713e4ed294..09f0649b46 100644 --- a/scripts/dockerLimitMemory.sh +++ b/scripts/dockerLimitMemory.sh @@ -12,7 +12,7 @@ docker update --memory=512Mb --memory-swap -1 --restart on-failure $(docker ps - docker update --memory=512Mb --memory-swap -1 --restart on-failure $(docker ps -a | grep '_transform-router_' | awk '{print $1}') docker update --memory=512Mb --memory-swap -1 --restart on-failure $(docker ps -a | grep '_imagemagick_' | awk '{print $1}') docker update --memory=512Mb --memory-swap -1 --restart on-failure $(docker ps -a | grep '_alfresco-pdf-renderer_' | awk '{print $1}') -docker update --memory=512Mb --memory-swap -1 --restart on-failure $(docker ps -a | grep '_shared-file-store_' | awk '{print $1}') +docker update --memory=300Mb --memory-swap -1 --restart on-failure $(docker ps -a | grep '_shared-file-store_' | awk '{print $1}') docker update --memory=512Mb --memory-swap -1 --restart on-failure $(docker ps -a | grep '_tika_' | awk '{print $1}') docker update --memory=512Mb --memory-swap -1 --restart on-failure $(docker ps -a | grep '_libreoffice_' | awk '{print $1}') docker update --memory=512Mb --memory-swap -1 --restart on-failure $(docker ps -a | grep '_activemq_' | awk '{print $1}') From ced45dcbccc85101f3e4179ecb9adc0610cfeb0d Mon Sep 17 00:00:00 2001 From: Claudia Agache Date: Tue, 3 Nov 2020 19:36:28 +0200 Subject: [PATCH 03/12] limit memory for alfresco-pdf-renderer container to 300Mb --- scripts/dockerLimitMemory.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/dockerLimitMemory.sh b/scripts/dockerLimitMemory.sh index 09f0649b46..2f9a8c17a4 100644 --- a/scripts/dockerLimitMemory.sh +++ b/scripts/dockerLimitMemory.sh @@ -11,7 +11,7 @@ docker update --memory=1Gb --memory-swap -1 --restart on-failure $(docker ps -a docker update --memory=512Mb --memory-swap -1 --restart on-failure $(docker ps -a | grep '_postgres_' | awk '{print $1}') docker update --memory=512Mb --memory-swap -1 --restart on-failure $(docker ps -a | grep '_transform-router_' | awk '{print $1}') docker update --memory=512Mb --memory-swap -1 --restart on-failure $(docker ps -a | grep '_imagemagick_' | awk '{print $1}') -docker update --memory=512Mb --memory-swap -1 --restart on-failure $(docker ps -a | grep '_alfresco-pdf-renderer_' | awk '{print $1}') +docker update --memory=300Mb --memory-swap -1 --restart on-failure $(docker ps -a | grep '_alfresco-pdf-renderer_' | awk '{print $1}') docker update --memory=300Mb --memory-swap -1 --restart on-failure $(docker ps -a | grep '_shared-file-store_' | awk '{print $1}') docker update --memory=512Mb --memory-swap -1 --restart on-failure $(docker ps -a | grep '_tika_' | awk '{print $1}') docker update --memory=512Mb --memory-swap -1 --restart on-failure $(docker ps -a | grep '_libreoffice_' | awk '{print $1}') From e7965cd213951c18f23fec4e0ffdac2345da4c6a Mon Sep 17 00:00:00 2001 From: Claudia Agache Date: Wed, 4 Nov 2020 07:45:55 +0200 Subject: [PATCH 04/12] limit memory for transformers containers to 400Mb --- scripts/dockerLimitMemory.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/dockerLimitMemory.sh b/scripts/dockerLimitMemory.sh index 2f9a8c17a4..966f9774d8 100644 --- a/scripts/dockerLimitMemory.sh +++ b/scripts/dockerLimitMemory.sh @@ -10,12 +10,12 @@ docker update --memory=1Gb --memory-swap -1 --restart on-failure $(docker ps -a #docker update --memory=1Gb --memory-swap -1 $(docker ps -a | grep '_zeppelin_' | awk '{print $1}') docker update --memory=512Mb --memory-swap -1 --restart on-failure $(docker ps -a | grep '_postgres_' | awk '{print $1}') docker update --memory=512Mb --memory-swap -1 --restart on-failure $(docker ps -a | grep '_transform-router_' | awk '{print $1}') -docker update --memory=512Mb --memory-swap -1 --restart on-failure $(docker ps -a | grep '_imagemagick_' | awk '{print $1}') -docker update --memory=300Mb --memory-swap -1 --restart on-failure $(docker ps -a | grep '_alfresco-pdf-renderer_' | awk '{print $1}') +docker update --memory=400Mb --memory-swap -1 --restart on-failure $(docker ps -a | grep '_imagemagick_' | awk '{print $1}') +docker update --memory=400Mb --memory-swap -1 --restart on-failure $(docker ps -a | grep '_alfresco-pdf-renderer_' | awk '{print $1}') docker update --memory=300Mb --memory-swap -1 --restart on-failure $(docker ps -a | grep '_shared-file-store_' | awk '{print $1}') -docker update --memory=512Mb --memory-swap -1 --restart on-failure $(docker ps -a | grep '_tika_' | awk '{print $1}') -docker update --memory=512Mb --memory-swap -1 --restart on-failure $(docker ps -a | grep '_libreoffice_' | awk '{print $1}') -docker update --memory=512Mb --memory-swap -1 --restart on-failure $(docker ps -a | grep '_activemq_' | awk '{print $1}') +docker update --memory=400Mb --memory-swap -1 --restart on-failure $(docker ps -a | grep '_tika_' | awk '{print $1}') +docker update --memory=400Mb --memory-swap -1 --restart on-failure $(docker ps -a | grep '_libreoffice_' | awk '{print $1}') +docker update --memory=400Mb --memory-swap -1 --restart on-failure $(docker ps -a | grep '_activemq_' | awk '{print $1}') shareContainerId=$(docker ps -a | grep '_share_' | awk '{print $1}') if [ -n "$shareContainerId" ]; then From 9fd242d4e832586f60f0ebcbfbbea30bcf7e4426 Mon Sep 17 00:00:00 2001 From: Claudia Agache Date: Wed, 4 Nov 2020 09:51:11 +0200 Subject: [PATCH 05/12] redirect logs to file and update target path for artifacts --- .travis.tests-stage.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.tests-stage.yml b/.travis.tests-stage.yml index b893fb6332..80406acbc0 100644 --- a/.travis.tests-stage.yml +++ b/.travis.tests-stage.yml @@ -54,7 +54,7 @@ jobs: - ./rm-automation/rm-automation-ui/target/surefire-reports - ./rm-automation/rm-automation-ui/target/reports - ./rm-automation/rm-automation-ui/target/screenshots - target_paths: $TRAVIS_BUILD_NUMBER + target_paths: $TRAVIS_BUILD_NUMBER/$TRAVIS_JOB_NUMBER before_install: - | if [[ ${TRAVIS_COMMIT_MESSAGE} == *"[tests on community]"* ]]; then @@ -97,7 +97,7 @@ jobs: - ./rm-automation/rm-automation-ui/target/surefire-reports - ./rm-automation/rm-automation-ui/target/reports - ./rm-automation/rm-automation-ui/target/screenshots - target_paths: $TRAVIS_BUILD_NUMBER + target_paths: $TRAVIS_BUILD_NUMBER/$TRAVIS_JOB_NUMBER before_install: - travis_retry travis_wait 90 mvn -B -q install $MVN_SKIP -PbuildDockerImage -pl :alfresco-governance-services-enterprise-repo,:alfresco-governance-services-enterprise-share -am - travis_retry travis_wait 30 mvn -B -U -q clean install ${MVN_SKIP} -pl :alfresco-governance-services-automation-ui -am From d6cb66df5908eda8b78131d02622518de410056b Mon Sep 17 00:00:00 2001 From: Claudia Agache Date: Wed, 4 Nov 2020 13:05:31 +0200 Subject: [PATCH 06/12] Revert "limit memory for transformers containers to 400Mb" This reverts commit e7965cd2 --- scripts/dockerLimitMemory.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/dockerLimitMemory.sh b/scripts/dockerLimitMemory.sh index 966f9774d8..2f9a8c17a4 100644 --- a/scripts/dockerLimitMemory.sh +++ b/scripts/dockerLimitMemory.sh @@ -10,12 +10,12 @@ docker update --memory=1Gb --memory-swap -1 --restart on-failure $(docker ps -a #docker update --memory=1Gb --memory-swap -1 $(docker ps -a | grep '_zeppelin_' | awk '{print $1}') docker update --memory=512Mb --memory-swap -1 --restart on-failure $(docker ps -a | grep '_postgres_' | awk '{print $1}') docker update --memory=512Mb --memory-swap -1 --restart on-failure $(docker ps -a | grep '_transform-router_' | awk '{print $1}') -docker update --memory=400Mb --memory-swap -1 --restart on-failure $(docker ps -a | grep '_imagemagick_' | awk '{print $1}') -docker update --memory=400Mb --memory-swap -1 --restart on-failure $(docker ps -a | grep '_alfresco-pdf-renderer_' | awk '{print $1}') +docker update --memory=512Mb --memory-swap -1 --restart on-failure $(docker ps -a | grep '_imagemagick_' | awk '{print $1}') +docker update --memory=300Mb --memory-swap -1 --restart on-failure $(docker ps -a | grep '_alfresco-pdf-renderer_' | awk '{print $1}') docker update --memory=300Mb --memory-swap -1 --restart on-failure $(docker ps -a | grep '_shared-file-store_' | awk '{print $1}') -docker update --memory=400Mb --memory-swap -1 --restart on-failure $(docker ps -a | grep '_tika_' | awk '{print $1}') -docker update --memory=400Mb --memory-swap -1 --restart on-failure $(docker ps -a | grep '_libreoffice_' | awk '{print $1}') -docker update --memory=400Mb --memory-swap -1 --restart on-failure $(docker ps -a | grep '_activemq_' | awk '{print $1}') +docker update --memory=512Mb --memory-swap -1 --restart on-failure $(docker ps -a | grep '_tika_' | awk '{print $1}') +docker update --memory=512Mb --memory-swap -1 --restart on-failure $(docker ps -a | grep '_libreoffice_' | awk '{print $1}') +docker update --memory=512Mb --memory-swap -1 --restart on-failure $(docker ps -a | grep '_activemq_' | awk '{print $1}') shareContainerId=$(docker ps -a | grep '_share_' | awk '{print $1}') if [ -n "$shareContainerId" ]; then From 2364ba9c9f0872e4099d63df5199a06dc6fbb90d Mon Sep 17 00:00:00 2001 From: Claudia Agache Date: Wed, 4 Nov 2020 13:05:39 +0200 Subject: [PATCH 07/12] Revert "limit memory for alfresco-pdf-renderer container to 300Mb" This reverts commit ced45dcb --- scripts/dockerLimitMemory.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/dockerLimitMemory.sh b/scripts/dockerLimitMemory.sh index 2f9a8c17a4..09f0649b46 100644 --- a/scripts/dockerLimitMemory.sh +++ b/scripts/dockerLimitMemory.sh @@ -11,7 +11,7 @@ docker update --memory=1Gb --memory-swap -1 --restart on-failure $(docker ps -a docker update --memory=512Mb --memory-swap -1 --restart on-failure $(docker ps -a | grep '_postgres_' | awk '{print $1}') docker update --memory=512Mb --memory-swap -1 --restart on-failure $(docker ps -a | grep '_transform-router_' | awk '{print $1}') docker update --memory=512Mb --memory-swap -1 --restart on-failure $(docker ps -a | grep '_imagemagick_' | awk '{print $1}') -docker update --memory=300Mb --memory-swap -1 --restart on-failure $(docker ps -a | grep '_alfresco-pdf-renderer_' | awk '{print $1}') +docker update --memory=512Mb --memory-swap -1 --restart on-failure $(docker ps -a | grep '_alfresco-pdf-renderer_' | awk '{print $1}') docker update --memory=300Mb --memory-swap -1 --restart on-failure $(docker ps -a | grep '_shared-file-store_' | awk '{print $1}') docker update --memory=512Mb --memory-swap -1 --restart on-failure $(docker ps -a | grep '_tika_' | awk '{print $1}') docker update --memory=512Mb --memory-swap -1 --restart on-failure $(docker ps -a | grep '_libreoffice_' | awk '{print $1}') From de4c5303f6d75ae4c808c43e60606b512a18f5b9 Mon Sep 17 00:00:00 2001 From: Claudia Agache Date: Wed, 4 Nov 2020 13:29:19 +0200 Subject: [PATCH 08/12] run UI tests with legacy transformers --- .travis.tests-stage.yml | 12 ++++++------ .travis.yml | 2 ++ scripts/dockerLimitMemory.sh | 22 ++++++++++++++-------- scripts/release.sh | 6 +----- scripts/startAlfresco.sh | 3 +-- 5 files changed, 24 insertions(+), 21 deletions(-) diff --git a/.travis.tests-stage.yml b/.travis.tests-stage.yml index 80406acbc0..f8f2f317a6 100644 --- a/.travis.tests-stage.yml +++ b/.travis.tests-stage.yml @@ -22,9 +22,9 @@ jobs: before_script: - | if [[ ${TRAVIS_BRANCH} == *community* ]]; then - bash scripts/startAlfresco.sh $COMMUNITY_REPO_PATH + bash scripts/startAlfresco.sh $COMMUNITY_REPO_PATH .env else - bash scripts/startAlfresco.sh $ENTERPRISE_REPO_PATH + bash scripts/startAlfresco.sh $ENTERPRISE_REPO_PATH .env fi - bash scripts/waitForAlfrescoToStart.sh script: mvn -B test -pl :alfresco-governance-services-automation-community-rest-api -Dskip.automationtests=false @@ -36,7 +36,7 @@ jobs: - travis_retry travis_wait 90 mvn -B -q install $MVN_SKIP -PbuildDockerImage -pl :alfresco-governance-services-enterprise-repo -am - travis_retry travis_wait 30 mvn -B -U -q clean install ${MVN_SKIP} -pl :alfresco-governance-services-automation-enterprise-rest-api -am before_script: - - bash scripts/startAlfresco.sh $ENTERPRISE_REPO_PATH + - bash scripts/startAlfresco.sh $ENTERPRISE_REPO_PATH .env - bash scripts/waitForAlfrescoToStart.sh - bash scripts/dockerLimitMemory.sh - sudo free -m -t @@ -66,9 +66,9 @@ jobs: install: - | if [[ ${TRAVIS_COMMIT_MESSAGE} == *"[tests on community]"* ]]; then - bash scripts/startAlfresco.sh $COMMUNITY_SHARE_PATH + bash scripts/startAlfresco.sh $COMMUNITY_SHARE_PATH .env else - bash scripts/startAlfresco.sh $ENTERPRISE_SHARE_PATH + bash scripts/startAlfresco.sh $ENTERPRISE_SHARE_PATH .env.ci fi - bash scripts/waitForAlfrescoToStart.sh - bash scripts/dockerLimitMemory.sh @@ -102,7 +102,7 @@ jobs: - travis_retry travis_wait 90 mvn -B -q install $MVN_SKIP -PbuildDockerImage -pl :alfresco-governance-services-enterprise-repo,:alfresco-governance-services-enterprise-share -am - travis_retry travis_wait 30 mvn -B -U -q clean install ${MVN_SKIP} -pl :alfresco-governance-services-automation-ui -am install: - - bash scripts/startAlfresco.sh $ENTERPRISE_SHARE_PATH + - bash scripts/startAlfresco.sh $ENTERPRISE_SHARE_PATH .env.ci - bash scripts/waitForAlfrescoToStart.sh - bash scripts/dockerLimitMemory.sh - sudo free -m -t diff --git a/.travis.yml b/.travis.yml index ca9f205318..96bbcf2161 100644 --- a/.travis.yml +++ b/.travis.yml @@ -29,6 +29,8 @@ before_cache: before_install: - "cp .travis.settings.xml $HOME/.m2/settings.xml" + - echo "${DOCKERHUB_PASSWORD}" | docker login -u="${DOCKERHUB_USERNAME}" --password-stdin + - echo "${QUAY_PASSWORD}" | docker login -u="${QUAY_USERNAME}" --password-stdin quay.io install: skip stages: diff --git a/scripts/dockerLimitMemory.sh b/scripts/dockerLimitMemory.sh index 09f0649b46..b0a1ce64dc 100644 --- a/scripts/dockerLimitMemory.sh +++ b/scripts/dockerLimitMemory.sh @@ -7,19 +7,25 @@ docker stats --no-stream #limit memory used by each container docker update --memory=2Gb --memory-swap -1 --restart on-failure $(docker ps -a | grep '_alfresco_' | awk '{print $1}') docker update --memory=1Gb --memory-swap -1 --restart on-failure $(docker ps -a | grep '_search_' | awk '{print $1}') -#docker update --memory=1Gb --memory-swap -1 $(docker ps -a | grep '_zeppelin_' | awk '{print $1}') docker update --memory=512Mb --memory-swap -1 --restart on-failure $(docker ps -a | grep '_postgres_' | awk '{print $1}') -docker update --memory=512Mb --memory-swap -1 --restart on-failure $(docker ps -a | grep '_transform-router_' | awk '{print $1}') -docker update --memory=512Mb --memory-swap -1 --restart on-failure $(docker ps -a | grep '_imagemagick_' | awk '{print $1}') -docker update --memory=512Mb --memory-swap -1 --restart on-failure $(docker ps -a | grep '_alfresco-pdf-renderer_' | awk '{print $1}') -docker update --memory=300Mb --memory-swap -1 --restart on-failure $(docker ps -a | grep '_shared-file-store_' | awk '{print $1}') -docker update --memory=512Mb --memory-swap -1 --restart on-failure $(docker ps -a | grep '_tika_' | awk '{print $1}') -docker update --memory=512Mb --memory-swap -1 --restart on-failure $(docker ps -a | grep '_libreoffice_' | awk '{print $1}') docker update --memory=512Mb --memory-swap -1 --restart on-failure $(docker ps -a | grep '_activemq_' | awk '{print $1}') shareContainerId=$(docker ps -a | grep '_share_' | awk '{print $1}') if [ -n "$shareContainerId" ]; then - docker update --memory=1Gb --memory-swap -1 --restart on-failure $shareContainerId + docker update --memory=1Gb --memory-swap -1 --restart on-failure $shareContainerId + docker stop $(docker ps -a | grep '_transform-router_' | awk '{print $1}') + docker stop $(docker ps -a | grep '_imagemagick_' | awk '{print $1}') + docker stop $(docker ps -a | grep '_alfresco-pdf-renderer_' | awk '{print $1}') + docker stop $(docker ps -a | grep '_shared-file-store_' | awk '{print $1}') + docker stop $(docker ps -a | grep '_tika_' | awk '{print $1}') + docker stop $(docker ps -a | grep '_libreoffice_' | awk '{print $1}') + else + docker update --memory=512Mb --memory-swap -1 --restart on-failure $(docker ps -a | grep '_transform-router_' | awk '{print $1}') + docker update --memory=512Mb --memory-swap -1 --restart on-failure $(docker ps -a | grep '_imagemagick_' | awk '{print $1}') + docker update --memory=512Mb --memory-swap -1 --restart on-failure $(docker ps -a | grep '_alfresco-pdf-renderer_' | awk '{print $1}') + docker update --memory=300Mb --memory-swap -1 --restart on-failure $(docker ps -a | grep '_shared-file-store_' | awk '{print $1}') + docker update --memory=512Mb --memory-swap -1 --restart on-failure $(docker ps -a | grep '_tika_' | awk '{print $1}') + docker update --memory=512Mb --memory-swap -1 --restart on-failure $(docker ps -a | grep '_libreoffice_' | awk '{print $1}') fi #stop not needed container diff --git a/scripts/release.sh b/scripts/release.sh index 4435342437..d9a13454ec 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -22,10 +22,6 @@ if [ -z ${RELEASE_VERSION} ] || [ -z ${DEVELOPMENT_VERSION} ]; then exit 1 fi -# Docker Logins -echo "${DOCKERHUB_PASSWORD}" | docker login -u="${DOCKERHUB_USERNAME}" --password-stdin -echo "${QUAY_PASSWORD}" | docker login -u="${QUAY_USERNAME}" --password-stdin quay.io - # Check if it's a hotfix version by counting the number of dots in the version number. if [ $(echo "${RELEASE_VERSION}" | grep -o "\." | wc -l) == 3 ] && [ ${release_type} == "enterprise" ]; then @@ -41,4 +37,4 @@ mvn --batch-mode \ -DdevelopmentVersion=${DEVELOPMENT_VERSION} \ -DscmCommentPrefix="[maven-release-plugin][skip ci] " \ "-Darguments=-DskipTests -D${release_type} -P${deployment_repository},release-${release_type}" \ - release:clean release:prepare release:perform \ No newline at end of file + release:clean release:prepare release:perform diff --git a/scripts/startAlfresco.sh b/scripts/startAlfresco.sh index b161dbde05..31f12d554e 100644 --- a/scripts/startAlfresco.sh +++ b/scripts/startAlfresco.sh @@ -3,5 +3,4 @@ set -ev cd $1 -docker login quay.io -u ${QUAY_USERNAME} -p ${QUAY_PASSWORD} -docker-compose up -d +docker-compose --env-file $2 up -d From a9e022749893241a8fb393cc5788791195706703 Mon Sep 17 00:00:00 2001 From: Claudia Agache Date: Wed, 4 Nov 2020 16:00:22 +0200 Subject: [PATCH 09/12] don't use before_install for tests jobs because it is defined a general one in travis.yml file --- .travis.tests-stage.yml | 26 ++++++++++---------------- 1 file changed, 10 insertions(+), 16 deletions(-) diff --git a/.travis.tests-stage.yml b/.travis.tests-stage.yml index f8f2f317a6..b53966a699 100644 --- a/.travis.tests-stage.yml +++ b/.travis.tests-stage.yml @@ -22,9 +22,9 @@ jobs: before_script: - | if [[ ${TRAVIS_BRANCH} == *community* ]]; then - bash scripts/startAlfresco.sh $COMMUNITY_REPO_PATH .env + bash scripts/startAlfresco.sh $COMMUNITY_REPO_PATH false else - bash scripts/startAlfresco.sh $ENTERPRISE_REPO_PATH .env + bash scripts/startAlfresco.sh $ENTERPRISE_REPO_PATH false fi - bash scripts/waitForAlfrescoToStart.sh script: mvn -B test -pl :alfresco-governance-services-automation-community-rest-api -Dskip.automationtests=false @@ -36,7 +36,7 @@ jobs: - travis_retry travis_wait 90 mvn -B -q install $MVN_SKIP -PbuildDockerImage -pl :alfresco-governance-services-enterprise-repo -am - travis_retry travis_wait 30 mvn -B -U -q clean install ${MVN_SKIP} -pl :alfresco-governance-services-automation-enterprise-rest-api -am before_script: - - bash scripts/startAlfresco.sh $ENTERPRISE_REPO_PATH .env + - bash scripts/startAlfresco.sh $ENTERPRISE_REPO_PATH false - bash scripts/waitForAlfrescoToStart.sh - bash scripts/dockerLimitMemory.sh - sudo free -m -t @@ -55,24 +55,19 @@ jobs: - ./rm-automation/rm-automation-ui/target/reports - ./rm-automation/rm-automation-ui/target/screenshots target_paths: $TRAVIS_BUILD_NUMBER/$TRAVIS_JOB_NUMBER - before_install: - - | - if [[ ${TRAVIS_COMMIT_MESSAGE} == *"[tests on community]"* ]]; then - travis_retry travis_wait 90 mvn -B -q install $MVN_SKIP -PbuildDockerImage -pl :alfresco-governance-services-community-repo,:alfresco-governance-services-community-share -am - else - travis_retry travis_wait 90 mvn -B -q install $MVN_SKIP -PbuildDockerImage -pl :alfresco-governance-services-enterprise-repo,:alfresco-governance-services-enterprise-share -am - fi - - travis_retry travis_wait 30 mvn -B -U -q clean install ${MVN_SKIP} -pl :alfresco-governance-services-automation-ui -am install: - | if [[ ${TRAVIS_COMMIT_MESSAGE} == *"[tests on community]"* ]]; then - bash scripts/startAlfresco.sh $COMMUNITY_SHARE_PATH .env + travis_retry travis_wait 90 mvn -B -q install $MVN_SKIP -PbuildDockerImage -pl :alfresco-governance-services-community-repo,:alfresco-governance-services-community-share -am + bash scripts/startAlfresco.sh $COMMUNITY_SHARE_PATH false else - bash scripts/startAlfresco.sh $ENTERPRISE_SHARE_PATH .env.ci + travis_retry travis_wait 90 mvn -B -q install $MVN_SKIP -PbuildDockerImage -pl :alfresco-governance-services-enterprise-repo,:alfresco-governance-services-enterprise-share -am + bash scripts/startAlfresco.sh $ENTERPRISE_SHARE_PATH true fi - bash scripts/waitForAlfrescoToStart.sh - bash scripts/dockerLimitMemory.sh - sudo free -m -t + - travis_retry travis_wait 30 mvn -B -U -q clean install ${MVN_SKIP} -pl :alfresco-governance-services-automation-ui -am script: - travis_wait 90 xvfb-run --server-args="-screen 0 1920x1080x24" mvn -B test -pl :alfresco-governance-services-automation-ui -Dskip.automationtests=false -DsuiteXmlFile=communitySmokeRecords.xml -Dshare.url=${SHARE_URL} -Dalfresco.url=${ALFRESCO_URL} ${WEBDRIVER_ARGUMENTS} @@ -98,11 +93,10 @@ jobs: - ./rm-automation/rm-automation-ui/target/reports - ./rm-automation/rm-automation-ui/target/screenshots target_paths: $TRAVIS_BUILD_NUMBER/$TRAVIS_JOB_NUMBER - before_install: + install: - travis_retry travis_wait 90 mvn -B -q install $MVN_SKIP -PbuildDockerImage -pl :alfresco-governance-services-enterprise-repo,:alfresco-governance-services-enterprise-share -am - travis_retry travis_wait 30 mvn -B -U -q clean install ${MVN_SKIP} -pl :alfresco-governance-services-automation-ui -am - install: - - bash scripts/startAlfresco.sh $ENTERPRISE_SHARE_PATH .env.ci + - bash scripts/startAlfresco.sh $ENTERPRISE_SHARE_PATH true - bash scripts/waitForAlfrescoToStart.sh - bash scripts/dockerLimitMemory.sh - sudo free -m -t From 9177b40aaa83e49599df89da35dad2f926b5f240 Mon Sep 17 00:00:00 2001 From: Claudia Agache Date: Wed, 4 Nov 2020 18:18:47 +0200 Subject: [PATCH 10/12] set only transform.service.enabled=false --- scripts/dockerLimitMemory.sh | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/scripts/dockerLimitMemory.sh b/scripts/dockerLimitMemory.sh index b0a1ce64dc..a2d454ebc9 100644 --- a/scripts/dockerLimitMemory.sh +++ b/scripts/dockerLimitMemory.sh @@ -9,23 +9,27 @@ docker update --memory=2Gb --memory-swap -1 --restart on-failure $(docker ps -a docker update --memory=1Gb --memory-swap -1 --restart on-failure $(docker ps -a | grep '_search_' | awk '{print $1}') docker update --memory=512Mb --memory-swap -1 --restart on-failure $(docker ps -a | grep '_postgres_' | awk '{print $1}') docker update --memory=512Mb --memory-swap -1 --restart on-failure $(docker ps -a | grep '_activemq_' | awk '{print $1}') +docker update --memory=512Mb --memory-swap -1 --restart on-failure $(docker ps -a | grep '_imagemagick_' | awk '{print $1}') +docker update --memory=512Mb --memory-swap -1 --restart on-failure $(docker ps -a | grep '_alfresco-pdf-renderer_' | awk '{print $1}') +docker update --memory=512Mb --memory-swap -1 --restart on-failure $(docker ps -a | grep '_tika_' | awk '{print $1}') +docker update --memory=512Mb --memory-swap -1 --restart on-failure $(docker ps -a | grep '_libreoffice_' | awk '{print $1}') shareContainerId=$(docker ps -a | grep '_share_' | awk '{print $1}') if [ -n "$shareContainerId" ]; then docker update --memory=1Gb --memory-swap -1 --restart on-failure $shareContainerId docker stop $(docker ps -a | grep '_transform-router_' | awk '{print $1}') - docker stop $(docker ps -a | grep '_imagemagick_' | awk '{print $1}') - docker stop $(docker ps -a | grep '_alfresco-pdf-renderer_' | awk '{print $1}') +# docker stop $(docker ps -a | grep '_imagemagick_' | awk '{print $1}') +# docker stop $(docker ps -a | grep '_alfresco-pdf-renderer_' | awk '{print $1}') docker stop $(docker ps -a | grep '_shared-file-store_' | awk '{print $1}') - docker stop $(docker ps -a | grep '_tika_' | awk '{print $1}') - docker stop $(docker ps -a | grep '_libreoffice_' | awk '{print $1}') +# docker stop $(docker ps -a | grep '_tika_' | awk '{print $1}') +# docker stop $(docker ps -a | grep '_libreoffice_' | awk '{print $1}') else docker update --memory=512Mb --memory-swap -1 --restart on-failure $(docker ps -a | grep '_transform-router_' | awk '{print $1}') - docker update --memory=512Mb --memory-swap -1 --restart on-failure $(docker ps -a | grep '_imagemagick_' | awk '{print $1}') - docker update --memory=512Mb --memory-swap -1 --restart on-failure $(docker ps -a | grep '_alfresco-pdf-renderer_' | awk '{print $1}') +# docker update --memory=512Mb --memory-swap -1 --restart on-failure $(docker ps -a | grep '_imagemagick_' | awk '{print $1}') +# docker update --memory=512Mb --memory-swap -1 --restart on-failure $(docker ps -a | grep '_alfresco-pdf-renderer_' | awk '{print $1}') docker update --memory=300Mb --memory-swap -1 --restart on-failure $(docker ps -a | grep '_shared-file-store_' | awk '{print $1}') - docker update --memory=512Mb --memory-swap -1 --restart on-failure $(docker ps -a | grep '_tika_' | awk '{print $1}') - docker update --memory=512Mb --memory-swap -1 --restart on-failure $(docker ps -a | grep '_libreoffice_' | awk '{print $1}') +# docker update --memory=512Mb --memory-swap -1 --restart on-failure $(docker ps -a | grep '_tika_' | awk '{print $1}') +# docker update --memory=512Mb --memory-swap -1 --restart on-failure $(docker ps -a | grep '_libreoffice_' | awk '{print $1}') fi #stop not needed container From b6b5691458044f302d23f7f7f57e4252a89c9958 Mon Sep 17 00:00:00 2001 From: Claudia Agache Date: Wed, 4 Nov 2020 14:56:54 +0200 Subject: [PATCH 11/12] try again to run UI tests with legacy transformers --- scripts/startAlfresco.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/scripts/startAlfresco.sh b/scripts/startAlfresco.sh index 31f12d554e..4e5f88e36c 100644 --- a/scripts/startAlfresco.sh +++ b/scripts/startAlfresco.sh @@ -3,4 +3,9 @@ set -ev cd $1 -docker-compose --env-file $2 up -d +# if 2nd input parameter is true then use legacy transformers +# (flags LOCAL_TRANSFORM_SERVICE_ENABLED and TRANSFORM_SERVICE_ENABLED are set to false in .env.ci ) +if $2 ; then + mv -u .env.ci .env +fi +docker-compose up -d From 728ec2ef560c6020fd0b311a80d1f43ce5d65a59 Mon Sep 17 00:00:00 2001 From: Claudia Agache Date: Thu, 5 Nov 2020 08:33:38 +0200 Subject: [PATCH 12/12] remove commented code and limit memory for docker containers only on enterprise --- .travis.tests-stage.yml | 7 ++++--- scripts/dockerLimitMemory.sh | 8 -------- scripts/startAlfresco.sh | 4 ++-- 3 files changed, 6 insertions(+), 13 deletions(-) diff --git a/.travis.tests-stage.yml b/.travis.tests-stage.yml index b53966a699..b368d33bec 100644 --- a/.travis.tests-stage.yml +++ b/.travis.tests-stage.yml @@ -60,13 +60,14 @@ jobs: if [[ ${TRAVIS_COMMIT_MESSAGE} == *"[tests on community]"* ]]; then travis_retry travis_wait 90 mvn -B -q install $MVN_SKIP -PbuildDockerImage -pl :alfresco-governance-services-community-repo,:alfresco-governance-services-community-share -am bash scripts/startAlfresco.sh $COMMUNITY_SHARE_PATH false + bash scripts/waitForAlfrescoToStart.sh else travis_retry travis_wait 90 mvn -B -q install $MVN_SKIP -PbuildDockerImage -pl :alfresco-governance-services-enterprise-repo,:alfresco-governance-services-enterprise-share -am bash scripts/startAlfresco.sh $ENTERPRISE_SHARE_PATH true + bash scripts/waitForAlfrescoToStart.sh + bash scripts/dockerLimitMemory.sh + sudo free -m -t fi - - bash scripts/waitForAlfrescoToStart.sh - - bash scripts/dockerLimitMemory.sh - - sudo free -m -t - travis_retry travis_wait 30 mvn -B -U -q clean install ${MVN_SKIP} -pl :alfresco-governance-services-automation-ui -am script: - travis_wait 90 xvfb-run --server-args="-screen 0 1920x1080x24" mvn -B test -pl :alfresco-governance-services-automation-ui -Dskip.automationtests=false -DsuiteXmlFile=communitySmokeRecords.xml -Dshare.url=${SHARE_URL} -Dalfresco.url=${ALFRESCO_URL} ${WEBDRIVER_ARGUMENTS} diff --git a/scripts/dockerLimitMemory.sh b/scripts/dockerLimitMemory.sh index a2d454ebc9..ca26b4c459 100644 --- a/scripts/dockerLimitMemory.sh +++ b/scripts/dockerLimitMemory.sh @@ -18,18 +18,10 @@ shareContainerId=$(docker ps -a | grep '_share_' | awk '{print $1}') if [ -n "$shareContainerId" ]; then docker update --memory=1Gb --memory-swap -1 --restart on-failure $shareContainerId docker stop $(docker ps -a | grep '_transform-router_' | awk '{print $1}') -# docker stop $(docker ps -a | grep '_imagemagick_' | awk '{print $1}') -# docker stop $(docker ps -a | grep '_alfresco-pdf-renderer_' | awk '{print $1}') docker stop $(docker ps -a | grep '_shared-file-store_' | awk '{print $1}') -# docker stop $(docker ps -a | grep '_tika_' | awk '{print $1}') -# docker stop $(docker ps -a | grep '_libreoffice_' | awk '{print $1}') else docker update --memory=512Mb --memory-swap -1 --restart on-failure $(docker ps -a | grep '_transform-router_' | awk '{print $1}') -# docker update --memory=512Mb --memory-swap -1 --restart on-failure $(docker ps -a | grep '_imagemagick_' | awk '{print $1}') -# docker update --memory=512Mb --memory-swap -1 --restart on-failure $(docker ps -a | grep '_alfresco-pdf-renderer_' | awk '{print $1}') docker update --memory=300Mb --memory-swap -1 --restart on-failure $(docker ps -a | grep '_shared-file-store_' | awk '{print $1}') -# docker update --memory=512Mb --memory-swap -1 --restart on-failure $(docker ps -a | grep '_tika_' | awk '{print $1}') -# docker update --memory=512Mb --memory-swap -1 --restart on-failure $(docker ps -a | grep '_libreoffice_' | awk '{print $1}') fi #stop not needed container diff --git a/scripts/startAlfresco.sh b/scripts/startAlfresco.sh index 4e5f88e36c..f1dad1fcb2 100644 --- a/scripts/startAlfresco.sh +++ b/scripts/startAlfresco.sh @@ -3,8 +3,8 @@ set -ev cd $1 -# if 2nd input parameter is true then use legacy transformers -# (flags LOCAL_TRANSFORM_SERVICE_ENABLED and TRANSFORM_SERVICE_ENABLED are set to false in .env.ci ) +# if 2nd input parameter is true then use .env.ci where TRANSFORM_SERVICE_ENABLED flag is set to false +# in order to not use anymore Transform router and Shared File Store if $2 ; then mv -u .env.ci .env fi