diff --git a/.travis.yml b/.travis.yml index 2db5f4fbee..cf8e187c8a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -46,11 +46,13 @@ jobs: include: - name: "Core, Data-Model, Repository - AllUnitTestsSuite - Build and test" + if: commit_message !~ /\[skip repo\]/ script: - travis_retry mvn -B test -pl core,data-model - travis_retry mvn -B test -pl repository -Dtest=AllUnitTestsSuite - name: "Repository - AppContext01TestSuite" + if: commit_message !~ /\[skip repo\]/ before_script: - docker run -d -p 5433:5432 -e POSTGRES_PASSWORD=alfresco -e POSTGRES_USER=alfresco -e POSTGRES_DB=alfresco postgres:13.1 postgres -c 'max_connections=300' - docker run -d -p 61616:61616 -p 5672:5672 alfresco/alfresco-activemq:5.16.1 @@ -58,12 +60,14 @@ jobs: script: travis_wait 20 mvn -B test -pl repository -Dtest=AppContext01TestSuite -Ddb.driver=org.postgresql.Driver -Ddb.name=alfresco -Ddb.url=jdbc:postgresql://localhost:5433/alfresco -Ddb.username=alfresco -Ddb.password=alfresco - name: "Repository - AppContext02TestSuite" + if: commit_message !~ /\[skip repo\]/ before_script: - docker run -d -p 5433:5432 -e POSTGRES_PASSWORD=alfresco -e POSTGRES_USER=alfresco -e POSTGRES_DB=alfresco postgres:13.1 postgres -c 'max_connections=300' - docker run -d -p 61616:61616 -p 5672:5672 alfresco/alfresco-activemq:5.16.1 script: travis_wait 20 mvn -B test -pl repository -Dtest=AppContext02TestSuite -Ddb.driver=org.postgresql.Driver -Ddb.name=alfresco -Ddb.url=jdbc:postgresql://localhost:5433/alfresco -Ddb.username=alfresco -Ddb.password=alfresco - name: "Repository - AppContext03TestSuite" + if: commit_message !~ /\[skip repo\]/ before_script: - docker run -d -p 5433:5432 -e POSTGRES_PASSWORD=alfresco -e POSTGRES_USER=alfresco -e POSTGRES_DB=alfresco postgres:13.1 postgres -c 'max_connections=300' - docker run -d -p 61616:61616 -p 5672:5672 alfresco/alfresco-activemq:5.16.1 @@ -71,6 +75,7 @@ jobs: script: travis_wait 20 mvn -B test -pl repository -Dtest=AppContext03TestSuite -Ddb.driver=org.postgresql.Driver -Ddb.name=alfresco -Ddb.url=jdbc:postgresql://localhost:5433/alfresco -Ddb.username=alfresco -Ddb.password=alfresco - name: "Repository - AppContext04TestSuite" + if: commit_message !~ /\[skip repo\]/ before_script: - docker run -d -p 5433:5432 -e POSTGRES_PASSWORD=alfresco -e POSTGRES_USER=alfresco -e POSTGRES_DB=alfresco postgres:13.1 postgres -c 'max_connections=300' - docker run -d -p 61616:61616 -p 5672:5672 alfresco/alfresco-activemq:5.16.1 @@ -78,6 +83,7 @@ jobs: script: travis_wait 20 mvn -B test -pl repository -Dtest=AppContext04TestSuite -Ddb.driver=org.postgresql.Driver -Ddb.name=alfresco -Ddb.url=jdbc:postgresql://localhost:5433/alfresco -Ddb.username=alfresco -Ddb.password=alfresco - name: "Repository - AppContext05TestSuite" + if: commit_message !~ /\[skip repo\]/ before_script: - docker run -d -p 5433:5432 -e POSTGRES_PASSWORD=alfresco -e POSTGRES_USER=alfresco -e POSTGRES_DB=alfresco postgres:13.1 postgres -c 'max_connections=300' - docker run -d -p 61616:61616 -p 5672:5672 alfresco/alfresco-activemq:5.16.1 @@ -88,6 +94,7 @@ jobs: script: travis_wait 20 mvn -B test -pl repository -Dtest=AppContext05TestSuite -Ddb.driver=org.postgresql.Driver -Ddb.name=alfresco -Ddb.url=jdbc:postgresql://localhost:5433/alfresco -Ddb.username=alfresco -Ddb.password=alfresco "-Didentity-service.auth-server-url=http://${HOST_IP}:8999/auth" - name: "Repository - AppContext06TestSuite" + if: commit_message !~ /\[skip repo\]/ before_script: - docker run -d -p 5433:5432 -e POSTGRES_PASSWORD=alfresco -e POSTGRES_USER=alfresco -e POSTGRES_DB=alfresco postgres:13.1 postgres -c 'max_connections=300' - docker run -d -p 61616:61616 -p 5672:5672 alfresco/alfresco-activemq:5.16.1 @@ -95,6 +102,7 @@ jobs: script: travis_wait 20 mvn -B test -pl repository -Dtest=AppContext06TestSuite -Ddb.driver=org.postgresql.Driver -Ddb.name=alfresco -Ddb.url=jdbc:postgresql://localhost:5433/alfresco -Ddb.username=alfresco -Ddb.password=alfresco - name: "Repository - AppContextExtraTestSuite" + if: commit_message !~ /\[skip repo\]/ before_script: - docker run -d -p 5433:5432 -e POSTGRES_PASSWORD=alfresco -e POSTGRES_USER=alfresco -e POSTGRES_DB=alfresco postgres:13.1 postgres -c 'max_connections=300' - docker run -d -p 61616:61616 -p 5672:5672 alfresco/alfresco-activemq:5.16.1 @@ -102,6 +110,7 @@ jobs: script: travis_wait 20 mvn -B test -pl repository -Dtest=AppContextExtraTestSuite -Ddb.driver=org.postgresql.Driver -Ddb.name=alfresco -Ddb.url=jdbc:postgresql://localhost:5433/alfresco -Ddb.username=alfresco -Ddb.password=alfresco - name: "Repository - MiscContextTestSuite" + if: commit_message !~ /\[skip repo\]/ before_script: - docker run -d -p 5433:5432 -e POSTGRES_PASSWORD=alfresco -e POSTGRES_USER=alfresco -e POSTGRES_DB=alfresco postgres:13.1 postgres -c 'max_connections=300' - docker run -d -p 61616:61616 -p 5672:5672 alfresco/alfresco-activemq:5.16.1 @@ -109,81 +118,85 @@ jobs: script: travis_wait 20 mvn -B test -pl repository -Dtest=MiscContextTestSuite -Ddb.driver=org.postgresql.Driver -Ddb.name=alfresco -Ddb.url=jdbc:postgresql://localhost:5433/alfresco -Ddb.username=alfresco -Ddb.password=alfresco - name: "Repository - SearchTestSuite" + if: commit_message !~ /\[skip repo\]/ before_script: - docker run -d -p 5433:5432 -e POSTGRES_PASSWORD=alfresco -e POSTGRES_USER=alfresco -e POSTGRES_DB=alfresco postgres:13.1 postgres -c 'max_connections=300' - docker run -d -p 61616:61616 -p 5672:5672 alfresco/alfresco-activemq:5.16.1 script: travis_wait 20 mvn -B test -pl repository -Dtest=SearchTestSuite -Ddb.driver=org.postgresql.Driver -Ddb.name=alfresco -Ddb.url=jdbc:postgresql://localhost:5433/alfresco -Ddb.username=alfresco -Ddb.password=alfresco -Dindex.subsystem.name=solr6 - name: "Repository - MariaDB 10.2.18 tests" - if: commit_message !~ /\[skip db\]/ + if: (branch =~ /(release\/.*$|master)/ AND commit_message !~ /\[skip db\]/ AND type != pull_request) OR commit_message =~ /\[db\]/ before_script: - docker run -d -p 3307:3306 --name mariadb -e MYSQL_ROOT_PASSWORD=alfresco -e MYSQL_USER=alfresco -e MYSQL_DATABASE=alfresco -e MYSQL_PASSWORD=alfresco mariadb:10.2.18 --transaction-isolation=READ-COMMITTED --max-connections=300 --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci - docker run -d -p 61616:61616 -p 5672:5672 alfresco/alfresco-activemq:5.16.1 script: travis_wait 20 mvn -B test -pl repository -Dtest=AllDBTestsTestSuite -Ddb.name=alfresco -Ddb.url=jdbc:mariadb://localhost:3307/alfresco?useUnicode=yes\&characterEncoding=UTF-8 -Ddb.username=alfresco -Ddb.password=alfresco -Ddb.driver=org.mariadb.jdbc.Driver - name: "Repository - MariaDB 10.4 tests" - if: commit_message !~ /\[skip db\]/ + if: (branch =~ /(release\/.*$|master)/ AND commit_message !~ /\[skip db\]/ AND type != pull_request) OR commit_message =~ /\[db\]/ before_script: - docker run -d -p 3307:3306 --name mariadb -e MYSQL_ROOT_PASSWORD=alfresco -e MYSQL_USER=alfresco -e MYSQL_DATABASE=alfresco -e MYSQL_PASSWORD=alfresco mariadb:10.4 --transaction-isolation=READ-COMMITTED --max-connections=300 --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci - docker run -d -p 61616:61616 -p 5672:5672 alfresco/alfresco-activemq:5.16.1 script: travis_wait 20 mvn -B test -pl repository -Dtest=AllDBTestsTestSuite -Ddb.name=alfresco -Ddb.url=jdbc:mariadb://localhost:3307/alfresco?useUnicode=yes\&characterEncoding=UTF-8 -Ddb.username=alfresco -Ddb.password=alfresco -Ddb.driver=org.mariadb.jdbc.Driver - name: "Repository - MariaDB 10.5 tests" - if: commit_message !~ /\[skip db\]/ + if: (branch =~ /(release\/.*$|master)/ AND commit_message !~ /\[skip db\]/ AND type != pull_request) OR commit_message =~ /\[db\]/ OR commit_message =~ /\[latest db\]/ before_script: - docker run -d -p 3307:3306 --name mariadb -e MYSQL_ROOT_PASSWORD=alfresco -e MYSQL_USER=alfresco -e MYSQL_DATABASE=alfresco -e MYSQL_PASSWORD=alfresco mariadb:10.5 --transaction-isolation=READ-COMMITTED --max-connections=300 --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci - docker run -d -p 61616:61616 -p 5672:5672 alfresco/alfresco-activemq:5.16.1 script: travis_wait 20 mvn -B test -pl repository -Dtest=AllDBTestsTestSuite -Ddb.name=alfresco -Ddb.url=jdbc:mariadb://localhost:3307/alfresco?useUnicode=yes\&characterEncoding=UTF-8 -Ddb.username=alfresco -Ddb.password=alfresco -Ddb.driver=org.mariadb.jdbc.Driver - name: "Repository - MySQL 5.7.23 tests" - if: commit_message !~ /\[skip db\]/ + if: (branch =~ /(release\/.*$|master)/ AND commit_message !~ /\[skip db\]/ AND type != pull_request) OR commit_message =~ /\[db\]/ before_script: - docker run -d -p 3307:3306 -e MYSQL_ROOT_PASSWORD=alfresco -e MYSQL_USER=alfresco -e MYSQL_DATABASE=alfresco -e MYSQL_PASSWORD=alfresco mysql:5.7.23 --transaction-isolation='READ-COMMITTED' - docker run -d -p 61616:61616 -p 5672:5672 alfresco/alfresco-activemq:5.16.1 script: travis_wait 20 mvn -B test -pl repository -Dtest=AllDBTestsTestSuite -Ddb.driver=com.mysql.jdbc.Driver -Ddb.name=alfresco -Ddb.url=jdbc:mysql://localhost:3307/alfresco -Ddb.username=alfresco -Ddb.password=alfresco - name: "Repository - MySQL 8 tests" - if: commit_message !~ /\[skip db\]/ + if: (branch =~ /(release\/.*$|master)/ AND commit_message !~ /\[skip db\]/ AND type != pull_request) OR commit_message =~ /\[db\]/ OR commit_message =~ /\[latest db\]/ before_script: - docker run -d -p 3307:3306 -e MYSQL_ROOT_PASSWORD=alfresco -e MYSQL_USER=alfresco -e MYSQL_DATABASE=alfresco -e MYSQL_PASSWORD=alfresco mysql:8 --transaction-isolation='READ-COMMITTED' - docker run -d -p 61616:61616 -p 5672:5672 alfresco/alfresco-activemq:5.16.1 script: travis_wait 20 mvn -B test -pl repository -Dtest=AllDBTestsTestSuite -Ddb.driver=com.mysql.jdbc.Driver -Ddb.name=alfresco -Ddb.url=jdbc:mysql://localhost:3307/alfresco -Ddb.username=alfresco -Ddb.password=alfresco - name: "Repository - PostgreSQL 10.9 tests" - if: commit_message !~ /\[skip db\]/ + if: (branch =~ /(release\/.*$|master)/ AND commit_message !~ /\[skip db\]/ AND type != pull_request) OR commit_message =~ /\[db\]/ before_script: - docker run -d -p 5433:5432 -e POSTGRES_PASSWORD=alfresco -e POSTGRES_USER=alfresco -e POSTGRES_DB=alfresco postgres:10.9 postgres -c 'max_connections=300' - docker run -d -p 61616:61616 -p 5672:5672 alfresco/alfresco-activemq:5.16.1 script: travis_wait 20 mvn -B test -pl repository -Dtest=AllDBTestsTestSuite -Ddb.driver=org.postgresql.Driver -Ddb.name=alfresco -Ddb.url=jdbc:postgresql://localhost:5433/alfresco -Ddb.username=alfresco -Ddb.password=alfresco - name: "Repository - PostgreSQL 11.7 tests" - if: commit_message !~ /\[skip db\]/ + if: (branch =~ /(release\/.*$|master)/ AND commit_message !~ /\[skip db\]/ AND type != pull_request) OR commit_message =~ /\[db\]/ before_script: - docker run -d -p 5433:5432 -e POSTGRES_PASSWORD=alfresco -e POSTGRES_USER=alfresco -e POSTGRES_DB=alfresco postgres:11.7 postgres -c 'max_connections=300' - docker run -d -p 61616:61616 -p 5672:5672 alfresco/alfresco-activemq:5.16.1 script: travis_wait 20 mvn -B test -pl repository -Dtest=AllDBTestsTestSuite -Ddb.driver=org.postgresql.Driver -Ddb.name=alfresco -Ddb.url=jdbc:postgresql://localhost:5433/alfresco -Ddb.username=alfresco -Ddb.password=alfresco - name: "Repository - PostgreSQL 12.4 tests" - if: commit_message !~ /\[skip db\]/ + if: (branch =~ /(release\/.*$|master)/ AND commit_message !~ /\[skip db\]/ AND type != pull_request) OR commit_message =~ /\[db\]/ before_script: - docker run -d -p 5433:5432 -e POSTGRES_PASSWORD=alfresco -e POSTGRES_USER=alfresco -e POSTGRES_DB=alfresco postgres:12.4 postgres -c 'max_connections=300' - docker run -d -p 61616:61616 -p 5672:5672 alfresco/alfresco-activemq:5.16.1 script: travis_wait 20 mvn -B test -pl repository -Dtest=AllDBTestsTestSuite -Ddb.driver=org.postgresql.Driver -Ddb.name=alfresco -Ddb.url=jdbc:postgresql://localhost:5433/alfresco -Ddb.username=alfresco -Ddb.password=alfresco - name: "Repository - PostgreSQL 13.1 tests" - if: commit_message !~ /\[skip db\]/ + # We only run DB tests on the latest version of PostgreSQL on feature branches + if: (branch =~ /(release\/.*$|master)/ AND commit_message !~ /\[skip db\]/ AND type != pull_request) OR commit_message =~ /\[db\]/ before_script: - docker run -d -p 5433:5432 -e POSTGRES_PASSWORD=alfresco -e POSTGRES_USER=alfresco -e POSTGRES_DB=alfresco postgres:13.1 postgres -c 'max_connections=300' - docker run -d -p 61616:61616 -p 5672:5672 alfresco/alfresco-activemq:5.16.1 script: travis_wait 20 mvn -B test -pl repository -Dtest=AllDBTestsTestSuite -Ddb.driver=org.postgresql.Driver -Ddb.name=alfresco -Ddb.url=jdbc:postgresql://localhost:5433/alfresco -Ddb.username=alfresco -Ddb.password=alfresco - name: "Remote-api - AppContext01TestSuite" + if: commit_message !~ /\[skip repo\]/ before_script: - docker run -d -p 5433:5432 -e POSTGRES_PASSWORD=alfresco -e POSTGRES_USER=alfresco -e POSTGRES_DB=alfresco postgres:13.1 postgres -c 'max_connections=300' - docker run -d -p 61616:61616 -p 5672:5672 alfresco/alfresco-activemq:5.16.1 script: travis_wait 20 mvn -B test -pl remote-api -Dtest=AppContext01TestSuite -Ddb.driver=org.postgresql.Driver -Ddb.name=alfresco -Ddb.url=jdbc:postgresql://localhost:5433/alfresco -Ddb.username=alfresco -Ddb.password=alfresco - name: "Remote-api - AppContext02TestSuite" + if: commit_message !~ /\[skip repo\]/ before_script: - docker run -d -p 5433:5432 -e POSTGRES_PASSWORD=alfresco -e POSTGRES_USER=alfresco -e POSTGRES_DB=alfresco postgres:13.1 postgres -c 'max_connections=300' - docker run -d -p 61616:61616 -p 5672:5672 alfresco/alfresco-activemq:5.16.1 @@ -191,6 +204,7 @@ jobs: script: travis_wait 20 mvn -B test -pl remote-api -Dtest=AppContext02TestSuite -Ddb.driver=org.postgresql.Driver -Ddb.name=alfresco -Ddb.url=jdbc:postgresql://localhost:5433/alfresco -Ddb.username=alfresco -Ddb.password=alfresco - name: "Remote-api - AppContext03TestSuite" + if: commit_message !~ /\[skip repo\]/ before_script: - docker run -d -p 5433:5432 -e POSTGRES_PASSWORD=alfresco -e POSTGRES_USER=alfresco -e POSTGRES_DB=alfresco postgres:13.1 postgres -c 'max_connections=300' - docker run -d -p 61616:61616 -p 5672:5672 alfresco/alfresco-activemq:5.16.1 @@ -198,6 +212,7 @@ jobs: script: travis_wait 20 mvn -B test -pl remote-api -Dtest=AppContext03TestSuite -Ddb.driver=org.postgresql.Driver -Ddb.name=alfresco -Ddb.url=jdbc:postgresql://localhost:5433/alfresco -Ddb.username=alfresco -Ddb.password=alfresco - name: "Remote-api - AppContext04TestSuite" + if: commit_message !~ /\[skip repo\]/ before_script: - docker run -d -p 5433:5432 -e POSTGRES_PASSWORD=alfresco -e POSTGRES_USER=alfresco -e POSTGRES_DB=alfresco postgres:13.1 postgres -c 'max_connections=300' - docker run -d -p 61616:61616 -p 5672:5672 alfresco/alfresco-activemq:5.16.1 @@ -205,13 +220,15 @@ jobs: script: travis_wait 20 mvn -B test -pl remote-api -Dtest=AppContext04TestSuite -Ddb.driver=org.postgresql.Driver -Ddb.name=alfresco -Ddb.url=jdbc:postgresql://localhost:5433/alfresco -Ddb.username=alfresco -Ddb.password=alfresco - name: "Remote-api - AppContextExtraTestSuite" + if: commit_message !~ /\[skip repo\]/ before_script: - docker run -d -p 5433:5432 -e POSTGRES_PASSWORD=alfresco -e POSTGRES_USER=alfresco -e POSTGRES_DB=alfresco postgres:13.1 postgres -c 'max_connections=300' - docker run -d -p 61616:61616 -p 5672:5672 alfresco/alfresco-activemq:5.16.1 script: travis_wait 20 mvn -B test -pl remote-api -Dtest=AppContextExtraTestSuite -Ddb.driver=org.postgresql.Driver -Ddb.name=alfresco -Ddb.url=jdbc:postgresql://localhost:5433/alfresco -Ddb.username=alfresco -Ddb.password=alfresco - name: "REST API TAS tests part1" - if: branch = master OR commit_message =~ /\[tas\]/ + # TAS tests are generally skipped on feature branches as they will be repeated on the enterprise repo or community packaging builds + if: (branch =~ /(release\/.*$|master)/ AND commit_message !~ /\[skip tas\]/) OR commit_message =~ /\[tas\]/ before_script: - ${TAS_SCRIPTS}/start-compose.sh ${TAS_ENVIRONMENT}/docker-compose-minimal+transforms.yml - ${TAS_SCRIPTS}/wait-for-alfresco-start.sh "http://localhost:8082/alfresco" @@ -219,7 +236,7 @@ jobs: - travis_wait 60 mvn -B install -f packaging/tests/tas-restapi/pom.xml -Pall-tas-tests,run-restapi-part1 -Denvironment=default -DrunBugs=false - name: "REST API TAS tests part2" - if: branch = master OR commit_message =~ /\[tas\]/ + if: (branch =~ /(release\/.*$|master)/ AND commit_message !~ /\[skip tas\]/) OR commit_message =~ /\[tas\]/ before_script: - ${TAS_SCRIPTS}/start-compose.sh ${TAS_ENVIRONMENT}/docker-compose-minimal+transforms.yml - ${TAS_SCRIPTS}/wait-for-alfresco-start.sh "http://localhost:8082/alfresco" @@ -227,7 +244,7 @@ jobs: - travis_wait 60 mvn -B install -f packaging/tests/tas-restapi/pom.xml -Pall-tas-tests,run-restapi-part2 -Denvironment=default -DrunBugs=false - name: "REST API TAS tests part3" - if: branch = master OR commit_message =~ /\[tas\]/ + if: (branch =~ /(release\/.*$|master)/ AND commit_message !~ /\[skip tas\]/) OR commit_message =~ /\[tas\]/ before_script: - ${TAS_SCRIPTS}/start-compose.sh ${TAS_ENVIRONMENT}/docker-compose-minimal+transforms.yml - ${TAS_SCRIPTS}/wait-for-alfresco-start.sh "http://localhost:8082/alfresco" @@ -235,7 +252,7 @@ jobs: - travis_wait 60 mvn -B install -f packaging/tests/tas-restapi/pom.xml -Pall-tas-tests,run-restapi-part3 -Denvironment=default -DrunBugs=false - name: "CMIS TAS tests - BROWSER binding" - if: branch = master OR commit_message =~ /\[tas\]/ + if: (branch =~ /(release\/.*$|master)/ AND commit_message !~ /\[skip tas\]/) OR commit_message =~ /\[tas\]/ before_script: - ${TAS_SCRIPTS}/start-compose.sh ${TAS_ENVIRONMENT}/docker-compose-minimal+transforms.yml - ${TAS_SCRIPTS}/wait-for-alfresco-start.sh "http://localhost:8082/alfresco" @@ -243,7 +260,7 @@ jobs: - travis_wait 40 mvn -B install -f packaging/tests/tas-cmis/pom.xml -Pall-tas-tests,run-cmis-browser -Denvironment=default -DrunBugs=false - name: "CMIS TAS tests - ATOM binding" - if: branch = master OR commit_message =~ /\[tas\]/ + if: (branch =~ /(release\/.*$|master)/ AND commit_message !~ /\[skip tas\]/) OR commit_message =~ /\[tas\]/ before_script: - ${TAS_SCRIPTS}/start-compose.sh ${TAS_ENVIRONMENT}/docker-compose-minimal+transforms.yml - ${TAS_SCRIPTS}/wait-for-alfresco-start.sh "http://localhost:8082/alfresco" @@ -251,7 +268,7 @@ jobs: - travis_wait 40 mvn -B install -f packaging/tests/tas-cmis/pom.xml -Pall-tas-tests,run-cmis-atom -Denvironment=default -DrunBugs=false - name: "CMIS TAS tests - WEBSERVICES binding" - if: branch = master OR commit_message =~ /\[tas\]/ + if: (branch =~ /(release\/.*$|master)/ AND commit_message !~ /\[skip tas\]/) OR commit_message =~ /\[tas\]/ before_script: - ${TAS_SCRIPTS}/start-compose.sh ${TAS_ENVIRONMENT}/docker-compose-minimal+transforms.yml - ${TAS_SCRIPTS}/wait-for-alfresco-start.sh "http://localhost:8082/alfresco" @@ -259,7 +276,7 @@ jobs: - travis_wait 40 mvn -B install -f packaging/tests/tas-cmis/pom.xml -Pall-tas-tests,run-cmis-webservices -Denvironment=default -DrunBugs=false - name: "Email TAS tests" - if: branch = master OR commit_message =~ /\[tas\]/ + if: (branch =~ /(release\/.*$|master)/ AND commit_message !~ /\[skip tas\]/) OR commit_message =~ /\[tas\]/ before_script: - ${TAS_SCRIPTS}/start-compose.sh ${TAS_ENVIRONMENT}/docker-compose-minimal.yml - ${TAS_SCRIPTS}/wait-for-alfresco-start.sh "http://localhost:8082/alfresco" @@ -267,7 +284,7 @@ jobs: - travis_wait 30 mvn -B install -f packaging/tests/tas-email/pom.xml -Pall-tas-tests -Denvironment=default -DrunBugs=false - name: "WebDAV TAS tests" - if: branch = master OR commit_message =~ /\[tas\]/ + if: (branch =~ /(release\/.*$|master)/ AND commit_message !~ /\[skip tas\]/) OR commit_message =~ /\[tas\]/ before_script: - ${TAS_SCRIPTS}/start-compose.sh ${TAS_ENVIRONMENT}/docker-compose-minimal.yml - ${TAS_SCRIPTS}/wait-for-alfresco-start.sh "http://localhost:8082/alfresco" @@ -275,7 +292,7 @@ jobs: - travis_wait 30 mvn -B install -f packaging/tests/tas-webdav/pom.xml -Pall-tas-tests -Denvironment=default -DrunBugs=false - name: "Integration TAS tests" - if: branch = master OR commit_message =~ /\[tas\]/ + if: (branch =~ /(release\/.*$|master)/ AND commit_message !~ /\[skip tas\]/) OR commit_message =~ /\[tas\]/ before_script: - ${TAS_SCRIPTS}/start-compose.sh ${TAS_ENVIRONMENT}/docker-compose-minimal.yml - ${TAS_SCRIPTS}/wait-for-alfresco-start.sh "http://localhost:8082/alfresco" diff --git a/data-model/src/main/java/org/alfresco/repo/search/impl/querymodel/QueryEngine.java b/data-model/src/main/java/org/alfresco/repo/search/impl/querymodel/QueryEngine.java index e543104e5f..b39cbd572e 100644 --- a/data-model/src/main/java/org/alfresco/repo/search/impl/querymodel/QueryEngine.java +++ b/data-model/src/main/java/org/alfresco/repo/search/impl/querymodel/QueryEngine.java @@ -2,7 +2,7 @@ * #%L * Alfresco Data model classes * %% - * Copyright (C) 2005 - 2016 Alfresco Software Limited + * Copyright (C) 2005 - 2021 Alfresco Software Limited * %% * This file is part of the Alfresco software. * If the software was purchased under a paid Alfresco license, the terms of diff --git a/repository/src/main/java/org/alfresco/repo/search/impl/querymodel/impl/db/DBQuery.java b/repository/src/main/java/org/alfresco/repo/search/impl/querymodel/impl/db/DBQuery.java index fd98d9bc77..abb7c40efe 100644 --- a/repository/src/main/java/org/alfresco/repo/search/impl/querymodel/impl/db/DBQuery.java +++ b/repository/src/main/java/org/alfresco/repo/search/impl/querymodel/impl/db/DBQuery.java @@ -1,28 +1,28 @@ -/* - * #%L - * Alfresco Repository - * %% - * Copyright (C) 2005 - 2016 Alfresco Software Limited - * %% - * This file is part of the Alfresco software. - * If the software was purchased under a paid Alfresco license, the terms of - * the paid license agreement will prevail. Otherwise, the software is - * provided under the following open source license terms: - * - * Alfresco is free software: you can redistribute it and/or modify - * it under the terms of the GNU Lesser General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * Alfresco is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public License - * along with Alfresco. If not, see . - * #L% - */ +/* + * #%L + * Alfresco Repository + * %% + * Copyright (C) 2005 - 2016 Alfresco Software Limited + * %% + * This file is part of the Alfresco software. + * If the software was purchased under a paid Alfresco license, the terms of + * the paid license agreement will prevail. Otherwise, the software is + * provided under the following open source license terms: + * + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + * #L% + */ package org.alfresco.repo.search.impl.querymodel.impl.db; import java.util.ArrayList; @@ -74,6 +74,10 @@ public class DBQuery extends BaseQuery implements DBQueryBuilderComponent Set selectorGroup; + private int limit = 0; + + private int offset = 0; + /** * @param source Source * @param constraint Constraint @@ -133,6 +137,22 @@ public class DBQuery extends BaseQuery implements DBQueryBuilderComponent this.sinceTxId = sinceTxId; } + public int getLimit() { + return limit; + } + + public void setLimit(int limit) { + this.limit = limit; + } + + public int getOffset() { + return offset; + } + + public void setOffset(int offset) { + this.offset = offset; + } + public List getJoins() { HashMap singleJoins = new HashMap(); diff --git a/repository/src/main/java/org/alfresco/repo/search/impl/querymodel/impl/db/DBQueryEngine.java b/repository/src/main/java/org/alfresco/repo/search/impl/querymodel/impl/db/DBQueryEngine.java index 612f958a65..96ca208c73 100644 --- a/repository/src/main/java/org/alfresco/repo/search/impl/querymodel/impl/db/DBQueryEngine.java +++ b/repository/src/main/java/org/alfresco/repo/search/impl/querymodel/impl/db/DBQueryEngine.java @@ -76,8 +76,10 @@ import org.alfresco.service.namespace.QName; import org.alfresco.util.Pair; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.ibatis.executor.result.DefaultResultContext; import org.apache.ibatis.session.ResultContext; import org.apache.ibatis.session.ResultHandler; +import org.apache.ibatis.session.RowBounds; import org.mybatis.spring.SqlSessionTemplate; /** @@ -89,7 +91,11 @@ public class DBQueryEngine implements QueryEngine protected static final Log logger = LogFactory.getLog(DBQueryEngine.class); protected static final String SELECT_BY_DYNAMIC_QUERY = "alfresco.metadata.query.select_byDynamicQuery"; - + + private static final int DEFAULT_MIN_PAGING_BATCH_SIZE = 2500; + + private static final int DEFAULT_MAX_PAGING_BATCH_SIZE = 10000; + protected SqlSessionTemplate template; protected QNameDAO qnameDAO; @@ -114,6 +120,12 @@ public class DBQueryEngine implements QueryEngine private boolean maxPermissionCheckEnabled; + private boolean usePagingQuery = false; + + private int minPagingBatchSize = DEFAULT_MIN_PAGING_BATCH_SIZE; + + private int maxPagingBatchSize = DEFAULT_MAX_PAGING_BATCH_SIZE; + protected EntityLookupCache nodesCache; private List> stores; @@ -149,7 +161,31 @@ public class DBQueryEngine implements QueryEngine { this.permissionService = permissionService; } - + + public boolean isUsePagingQuery() { + return usePagingQuery; + } + + public void setUsePagingQuery(boolean usePagingQuery) { + this.usePagingQuery = usePagingQuery; + } + + public int getMinPagingBatchSize() { + return minPagingBatchSize; + } + + public void setMinPagingBatchSize(int minPagingBatchSize) { + this.minPagingBatchSize = minPagingBatchSize; + } + + public int getMaxPagingBatchSize() { + return maxPagingBatchSize; + } + + public void setMaxPagingBatchSize(int maxPagingBatchSize) { + this.maxPagingBatchSize = maxPagingBatchSize; + } + public void setMetadataIndexCheck2(OptionalPatchApplicationCheckBootstrapBean metadataIndexCheck2) { this.metadataIndexCheck2 = metadataIndexCheck2; @@ -331,7 +367,7 @@ public class DBQueryEngine implements QueryEngine int requiredNodes = computeRequiredNodesCount(options); logger.debug("- query sent to the database"); - template.select(pickQueryTemplate(options, dbQuery), dbQuery, new ResultHandler() + performTmdqSelect(pickQueryTemplate(options, dbQuery), dbQuery, requiredNodes, new ResultHandler() { @Override public void handleResult(ResultContext context) @@ -399,6 +435,54 @@ public class DBQueryEngine implements QueryEngine return frs; } + private void performTmdqSelect(String statement, DBQuery dbQuery, int requiredNodes, ResultHandler handler) + { + if (usePagingQuery) + { + performTmdqSelectPaging(statement, dbQuery, requiredNodes, handler); + } + else + { + performTmdqSelectStreaming(statement, dbQuery, handler); + } + } + + private void performTmdqSelectStreaming(String statement, DBQuery dbQuery, ResultHandler handler) + { + template.select(statement, dbQuery, handler); + } + + private void performTmdqSelectPaging(String statement, DBQuery dbQuery, int requiredNodes, ResultHandler handler) + { + int batchStart = 0; + int batchSize = requiredNodes * 2; + batchSize = Math.min(Math.max(batchSize, minPagingBatchSize), maxPagingBatchSize); + DefaultResultContext resultCtx = new DefaultResultContext<>(); + while (!resultCtx.isStopped()) + { + dbQuery.setOffset(batchStart); + dbQuery.setLimit(batchSize); + List batch = template.selectList(statement, dbQuery); + for (Node node : batch) + { + resultCtx.nextResultObject(node); + handler.handleResult(resultCtx); + if (resultCtx.isStopped()) + { + break; + } + } + if (batch.size() < batchSize) + { + resultCtx.stop(); + } + else + { + batchStart += batchSize; + } + } + } + private DBResultSet createResultSet(QueryOptions options, List nodes, int numberFound) { DBResultSet dbResultSet = new DBResultSet(options.getAsSearchParmeters(), nodes, nodeDAO, nodeService, tenantService, Integer.MAX_VALUE); diff --git a/repository/src/main/resources/alfresco/ibatis/org.alfresco.repo.domain.dialect.Dialect/metadata-query-common-SqlMap.xml b/repository/src/main/resources/alfresco/ibatis/org.alfresco.repo.domain.dialect.Dialect/metadata-query-common-SqlMap.xml index 3fa7690db8..715537bff4 100644 --- a/repository/src/main/resources/alfresco/ibatis/org.alfresco.repo.domain.dialect.Dialect/metadata-query-common-SqlMap.xml +++ b/repository/src/main/resources/alfresco/ibatis/org.alfresco.repo.domain.dialect.Dialect/metadata-query-common-SqlMap.xml @@ -180,7 +180,8 @@ - - + + limit #{offset}, #{limit} + \ No newline at end of file diff --git a/repository/src/main/resources/alfresco/subsystems/Search/common-search-context.xml b/repository/src/main/resources/alfresco/subsystems/Search/common-search-context.xml index 7410e27f2b..932c4a06ef 100644 --- a/repository/src/main/resources/alfresco/subsystems/Search/common-search-context.xml +++ b/repository/src/main/resources/alfresco/subsystems/Search/common-search-context.xml @@ -105,8 +105,35 @@ - - + + + + search.dbQueryEngineImpl.#bean.dialect# + + + org.alfresco.repo.search.impl.querymodel.QueryEngine + + + org.alfresco.repo.domain.dialect.Dialect + + + + + + + + + + + + + + + + + diff --git a/repository/src/test/java/org/alfresco/AllDBTestsTestSuite.java b/repository/src/test/java/org/alfresco/AllDBTestsTestSuite.java index 6dddc58bbe..1737976f0e 100644 --- a/repository/src/test/java/org/alfresco/AllDBTestsTestSuite.java +++ b/repository/src/test/java/org/alfresco/AllDBTestsTestSuite.java @@ -2,7 +2,7 @@ * #%L * Alfresco Repository * %% - * Copyright (C) 2005 - 2017 Alfresco Software Limited + * Copyright (C) 2005 - 2021 Alfresco Software Limited * %% * This file is part of the Alfresco software. * If the software was purchased under a paid Alfresco license, the terms of @@ -77,7 +77,10 @@ import org.junit.runners.Suite; // From MiscContextTestSuite org.alfresco.repo.domain.query.CannedQueryDAOTest.class, - // REPO-2963 : Tests causing a cascade of failures in AllDBTestsTestSuite on PostgreSQL/MySQL + // ACS-1907 + org.alfresco.repo.search.impl.querymodel.impl.db.ACS1907Test.class, + + // REPO-2963 : Tests causing a cascade of failures in AllDBTestsTestSuite on PostgreSQL/MySQL // Moved at the bottom of the suite because DbNodeServiceImplTest.testNodeCleanupRegistry() takes a long time on a clean DB. org.alfresco.repo.node.db.DbNodeServiceImplTest.class, diff --git a/repository/src/test/java/org/alfresco/repo/search/impl/querymodel/impl/db/ACS1907Test.java b/repository/src/test/java/org/alfresco/repo/search/impl/querymodel/impl/db/ACS1907Test.java new file mode 100644 index 0000000000..50232978e9 --- /dev/null +++ b/repository/src/test/java/org/alfresco/repo/search/impl/querymodel/impl/db/ACS1907Test.java @@ -0,0 +1,260 @@ +/* + * #%L + * Alfresco Repository + * %% + * Copyright (C) 2005 - 2021 Alfresco Software Limited + * %% + * This file is part of the Alfresco software. + * If the software was purchased under a paid Alfresco license, the terms of + * the paid license agreement will prevail. Otherwise, the software is + * provided under the following open source license terms: + * + * Alfresco is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Alfresco is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with Alfresco. If not, see . + * #L% + */ +package org.alfresco.repo.search.impl.querymodel.impl.db; + +import junit.framework.TestCase; +import org.alfresco.model.ContentModel; +import org.alfresco.repo.cache.TransactionalCache; +import org.alfresco.repo.management.subsystems.SwitchableApplicationContextFactory; +import org.alfresco.repo.security.authentication.AuthenticationComponent; +import org.alfresco.repo.security.authentication.AuthenticationUtil; +import org.alfresco.repo.security.authentication.MutableAuthenticationDao; +import org.alfresco.repo.security.permissions.AccessControlList; +import org.alfresco.repo.transaction.RetryingTransactionHelper; +import org.alfresco.service.cmr.repository.NodeRef; +import org.alfresco.service.cmr.repository.NodeService; +import org.alfresco.service.cmr.repository.StoreRef; +import org.alfresco.service.cmr.search.*; +import org.alfresco.service.cmr.security.MutableAuthenticationService; +import org.alfresco.service.cmr.security.PermissionService; +import org.alfresco.service.namespace.QName; +import org.alfresco.service.transaction.TransactionService; +import org.alfresco.test_category.OwnJVMTestsCategory; +import org.alfresco.util.ApplicationContextHelper; +import org.alfresco.util.testing.category.DBTests; +import org.junit.experimental.categories.Category; +import org.springframework.context.ApplicationContext; + +import java.io.Serializable; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; + +@Category({OwnJVMTestsCategory.class, DBTests.class}) +public class ACS1907Test extends TestCase +{ + + private ApplicationContext ctx; + + private NodeService nodeService; + private AuthenticationComponent authenticationComponent; + private MutableAuthenticationService authenticationService; + private MutableAuthenticationDao authenticationDAO; + private SearchService pubSearchService; + private PermissionService pubPermissionService; + private TransactionService transactionService; + private RetryingTransactionHelper txnHelper; + private DBQueryEngine queryEngine; + + private TransactionalCache aclCache; + private TransactionalCache aclEntityCache; + private TransactionalCache permissionEntityCache; + + private NodeRef rootNodeRef; + + @Override + public void setUp() throws Exception + { + setupServices(); + this.authenticationComponent.setSystemUserAsCurrentUser(); + rootNodeRef = nodeService.getRootNode(StoreRef.STORE_REF_WORKSPACE_SPACESSTORE); + setupTestUsers(); + setupTestContent(); + dropCaches(); + } + + @Override + protected void tearDown() throws Exception + { + authenticationComponent.clearCurrentSecurityContext(); + } + + private void setupServices() + { + ctx = ApplicationContextHelper.getApplicationContext(); + nodeService = (NodeService) ctx.getBean("dbNodeService"); + authenticationComponent = (AuthenticationComponent) ctx.getBean("authenticationComponent"); + authenticationService = (MutableAuthenticationService) ctx.getBean("authenticationService"); + authenticationDAO = (MutableAuthenticationDao) ctx.getBean("authenticationDao"); + pubSearchService = (SearchService) ctx.getBean("SearchService"); + pubPermissionService = (PermissionService) ctx.getBean("PermissionService"); + transactionService = (TransactionService) ctx.getBean("TransactionService"); + aclCache = (TransactionalCache) ctx.getBean("aclCache"); + aclEntityCache = (TransactionalCache) ctx.getBean("aclEntityCache"); + permissionEntityCache = (TransactionalCache) ctx.getBean("permissionEntityCache"); + SwitchableApplicationContextFactory searchContextFactory = (SwitchableApplicationContextFactory) ctx.getBean("Search"); + ApplicationContext searchCtx = searchContextFactory.getApplicationContext(); + queryEngine = (DBQueryEngine) searchCtx.getBean("search.dbQueryEngineImpl"); + txnHelper = new RetryingTransactionHelper(); + txnHelper.setTransactionService(transactionService); + txnHelper.setReadOnly(false); + txnHelper.setMaxRetries(1); + txnHelper.setMinRetryWaitMs(1); + txnHelper.setMaxRetryWaitMs(10); + txnHelper.setRetryWaitIncrementMs(1); + } + + private void setupTestUser(String userName) + { + if (!authenticationDAO.userExists(userName)) + { + authenticationService.createAuthentication(userName, userName.toCharArray()); + } + } + + private void setupTestUsers() + { + txnHelper.doInTransaction(new RetryingTransactionHelper.RetryingTransactionCallback() { + @Override + public Object execute() throws Throwable { + setupTestUser("userA"); + setupTestUser("userB"); + setupTestUser(AuthenticationUtil.getAdminUserName()); + return null; + } + }, false, false); + } + + private void setupTestContent() + { + for(int f = 0; f < 5; f++) + { + final int ff = f; + txnHelper.doInTransaction(new RetryingTransactionHelper.RetryingTransactionCallback() { + @Override + public Object execute() throws Throwable { + Map testFolderProps = new HashMap<>(); + testFolderProps.put(ContentModel.PROP_NAME, "folder"+ff); + NodeRef testFolder = nodeService.createNode( + rootNodeRef, + ContentModel.ASSOC_CHILDREN, + QName.createQName("https://example.com/test", "folder"+ff), + ContentModel.TYPE_FOLDER, + testFolderProps + ).getChildRef(); + for(int c = 0; c < 5; c++) + { + Map testContentProps = new HashMap<>(); + testContentProps.put(ContentModel.PROP_NAME, "content"+c); + NodeRef testContent = nodeService.createNode( + testFolder, + ContentModel.ASSOC_CONTAINS, + QName.createQName("https://example.com/test", "content"+c), + ContentModel.TYPE_CONTENT, + testContentProps + ).getChildRef(); + String user = c % 2 == 0 ? "userA" : "userB"; + pubPermissionService.setPermission(testContent, user, "Read", true); + } + return null; + } + }, false, false); + } + } + + private void dropCaches() + { + aclCache.clear(); + aclEntityCache.clear(); + permissionEntityCache.clear(); + } + + public void testACS1907() + { + txnHelper.doInTransaction(new RetryingTransactionHelper.RetryingTransactionCallback() { + @Override + public Object execute() throws Throwable { + AuthenticationUtil.runAs(new AuthenticationUtil.RunAsWork() { + @Override + public Object doWork() throws Exception { + SearchParameters sp = new SearchParameters(); + sp.addStore(StoreRef.STORE_REF_WORKSPACE_SPACESSTORE); + sp.setLanguage(SearchService.LANGUAGE_FTS_ALFRESCO); + sp.setQueryConsistency(QueryConsistency.TRANSACTIONAL); + sp.setQuery("TYPE:\"cm:content\""); + ResultSet rs = pubSearchService.query(sp); + int cnt = 0; + for (ResultSetRow row : rs) + { + assertNotNull(row.getValue(ContentModel.PROP_NAME)); + cnt++; + } + return null; + } + }, "userA"); + return null; + } + }, false, false); + } + + public void testPaging() + { + HashSet resultPageSize2 = queryNodes(2); + HashSet resultPageSize5 = queryNodes(5); + HashSet resultPageSize10 = queryNodes(10); + HashSet resultPageSizeAll = queryNodes(10000); + // all result sets must be equal, independent of page size used to retrieve them + assertTrue(resultPageSize2.size() >= 25); + assertTrue(resultPageSize5.size() >= 25); + assertTrue(resultPageSize10.size() >= 25); + assertTrue(resultPageSizeAll.size() >= 25); + assertTrue(resultPageSize2.containsAll(resultPageSize5)); + assertTrue(resultPageSize2.containsAll(resultPageSize10)); + assertTrue(resultPageSize2.containsAll(resultPageSizeAll)); + assertTrue(resultPageSize5.containsAll(resultPageSize2)); + assertTrue(resultPageSize5.containsAll(resultPageSize10)); + assertTrue(resultPageSize5.containsAll(resultPageSizeAll)); + assertTrue(resultPageSize10.containsAll(resultPageSize2)); + assertTrue(resultPageSize10.containsAll(resultPageSize5)); + assertTrue(resultPageSize10.containsAll(resultPageSizeAll)); + assertTrue(resultPageSizeAll.containsAll(resultPageSize2)); + assertTrue(resultPageSizeAll.containsAll(resultPageSize5)); + assertTrue(resultPageSizeAll.containsAll(resultPageSize10)); + // reset + queryEngine.setMinPagingBatchSize(2500); + queryEngine.setMaxPagingBatchSize(10000); + } + + HashSet queryNodes(int pageSize) + { + queryEngine.setMinPagingBatchSize(pageSize); + queryEngine.setMaxPagingBatchSize(pageSize); + HashSet result = new HashSet<>(); + SearchParameters sp = new SearchParameters(); + sp.addStore(StoreRef.STORE_REF_WORKSPACE_SPACESSTORE); + sp.setLanguage(SearchService.LANGUAGE_FTS_ALFRESCO); + sp.setQueryConsistency(QueryConsistency.TRANSACTIONAL); + sp.setQuery("TYPE:\"cm:content\""); + ResultSet rs = pubSearchService.query(sp); + int cnt = 0; + for (ResultSetRow row : rs) + { + result.add(row.getNodeRef()); + } + return result; + } + +}