diff --git a/.github/actions/download-build-artifacts/action.yml b/.github/actions/download-build-artifacts/action.yml deleted file mode 100644 index bfd93ad06f..0000000000 --- a/.github/actions/download-build-artifacts/action.yml +++ /dev/null @@ -1,53 +0,0 @@ -name: "Download build artifacts" -description: "Download build artifacts" -runs: - using: "composite" - steps: - ## NXCACHE - - name: update NX cache folder - uses: actions/download-artifact@v3 - with: - path: . - name: nxcache - ## DIST - - name: update Dist folder - uses: actions/download-artifact@v3 - with: - path: . - name: dist - ## ADF CLI - - name: update Dist folder - uses: actions/download-artifact@v3 - with: - path: . - name: adf-cli - ## ADF TESTING - - name: update Dist folder - uses: actions/download-artifact@v3 - with: - path: . - name: adf-testing - - name: extract nxcache - run: | - tar xzf nxcache.tar.gz - shell: bash - - name: extract dist - run: | - tar xzf dist.tar.gz - shell: bash - - name: extract adf cli - run: | - tar xzf adf-cli.tar.gz - tar xzf adf-testing.tar.gz - shell: bash - - name: show files - shell: bash - run: | - pwd - ls -lha - echo "====DIST====" - find dist -maxdepth 1 -type d - echo "====NXCACHE====" - find nxcache -maxdepth 1 -type d - echo "====ADF====" - find node_modules/@alfresco/ -maxdepth 1 -type d \ No newline at end of file diff --git a/.github/actions/download-cache-and-artifacts/action.yml b/.github/actions/download-cache-and-artifacts/action.yml new file mode 100644 index 0000000000..a61404aa3a --- /dev/null +++ b/.github/actions/download-cache-and-artifacts/action.yml @@ -0,0 +1,27 @@ +name: "Download build artifacts" +description: "Download build artifacts" +runs: + using: "composite" + steps: + - name: download and extract artifacts from s3 + shell: bash + env: + REMOTE_PATH: "alfresco-ng2-components/build-cache/${{ github.run_id }}" + run: | + packages=( dist nxcache node_modules ) + for i in "${packages[@]}"; do + time aws s3 cp --no-progress s3://${S3_BUILD_BUCKET_SHORT_NAME}/${REMOTE_PATH}/$i.tar.gz $i.tar.gz + du -h $i.tar.gz + time tar xzf $i.tar.gz + done + - name: show files + shell: bash + run: | + pwd + ls -lha + echo "====DIST====" + find dist -maxdepth 1 -type d + echo "====NXCACHE====" + find nxcache -maxdepth 1 -type d + echo "====ADF====" + find node_modules/@alfresco/ -maxdepth 1 -type d \ No newline at end of file diff --git a/.github/actions/e2e/action.yml b/.github/actions/e2e/action.yml index 0d21e67395..f3e4b19491 100644 --- a/.github/actions/e2e/action.yml +++ b/.github/actions/e2e/action.yml @@ -129,6 +129,7 @@ runs: echo "NX_CALCULATION_FLAGS=$NX_CALCULATION_FLAGS" >> $GITHUB_ENV GIT_HASH=$(echo $GIT_HASH | sed "s@'@@g") echo "GIT_HASH=$GIT_HASH" >> $GITHUB_ENV + - name: run test with retries id: retry_run env: @@ -136,6 +137,7 @@ runs: PROVIDER: "${{ inputs.e2e-test-provider }}" AUTH_TYPE: "${{ inputs.e2e-test-auth }}" E2E_TEST_ID: "${{ inputs.e2e-test-id }}" + uses: nick-fields/retry@v2.8.2 with: timeout_minutes: 40 @@ -145,9 +147,15 @@ runs: command: | set -u; export GH_ACTION_RETRY_COUNT=$(cat ${GITHUB_OUTPUT} | grep -E '^[0-9]{1,2}$' | tail -n1) - echo "RETRY GH_ACTION_RETRY_COUNT = <$GH_ACTION_RETRY_COUNT>" - bash ./scripts/travis/e2e/e2e.sh "$E2E_TEST_ID" || exit 1 - + echo "RETRY GH_ACTION_RETRY_COUNT = <$GH_ACTION_RETRY_COUNT>" + if [[ ${{ inputs.e2e-test-folder }} == 'content-services/upload' ]]; then + export DISPLAY=:99 + chromedriver --url-base=/wd/hub & + sudo Xvfb -ac :99 -screen 0 1280x1024x24 > /dev/null 2>&1 & # optional + bash ./scripts/travis/e2e/e2e.sh "$E2E_TEST_ID" "browser" || exit 1 + else + bash ./scripts/travis/e2e/e2e.sh "$E2E_TEST_ID" || exit 1 + fi - name: upload smartrunner tests results on s3 to cache tests shell: bash if: always() @@ -157,5 +165,3 @@ runs: run: | tar czf "${{ inputs.e2e-tar-name }}" "${SMART_RUNNER_PATH}" aws s3 cp "${{ inputs.e2e-tar-name }}" "${S3_BUILD_BUCKET}/adf/$REMOTE_PATH" - - diff --git a/.github/actions/enable-dryrun/action.yml b/.github/actions/enable-dryrun/action.yml new file mode 100644 index 0000000000..79e52df659 --- /dev/null +++ b/.github/actions/enable-dryrun/action.yml @@ -0,0 +1,30 @@ +name: 'enable dryrun' +description: 'check if must run pipeline in dryrun mode' + +inputs: + dry-run-flag: + description: 'enable dryrun' + required: false + type: boolean + default: true + +outputs: + dryrun: + description: "long sha of the tag" + value: ${{ steps.dryrun.outputs.dryrun }} + +runs: + using: "composite" + + steps: + - name: set dryrun flag to TRUE + shell: bash + id: dryrun + run: | + if [[ '${{ inputs.dry-run-flag }}' == 'true' ]]; then + echo "dryrun=--dryrun" >> $GITHUB_OUTPUT; + echo "enabling dryrun" + else + echo "dryrun=" >> $GITHUB_OUTPUT; + echo "dryrun NOT enabled" + fi diff --git a/.github/actions/npm-check-bundle/action.yml b/.github/actions/npm-check-bundle/action.yml new file mode 100644 index 0000000000..fbabf21036 --- /dev/null +++ b/.github/actions/npm-check-bundle/action.yml @@ -0,0 +1,19 @@ +name: 'Check NPM bundle' +description: 'check npm bundle' + +outputs: + tag_sha: + description: "long sha of the tag" + value: ${{ steps.sha_out.outputs.tag_sha }} + +runs: + using: "composite" + + steps: + - name: check npm bundle + shell: bash + id: sha_out + run: | + ADF_VERSION=$(npm view @alfresco/adf-core@${TAG_NPM} version) + echo "check bundle on TAG_NPM='${TAG_NPM}' and ADF_VERSION='${ADF_VERSION}'" + ./scripts/travis/build/npm-check-bundles.sh -v ${ADF_VERSION} \ No newline at end of file diff --git a/.github/actions/setup-chrome/action.yml b/.github/actions/setup-chrome/action.yml new file mode 100644 index 0000000000..cddbc7b2cb --- /dev/null +++ b/.github/actions/setup-chrome/action.yml @@ -0,0 +1,13 @@ +name: 'Install Google Chrome' +description: 'Install Google Chrome' + +runs: + using: "composite" + steps: + - name: Install google chrome + shell: bash + run: | + wget -q https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb + sudo apt install ./google-chrome-stable_current_amd64.deb + sudo ln -s /usr/bin/google-chrome /usr/bin/chrome + chrome --version \ No newline at end of file diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index ef83280bec..629f3a9c43 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -27,7 +27,6 @@ runs: cache-name: node-cache with: path: | - node_modules ~/.npm nxcache dist @@ -36,6 +35,20 @@ runs: node-${{ runner.os }}-build-${{ env.cache-name }}- node-${{ runner.os }}-build- node-${{ runner.os }}- + - name: Node Modules cache + id: node-modules-cache + if: ${{ inputs.enable-node-modules-cache == 'true' }} + uses: actions/cache@v3 + env: + cache-name: node-modules-cache + with: + path: | + node_modules + key: .npm-${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }}-${{ steps.tag-sha.outputs.tag_sha }} + restore-keys: | + node_modules-${{ runner.os }}-build-${{ env.cache-name }}- + node_modules-${{ runner.os }}-build- + node_modules-${{ runner.os }}- - name: pip cache uses: actions/cache@v3 if: ${{ inputs.enable-cache == 'true' }} @@ -43,7 +56,7 @@ runs: path: ~/.cache/pip key: ${{ runner.os }}-pip- restore-keys: | - ${{ runner.os }} + ${{ runner.os }} # ENV LOAD - name: load .travis.yml env.global variables uses: Alfresco/alfresco-build-tools/.github/actions/travis-env-load@v1.17.0 diff --git a/.github/actions/travis-env-var-facade/action.yml b/.github/actions/travis-env-var-facade/action.yml index 068f68307c..beb895419e 100644 --- a/.github/actions/travis-env-var-facade/action.yml +++ b/.github/actions/travis-env-var-facade/action.yml @@ -16,6 +16,7 @@ runs: EVENT_TYPE="" TRAVIS_PULL_REQUEST_BRANCH="" TRAVIS_BRANCH="" + TAG_NPM="" REGEX="(repository|workflow)_dispatch" if [[ "${{ inputs.event_name }}" == "schedule" ]]; then EVENT_TYPE="cron"; @@ -29,6 +30,14 @@ runs: elif [[ "${{ inputs.event_name }}" =~ $REGEX ]]; then EVENT_TYPE="api"; fi + + if [[ GITHUB_REF_NAME == "master" ]]; then + TAG_NPM="latest" + fi + if [[ GITHUB_REF_NAME == "develop" ]]; then + TAG_NPM="alpha" + fi + echo "TAG_NPM=${TAG_NPM}" >> $GITHUB_ENV echo "TRAVIS_EVENT_TYPE=${EVENT_TYPE}" >> $GITHUB_ENV echo "TRAVIS_PULL_REQUEST_BRANCH=${TRAVIS_PULL_REQUEST_BRANCH}" >> $GITHUB_ENV echo "TRAVIS_BRANCH=${TRAVIS_BRANCH}" >> $GITHUB_ENV diff --git a/.github/actions/upload-build-artifacts/action.yml b/.github/actions/upload-build-artifacts/action.yml deleted file mode 100644 index 487e9b590c..0000000000 --- a/.github/actions/upload-build-artifacts/action.yml +++ /dev/null @@ -1,41 +0,0 @@ -name: "Upload build artifacts" -description: "Upload build artifacts" - -runs: - using: "composite" - steps: - ## NXCACHE - - name: tar artifacts - run: | - tar czf dist.tar.gz dist - tar czf nxcache.tar.gz nxcache - tar czf adf-cli.tar.gz node_modules/@alfresco/adf-cli - tar czf adf-testing.tar.gz node_modules/@alfresco/adf-testing - shell: bash - - name: update NX cache folder - uses: actions/upload-artifact@v3 - continue-on-error: true - with: - path: nxcache.tar.gz - name: nxcache - ## DIST - - name: update Dist folder - uses: actions/upload-artifact@v3 - continue-on-error: true - with: - path: dist.tar.gz - name: dist - ## ADF CLI - - name: save ADF cli - uses: actions/upload-artifact@v3 - continue-on-error: true - with: - path: adf-cli.tar.gz - name: adf-cli - ## ADF TESTING - - name: save ADF testing - uses: actions/upload-artifact@v3 - continue-on-error: true - with: - path: adf-testing.tar.gz - name: adf-testing diff --git a/.github/actions/upload-cache-and-artifacts/action.yml b/.github/actions/upload-cache-and-artifacts/action.yml new file mode 100644 index 0000000000..a6c769bdf6 --- /dev/null +++ b/.github/actions/upload-cache-and-artifacts/action.yml @@ -0,0 +1,19 @@ +name: "Upload build artifacts" +description: "Upload build artifacts" + +runs: + using: "composite" + steps: + - name: tar and upload artifacts + shell: bash + env: + REMOTE_PATH: "alfresco-ng2-components/build-cache/${{ github.run_id }}" + run: | + packages=( dist nxcache node_modules ) + for i in "${packages[@]}"; do + time tar czf $i.tar.gz $i + du -h $i.tar.gz + time aws s3 cp --no-progress $i.tar.gz s3://${S3_BUILD_BUCKET_SHORT_NAME}/${REMOTE_PATH}/$i.tar.gz + done + + diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml index c613fc9b48..01faa0b804 100644 --- a/.github/workflows/pull-request.yml +++ b/.github/workflows/pull-request.yml @@ -1,6 +1,13 @@ name: "pull-request" on: + workflow_call: + inputs: + dry-run-release: + description: 'enable dry-run on artifact push' + required: false + type: boolean + default: true pull_request: types: [opened, synchronize, reopened] branches: @@ -49,7 +56,14 @@ env: SMART_RUNNER_PATH: ".protractor-smartrunner" S3_DBP_PATH: ${{ secrets.S3_DBP_PATH }} S3_BUILD_BUCKET: ${{ secrets.S3_BUILD_BUCKET }} + S3_BUILD_BUCKET_SHORT_NAME: ${{ secrets.S3_BUILD_BUCKET_SHORT_NAME }} NODE_OPTIONS: "--max-old-space-size=5120" + DOCKER_REPOSITORY_DOMAIN: ${{ secrets.DOCKER_REPOSITORY_DOMAIN }} + DOCKER_REPOSITORY_USER: ${{ secrets.DOCKER_REPOSITORY_USER }} + DOCKER_REPOSITORY_PASSWORD: ${{ secrets.DOCKER_REPOSITORY_PASSWORD }} + DOCKER_REPOSITORY_STORYBOOK: "${{ secrets.DOCKER_REPOSITORY_DOMAIN }}/alfresco/storybook" + DOCKER_REPOSITORY: "${{ secrets.DOCKER_REPOSITORY_DOMAIN }}/alfresco/demo-shell" + jobs: main: # long timeout required when cache has to be recreated @@ -72,7 +86,7 @@ jobs: - run: nx affected:build $NX_CALCULATION_FLAGS --prod - run: nx build demoshell --configuration production - run: nx run stories:build-storybook --configuration ci - - uses: ./.github/actions/upload-build-artifacts + - uses: ./.github/actions/upload-cache-and-artifacts playwright: timeout-minutes: 20 @@ -85,20 +99,13 @@ jobs: with: fetch-depth: 0 # Fetch all history for all - uses: ./.github/actions/setup - - name: clean - run: | - rm -rf nxcache - rm -rf dist - - uses: ./.github/actions/download-build-artifacts - - name: build adf cli - run: | - nx build cli + - uses: ./.github/actions/download-cache-and-artifacts - name: Process Cloud Storybook Playwright run: | npx playwright install chromium sudo sysctl -w fs.inotify.max_user_watches=524288 npx playwright test --config='e2e-playwright/playwright.config.ts' || exit 1 - - uses: ./.github/actions/upload-build-artifacts + - uses: ./.github/actions/upload-cache-and-artifacts e2e: timeout-minutes: 90 @@ -142,7 +149,7 @@ jobs: auth: "BASIC" check-cs-env: "true" - description: "Content: Upload and Versioning" - test-id: "content-services browser" + test-id: "content-services" folder: "content-services/upload" provider: "ECM" auth: "BASIC" @@ -230,11 +237,10 @@ jobs: - uses: ./.github/actions/setup with: enable-cache: "true" - - name: clean - run: | - rm -rf nxcache - rm -rf dist - - uses: ./.github/actions/download-build-artifacts + enable-node-modules-cache: "true" + - uses: ./.github/actions/download-cache-and-artifacts + - name: setup chrome + uses: ./.github/actions/setup-chrome - name: e2e uses: ./.github/actions/e2e with: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b69add0df6..19d8b745a4 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,6 +1,13 @@ name: "release" on: + workflow_call: + inputs: + dry-run-release: + description: 'enable dry-run on artifact push' + required: false + type: boolean + default: true pull_request: types: [closed] branches: @@ -10,6 +17,8 @@ on: - master-patch* env: + BASE_REF: ${{ github.base_ref }} + HEAD_REF: ${{ github.head_ref }} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} GITHUB_BRANCH: ${{ github.ref_name }} @@ -17,7 +26,7 @@ env: TRAVIS_COMMIT: ${{ github.sha }} BUILD_ID: ${{ github.run_id }} TRAVIS_RUN_NUMBER: ${{ github.run_attempt }} - TRAVIS_BUILD_NUMBER: github-run-${{ github.run_id }} + TRAVIS_BUILD_NUMBER: ${{ github.run_id }} JOB_ID: ${{ github.run_id }} PROXY_HOST_BPM: ${{ secrets.E2E_HOST }} E2E_HOST_APA: ${{ secrets.E2E_HOST_APA }} @@ -44,20 +53,23 @@ env: PASSWORD_SUPER_ADMIN_ADF: ${{ secrets.PASSWORD_SUPER_ADMIN_ADF }} HR_USER: ${{ secrets.HR_USER }} HR_USER_PASSWORD: ${{ secrets.HR_USER_PASSWORD }} - NPM_REGISTRY_ADDRESS: ${{ secrets.NPM_REGISTRY_ADDRESS }} - NPM_REGISTRY_TOKEN: ${{ secrets.NPM_REGISTRY_TOKEN }} + SMART_RUNNER_PATH: ".protractor-smartrunner" + S3_DBP_PATH: ${{ secrets.S3_DBP_PATH }} + S3_BUILD_BUCKET: ${{ secrets.S3_BUILD_BUCKET }} + S3_BUILD_BUCKET_SHORT_NAME: ${{ secrets.S3_BUILD_BUCKET_SHORT_NAME }} + NODE_OPTIONS: "--max-old-space-size=5120" + DOCKER_REPOSITORY_DOMAIN: ${{ secrets.DOCKER_REPOSITORY_DOMAIN }} DOCKER_REPOSITORY_USER: ${{ secrets.DOCKER_REPOSITORY_USER }} DOCKER_REPOSITORY_PASSWORD: ${{ secrets.DOCKER_REPOSITORY_PASSWORD }} - DOCKER_REPOSITORY_DOMAIN: ${{ secrets.DOCKER_REPOSITORY_DOMAIN }} DOCKER_REPOSITORY_STORYBOOK: "${{ secrets.DOCKER_REPOSITORY_DOMAIN }}/alfresco/storybook" DOCKER_REPOSITORY: "${{ secrets.DOCKER_REPOSITORY_DOMAIN }}/alfresco/demo-shell" - NODE_OPTIONS: "--max-old-space-size=5120" + jobs: - build: + setup: timeout-minutes: 20 - if: github.event.pull_request.merged == true - name: "Build Components" + if: github.event.pull_request.merged || ${{ inputs.dry-run-release }} + name: "Setup" runs-on: ubuntu-22.04 steps: - name: Checkout repository @@ -65,20 +77,17 @@ jobs: with: fetch-depth: 0 - uses: ./.github/actions/setup - - run: npm -v - name: install run: | npm ci nx run cli:bundle nx run testing:bundle - - run: npx nx affected --target=lint $NX_CALCULATION_FLAGS - - run: nx affected:build $NX_CALCULATION_FLAGS --prod --exclude="demoshell" - - uses: ./.github/actions/upload-build-artifacts + - uses: ./.github/actions/upload-cache-and-artifacts release-demoshell: - needs: [release-npm] + needs: [setup] timeout-minutes: 15 - if: github.event.pull_request.merged == true + if: github.event.pull_request.merged || ${{ inputs.dry-run-release }} runs-on: ubuntu-22.04 steps: - name: Checkout repository @@ -86,48 +95,51 @@ jobs: with: fetch-depth: 1 - run: git fetch --all + - id: set-dryrun + uses: ./.github/actions/enable-dryrun + with: + dry-run-flag: ${{ inputs.dry-run-release }} - uses: ./.github/actions/setup with: - enable-cache: false - - name: install nx and cli - run: | - npm ci - nx run cli:bundle - - name: push Demoshell docker image + enable-cache: false + enable-node-modules-cache: false + - uses: ./.github/actions/download-cache-and-artifacts + - name: release Demoshell docker run: | nx build demoshell --configuration production . ./scripts/travis/release/docker-tag.sh - echo $TAGS - ./scripts/travis/release/release-demoshell-docker.sh + ./scripts/travis/release/release-demoshell-docker.sh ${{ steps.set-dryrun.outputs.dryrun }} release-storybook: - needs: [release-npm] + needs: [setup] timeout-minutes: 15 - if: github.event.pull_request.merged == true + if: github.event.pull_request.merged || ${{ inputs.dry-run-release }} runs-on: ubuntu-22.04 steps: - name: Checkout repository uses: actions/checkout@v3 with: - fetch-depth: 0 + fetch-depth: 1 + - run: git fetch --all + - id: set-dryrun + uses: ./.github/actions/enable-dryrun + with: + dry-run-flag: ${{ inputs.dry-run-release }} - uses: ./.github/actions/setup with: enable-cache: false - - name: install nx and cli - run: | - npm ci - nx run cli:bundle - - name: push Storybook docker image + enable-node-modules-cache: false + - uses: ./.github/actions/download-cache-and-artifacts + - name: release Storybook docker run: | nx run stories:build-storybook --configuration ci - . ./scripts/travis/release/docker-tag.sh - echo $TAGS - ./scripts/travis/release/release-storybook-docker.sh + . ./scripts/travis/release/docker-tag.sh + ./scripts/travis/release/release-storybook-docker.sh ${{ steps.set-dryrun.outputs.dryrun }} release-npm: - needs: [build] + needs: [setup] timeout-minutes: 15 - if: github.event.pull_request.merged == true + if: github.event.pull_request.merged == true || ${{ inputs.dry-run-release }} runs-on: ubuntu-22.04 steps: - name: Checkout repository @@ -137,23 +149,25 @@ jobs: - uses: ./.github/actions/setup with: enable-cache: false - - name: install nx and cli - run: | - npm ci - nx run cli:bundle - - name: nx build + enable-node-modules-cache: false + - id: set-dryrun + uses: ./.github/actions/enable-dryrun + with: + dry-run-flag: ${{ inputs.dry-run-release }} + - uses: ./.github/actions/download-cache-and-artifacts + - name: build libraries run: | set -u; ./scripts/travis/build/bumpversion.sh nx affected:build $NX_CALCULATION_FLAGS --prod --exclude="demoshell" nx affected $NX_CALCULATION_FLAGS --target=pretheme - - name: release - run: ./scripts/travis/release/release-npm.sh + - name: release libraries + run: ./scripts/travis/release/release-npm.sh ${{ steps.set-dryrun.outputs.dryrun }} npm-check-bundle: needs: [release-npm] timeout-minutes: 15 - if: github.event.pull_request.merged == true + if: github.event.pull_request.merged == true || ${{ inputs.dry-run-release }} runs-on: ubuntu-22.04 steps: - name: Checkout repository