From 48f02a77cac13c24d1cfe9ccd8d520ed37d973ba Mon Sep 17 00:00:00 2001 From: Marco Carrozzo Date: Mon, 23 Jan 2023 11:31:49 +0100 Subject: [PATCH] dev --- .../download-cache-and-artifacts/action.yml | 59 +++++++++++--- .github/actions/npm-check-bundle/action.yml | 2 +- .github/actions/setup/action.yml | 45 +++++------ .../upload-cache-and-artifacts/action.yml | 42 ++++++++-- .github/workflows/devel.yml | 69 +++++++++++++++++ .github/workflows/pull-request.yml | 77 +++++++++++++++---- .github/workflows/release.yml | 24 ++++-- 7 files changed, 253 insertions(+), 65 deletions(-) create mode 100644 .github/workflows/devel.yml diff --git a/.github/actions/download-cache-and-artifacts/action.yml b/.github/actions/download-cache-and-artifacts/action.yml index a61404aa3a..79ea01f5d1 100644 --- a/.github/actions/download-cache-and-artifacts/action.yml +++ b/.github/actions/download-cache-and-artifacts/action.yml @@ -1,27 +1,66 @@ name: "Download build artifacts" description: "Download build artifacts" + +inputs: + packages: + description: 'packages to cache across the current run' + required: false + type: string + persistent-packages: + description: 'packages to cache in s3' + required: false + type: string + packages-key-suffix: + description: 'cache key suffix' + required: false + type: string + default: '' runs: using: "composite" steps: - - name: download and extract artifacts from s3 + - name: download and extract current run cache artifacts from s3 shell: bash env: - REMOTE_PATH: "alfresco-ng2-components/build-cache/${{ github.run_id }}" + REMOTE_PATH: "adf/build-cache/${{ github.run_id }}" run: | - packages=( dist nxcache node_modules ) + packages=( $(echo ${{ inputs.packages }}) ) for i in "${packages[@]}"; do - time aws s3 cp --no-progress s3://${S3_BUILD_BUCKET_SHORT_NAME}/${REMOTE_PATH}/$i.tar.gz $i.tar.gz + echo "downloading current run cache artifacts from ${REMOTE_PATH}/$i.tar.gz" + aws s3 cp --no-progress s3://${S3_BUILD_BUCKET_SHORT_NAME}/${REMOTE_PATH}/$i.tar.gz $i.tar.gz du -h $i.tar.gz - time tar xzf $i.tar.gz + tar xzf $i.tar.gz done + - name: download and extract persistent cache artifacts from s3 + shell: bash + run: | + HASH=$(cat $GITHUB_WORKSPACE/package-lock.json | shasum | head -c 40) + REMOTE_PATH="adf/build-cache/${HASH}" + if [ -n "${{ inputs.packages-key-suffix }}" ]; then + REMOTE_PATH="${REMOTE_PATH}/${{ inputs.packages-key-suffix }}" + fi + packages=( $(echo ${{ inputs.persistent-packages }}) ) + set +e; + for i in "${packages[@]}"; do + REMOTE_FILE_PATH="s3://${S3_BUILD_BUCKET_SHORT_NAME}/${REMOTE_PATH}/$i.tar.gz" + aws s3 ls $REMOTE_FILE_PATH > /dev/null + if [ $? -eq 0 ]; then + echo "downloading persistent cache artifacts from $REMOTE_FILE_PATH" + aws s3 cp --no-progress $REMOTE_FILE_PATH $i.tar.gz + echo "package $i size: $(du -h $i.tar.gz)" + tar xzf $i.tar.gz + else + echo "persistent cache artifact file not found on $REMOTE_FILE_PATH. continue." + fi + set -e; + done - name: show files shell: bash run: | - pwd - ls -lha echo "====DIST====" - find dist -maxdepth 1 -type d + find dist -maxdepth 1 -type d || true echo "====NXCACHE====" - find nxcache -maxdepth 1 -type d + find nxcache -maxdepth 1 -type d || true echo "====ADF====" - find node_modules/@alfresco/ -maxdepth 1 -type d \ No newline at end of file + find node_modules/@alfresco/ -maxdepth 1 -type d || true + echo "====NX====" + find node_modules/nx/ -maxdepth 1 -type d || true diff --git a/.github/actions/npm-check-bundle/action.yml b/.github/actions/npm-check-bundle/action.yml index edd2491b5b..558f09bff2 100644 --- a/.github/actions/npm-check-bundle/action.yml +++ b/.github/actions/npm-check-bundle/action.yml @@ -25,4 +25,4 @@ runs: fi ADF_VERSION=$(npm view @alfresco/adf-core@${TAG_NPM} version) echo "check bundle on TAG_NPM='${TAG_NPM}' and ADF_VERSION='${ADF_VERSION}'" - ./scripts/travis/build/npm-check-bundles.sh -v ${ADF_VERSION} \ No newline at end of file + ./scripts/travis/build/npm-check-bundles.sh -v ${ADF_VERSION} diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index 5652ccf7c6..eecf689c2a 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -1,12 +1,17 @@ name: 'Setup' description: 'Initialize cache, env var load' inputs: - enable-cache: + cache-key-suffix: + description: 'suffix for cache key' + required: false + type: string + default: '' + use-gh-nx-cache: description: 'enable caching' required: false type: boolean default: 'true' - enable-node-modules-cache: + use-gh-node-modules-cache: description: 'enable caching for node modules' required: false type: boolean @@ -19,6 +24,7 @@ inputs: runs: using: "composite" steps: + - name: install NPM uses: actions/setup-node@v3 with: @@ -28,44 +34,31 @@ runs: id: tag-sha uses: Alfresco/alfresco-build-tools/.github/actions/git-latest-tag@v1.29.0 # CACHE - - name: Node cache - id: node-cache - if: ${{ inputs.enable-cache == 'true' }} + - name: NX cache + id: nx-cache + if: ${{ inputs.use-gh-nx-cache == 'true' }} uses: actions/cache@v3 env: - cache-name: node-cache + cache-name: nx-cache with: path: | - ~/.npm nxcache - dist - key: .npm-${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }}-${{ steps.tag-sha.outputs.tag_sha }} + key: nxcache-${{ inputs.cache-key-suffix }}-${{ hashFiles('**/package-lock.json') }} restore-keys: | - node-${{ runner.os }}-build-${{ env.cache-name }}- - node-${{ runner.os }}-build- - node-${{ runner.os }}- + nxcache-build-${{ inputs.cache-key-suffix }} + nxcache- - name: Node Modules cache id: node-modules-cache - if: ${{ inputs.enable-node-modules-cache == 'true' }} + if: ${{ inputs.use-gh-node-modules-cache == 'true' }} uses: actions/cache@v3 env: cache-name: node-modules-cache with: path: | node_modules - key: .npm-${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }}-${{ steps.tag-sha.outputs.tag_sha }} + key: ${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }} restore-keys: | - node_modules-${{ runner.os }}-build-${{ env.cache-name }}- - node_modules-${{ runner.os }}-build- - node_modules-${{ runner.os }}- - - name: pip cache - uses: actions/cache@v3 - if: ${{ inputs.enable-cache == 'true' }} - with: - path: ~/.cache/pip - key: ${{ runner.os }}-pip- - restore-keys: | - ${{ runner.os }} + ${{ env.cache-name }}- # ENV LOAD - name: load .travis.yml env.global variables uses: Alfresco/alfresco-build-tools/.github/actions/travis-env-load@v1.17.0 @@ -79,4 +72,4 @@ runs: act: ${{ inputs.act }} - name: link nx executable run: sudo ln -s $(npm bin)/nx /usr/bin/nx - shell: bash + shell: bash \ No newline at end of file diff --git a/.github/actions/upload-cache-and-artifacts/action.yml b/.github/actions/upload-cache-and-artifacts/action.yml index a6c769bdf6..15bfa9ebc6 100644 --- a/.github/actions/upload-cache-and-artifacts/action.yml +++ b/.github/actions/upload-cache-and-artifacts/action.yml @@ -1,19 +1,49 @@ name: "Upload build artifacts" description: "Upload build artifacts" - +inputs: + packages: + description: 'packages to cache across the current run' + required: false + type: string + persistent-packages: + description: 'packages to cache in s3' + required: false + type: string + packages-key-suffix: + description: 'cache key suffix' + required: false + type: string + default: '' runs: using: "composite" steps: - - name: tar and upload artifacts + - name: tar and upload current run cache artifacts shell: bash env: - REMOTE_PATH: "alfresco-ng2-components/build-cache/${{ github.run_id }}" + REMOTE_PATH: "adf/build-cache/${{ github.run_id }}" run: | - packages=( dist nxcache node_modules ) + packages=( $(echo ${{ inputs.packages }}) ) for i in "${packages[@]}"; do - time tar czf $i.tar.gz $i + echo "processing $i" + tar czf $i.tar.gz $i + echo "package size: $(du -h $i.tar.gz)" + echo "uploading current run artifacts to ${REMOTE_PATH}/$i.tar.gz" + aws s3 cp --no-progress $i.tar.gz s3://${S3_BUILD_BUCKET_SHORT_NAME}/${REMOTE_PATH}/$i.tar.gz + done + - name: tar and upload persistent cache artifacts + shell: bash + run: | + HASH=$(cat $GITHUB_WORKSPACE/package-lock.json | shasum | head -c 40) + REMOTE_PATH="adf/build-cache/${HASH}" + if [ -n "${{ inputs.packages-key-suffix }}" ]; then + REMOTE_PATH="${REMOTE_PATH}/${{ inputs.packages-key-suffix }}" + fi + packages=( $(echo ${{ inputs.persistent-packages }}) ) + for i in "${packages[@]}"; do + tar czf $i.tar.gz $i du -h $i.tar.gz - time aws s3 cp --no-progress $i.tar.gz s3://${S3_BUILD_BUCKET_SHORT_NAME}/${REMOTE_PATH}/$i.tar.gz + echo "pushing persistent cache to $REMOTE_PATH" + aws s3 cp --no-progress $i.tar.gz s3://${S3_BUILD_BUCKET_SHORT_NAME}/${REMOTE_PATH}/$i.tar.gz done diff --git a/.github/workflows/devel.yml b/.github/workflows/devel.yml new file mode 100644 index 0000000000..f8d3f1355c --- /dev/null +++ b/.github/workflows/devel.yml @@ -0,0 +1,69 @@ +name: "devel" + +on: + push: + branches: + - AAE-12190* + +env: + BASE_REF: ${{ github.base_ref }} + HEAD_REF: ${{ github.head_ref }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + GITHUB_BRANCH: ${{ github.ref_name }} + TRAVIS_BUILD_DIR: ${{ github.workspace }} + TRAVIS_COMMIT: ${{ github.sha }} + BUILD_ID: ${{ github.run_id }} + TRAVIS_RUN_NUMBER: ${{ github.run_attempt }} + TRAVIS_BUILD_NUMBER: ${{ github.run_id }} + JOB_ID: ${{ github.run_id }} + PROXY_HOST_BPM: ${{ secrets.E2E_HOST }} + E2E_HOST_APA: ${{ secrets.E2E_HOST_APA }} + E2E_HOST: ${{ secrets.E2E_HOST }} + E2E_USERNAME: ${{ secrets.E2E_ADMIN_EMAIL_IDENTITY }} + E2E_PASSWORD: ${{ secrets.E2E_PASSWORD }} + E2E_ADMIN_EMAIL_IDENTITY: ${{ secrets.E2E_ADMIN_EMAIL_IDENTITY }} + E2E_ADMIN_PASSWORD_IDENTITY: ${{ secrets.E2E_ADMIN_PASSWORD_IDENTITY }} + USERNAME_ADF: ${{ secrets.E2E_USERNAME }} + PASSWORD_ADF: ${{ secrets.E2E_PASSWORD }} + URL_HOST_ADF: ${{ secrets.URL_HOST_ADF }} + IDENTITY_ADMIN_EMAIL: ${{ secrets.E2E_ADMIN_EMAIL_IDENTITY }} + IDENTITY_ADMIN_PASSWORD: ${{ secrets.E2E_ADMIN_PASSWORD_IDENTITY }} + AWS_S3_BUCKET_ACTIVITI_LICENSE: ${{ secrets.AWS_S3_BUCKET_ACTIVITI_LICENSE }} + HOST_SSO: ${{ secrets.HOST_SSO }} + LOG_LEVEL: "ERROR" + E2E_LOG_LEVEL: "ERROR" + E2E_MODELER_USERNAME: ${{ secrets.E2E_MODELER_USERNAME }} + E2E_MODELER_PASSWORD: ${{ secrets.E2E_MODELER_PASSWORD }} + EXTERNAL_ACS_HOST: ${{ secrets.EXTERNAL_ACS_HOST }} + E2E_DEVOPS_USERNAME: ${{ secrets.E2E_DEVOPS_USERNAME }} + E2E_DEVOPS_PASSWORD: ${{ secrets.E2E_DEVOPS_PASSWORD }} + USERNAME_SUPER_ADMIN_ADF: ${{ secrets.USERNAME_SUPER_ADMIN_ADF }} + PASSWORD_SUPER_ADMIN_ADF: ${{ secrets.PASSWORD_SUPER_ADMIN_ADF }} + HR_USER: ${{ secrets.HR_USER }} + HR_USER_PASSWORD: ${{ secrets.HR_USER_PASSWORD }} + SMART_RUNNER_PATH: ".protractor-smartrunner" + S3_DBP_PATH: ${{ secrets.S3_DBP_PATH }} + S3_BUILD_BUCKET: ${{ secrets.S3_BUILD_BUCKET }} + S3_BUILD_BUCKET_SHORT_NAME: ${{ secrets.S3_BUILD_BUCKET_SHORT_NAME }} + NODE_OPTIONS: "--max-old-space-size=5120" + DOCKER_REPOSITORY_DOMAIN: ${{ secrets.DOCKER_REPOSITORY_DOMAIN }} + DOCKER_REPOSITORY_USER: ${{ secrets.DOCKER_REPOSITORY_USER }} + DOCKER_REPOSITORY_PASSWORD: ${{ secrets.DOCKER_REPOSITORY_PASSWORD }} + DOCKER_REPOSITORY_STORYBOOK: "${{ secrets.DOCKER_REPOSITORY_DOMAIN }}/alfresco/storybook" + DOCKER_REPOSITORY: "${{ secrets.DOCKER_REPOSITORY_DOMAIN }}/alfresco/demo-shell" + +jobs: + devel: + uses: ./.github/workflows/pull-request.yml + secrets: inherit + with: + dry-run-release: true + skip-pr-approve: true + + # release: + # uses: ./.github/workflows/release.yml + # secrets: inherit + # with: + # dry-run-release: true + diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml index 5be49dbf96..18fb846699 100644 --- a/.github/workflows/pull-request.yml +++ b/.github/workflows/pull-request.yml @@ -7,7 +7,12 @@ on: description: 'enable dry-run on artifact push' required: false type: boolean - default: true + default: true + skip-pr-approve: + description: 'skip pr approve check' + required: false + type: boolean + default: false pull_request: types: [opened, synchronize, reopened] branches: @@ -83,7 +88,7 @@ jobs: - name: Show the number env: GH_TOKEN: ${{ github.token }} - DEVEL_FLAG: false #put it to true if you need to develop the pipeline on your own branch. before the final merge put it to false + DEVEL_FLAG: ${{ inputs.skip-pr-approve }} #put it to true if you need to develop the pipeline on your own branch. before the final merge put it to false shell: bash if: ${{ github.event_name != 'schedule' }} run: | @@ -113,13 +118,18 @@ jobs: with: fetch-depth: 0 # Fetch all history for all tags and branches - uses: ./.github/actions/setup - - name: install - run: | - npm ci - nx run cli:bundle - nx run testing:bundle - - run: nx print-affected $NX_CALCULATION_FLAGS + id: setup + with: + use-gh-nx-cache: false + use-gh-node-modules-cache: false + - name: run ci + if: steps.setup.outputs.cache-hit != 'true' + run: npm ci + - run: nx run cli:bundle + - run: nx run testing:bundle - uses: ./.github/actions/upload-cache-and-artifacts + with: + packages: 'dist node_modules' unit-tests: timeout-minutes: 30 @@ -145,11 +155,22 @@ jobs: with: fetch-depth: 0 # Fetch all history for all tags and branches - uses: ./.github/actions/setup + with: + use-gh-nx-cache: false + use-gh-node-modules-cache: true + cache-key-suffix: unit-tests-${{ matrix.unit-tests.name }} - uses: ./.github/actions/download-cache-and-artifacts + with: + persistent-packages: 'nxcache' + packages-key-suffix: ${{ github.ref_name }}/${{ matrix.unit-tests.name }} - name: Run unit tests run: | /usr/bin/xvfb-run --auto-servernum nx affected:test $NX_CALCULATION_FLAGS --exclude=${{ matrix.unit-tests.exclude }} - + - uses: ./.github/actions/upload-cache-and-artifacts + with: + persistent-packages: 'nxcache' + packages-key-suffix: ${{ github.ref_name }}/${{ matrix.unit-tests.name }} + lint: # long timeout required when cache has to be recreated timeout-minutes: 30 @@ -162,8 +183,19 @@ jobs: with: fetch-depth: 0 # Fetch all history for all tags and branches - uses: ./.github/actions/setup + with: + use-gh-nx-cache: false + use-gh-node-modules-cache: false + cache-key-suffix: ${{ github.ref_name }}-lint - uses: ./.github/actions/download-cache-and-artifacts + with: + packages: 'node_modules' + persistent-packages: 'nxcache' - run: nx affected --target=lint $NX_CALCULATION_FLAGS + - uses: ./.github/actions/upload-cache-and-artifacts + with: + persistent-packages: 'nxcache' + packages-key-suffix: ${{ github.ref_name }}-lint build-libs: # long timeout required when cache has to be recreated @@ -177,12 +209,24 @@ jobs: with: fetch-depth: 0 # Fetch all history for all tags and branches - uses: ./.github/actions/setup + with: + use-gh-nx-cache: false + use-gh-node-modules-cache: false + cache-key-suffix: build-libs - uses: ./.github/actions/download-cache-and-artifacts + with: + packages: 'node_modules' + persistent-packages: 'nxcache' + packages-key-suffix: ${{ github.ref_name }}-build-libs + - run: nx affected:build $NX_CALCULATION_FLAGS --prod - run: nx build demoshell --configuration production - - run: rm -f /home/runner/work/alfresco-ng2-components/alfresco-ng2-components/node_modules/.ngcc_lock_file - run: nx affected --target=build-storybook $NX_CALCULATION_FLAGS --configuration ci - uses: ./.github/actions/upload-cache-and-artifacts + with: + packages: 'dist node_modules' + persistent-packages: 'nxcache' + packages-key-suffix: ${{ github.ref_name }}-build-libs e2e-storybook: timeout-minutes: 20 @@ -195,13 +239,17 @@ jobs: with: fetch-depth: 0 # Fetch all history for all - uses: ./.github/actions/setup + with: + use-gh-nx-cache: false + use-gh-node-modules-cache: false - uses: ./.github/actions/download-cache-and-artifacts + with: + packages: 'dist node_modules' - name: Process Cloud Storybook Playwright run: | npx playwright install chromium sudo sysctl -w fs.inotify.max_user_watches=524288 npx nx affected --target=e2e-playwright $NX_CALCULATION_FLAGS || exit 1 - - uses: ./.github/actions/upload-cache-and-artifacts e2e: timeout-minutes: 90 @@ -332,9 +380,11 @@ jobs: fetch-depth: 0 # Fetch all history for all - uses: ./.github/actions/setup with: - enable-cache: "true" - enable-node-modules-cache: "true" + use-gh-nx-cache: false + use-gh-node-modules-cache: false - uses: ./.github/actions/download-cache-and-artifacts + with: + packages: 'dist node_modules' - name: setup chrome uses: ./.github/actions/setup-chrome - name: e2e @@ -349,4 +399,3 @@ jobs: check-ps-cloud-env: ${{ matrix.e2e-test.check-ps-cloud-env }} check-external-cs-env: ${{ matrix.e2e-test.check-external-cs-env }} apa-proxy: ${{ matrix.e2e-test.apa-proxy }} -# diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d093df31fa..a1091d85fe 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -80,14 +80,16 @@ jobs: fetch-depth: 0 - uses: ./.github/actions/setup with: - enable-cache: false - enable-node-modules-cache: false + use-gh-nx-cache: false + use-gh-node-modules-cache: false - name: install run: | npm ci nx run cli:bundle nx run testing:bundle - uses: ./.github/actions/upload-cache-and-artifacts + with: + packages: 'dist nxcache node_modules' release-demoshell: needs: [setup] @@ -106,9 +108,11 @@ jobs: dry-run-flag: ${{ inputs.dry-run-release }} - uses: ./.github/actions/setup with: - enable-cache: false - enable-node-modules-cache: false + use-gh-nx-cache: false + use-gh-node-modules-cache: false - uses: ./.github/actions/download-cache-and-artifacts + with: + packages: 'dist nxcache node_modules' - name: release Demoshell docker run: | nx build demoshell --configuration production @@ -132,10 +136,12 @@ jobs: dry-run-flag: ${{ inputs.dry-run-release }} - uses: ./.github/actions/setup with: - enable-cache: false - enable-node-modules-cache: false + use-gh-nx-cache: false + use-gh-node-modules-cache: false act: ${{ inputs.dry-run-release }} - uses: ./.github/actions/download-cache-and-artifacts + with: + packages: 'dist nxcache node_modules' - name: release Storybook docker run: | nx run stories:build-storybook --configuration ci @@ -154,13 +160,15 @@ jobs: fetch-depth: 0 - uses: ./.github/actions/setup with: - enable-cache: false - enable-node-modules-cache: false + use-gh-nx-cache: false + use-gh-node-modules-cache: false - id: set-dryrun uses: ./.github/actions/enable-dryrun with: dry-run-flag: ${{ inputs.dry-run-release }} - uses: ./.github/actions/download-cache-and-artifacts + with: + packages: 'dist nxcache node_modules' - name: build libraries run: | set -u;