diff --git a/.github/actions/download-cache-and-artifacts/action.yml b/.github/actions/download-cache-and-artifacts/action.yml index e396b1bffd..1800d9eb4b 100644 --- a/.github/actions/download-cache-and-artifacts/action.yml +++ b/.github/actions/download-cache-and-artifacts/action.yml @@ -7,7 +7,16 @@ inputs: required: false type: string default: 'dist nxcache' - + persistent-packages: + description: 'cache key' + required: false + type: string + default: 'nxcache' + cache-key-suffix: + description: 'cache key suffix' + required: false + type: string + default: '' runs: using: "composite" steps: @@ -22,6 +31,20 @@ runs: du -h $i.tar.gz time tar xzf $i.tar.gz done + - name: download and extract persistent artifacts from s3 + shell: bash + env: + REMOTE_PATH: "alfresco-ng2-components/build-cache/${{ hashFiles('**/package-lock.json') }}" + run: | + if [ -n "${{ inputs.cache-key-suffix }}" ]; then + REMOTE_PATH="${REMOTE_PATH}-${{ inputs.cache-key-suffix }}" + fi + packages=( $(echo ${{ inputs.persistent-packages }}) ) + for i in "${packages[@]}"; do + time aws s3 cp --no-progress s3://${S3_BUILD_BUCKET_SHORT_NAME}/${REMOTE_PATH}/$i.tar.gz $i.tar.gz + du -h $i.tar.gz + time tar xzf $i.tar.gz + done - name: show files shell: bash run: | diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index 9262862d97..eecf689c2a 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -6,12 +6,12 @@ inputs: required: false type: string default: '' - enable-global-nx-cache: + use-gh-nx-cache: description: 'enable caching' required: false type: boolean default: 'true' - enable-global-node-modules-cache: + use-gh-node-modules-cache: description: 'enable caching for node modules' required: false type: boolean @@ -36,7 +36,7 @@ runs: # CACHE - name: NX cache id: nx-cache - if: ${{ inputs.enable-global-nx-cache == 'true' }} + if: ${{ inputs.use-gh-nx-cache == 'true' }} uses: actions/cache@v3 env: cache-name: nx-cache @@ -49,7 +49,7 @@ runs: nxcache- - name: Node Modules cache id: node-modules-cache - if: ${{ inputs.enable-global-node-modules-cache == 'true' }} + if: ${{ inputs.use-gh-node-modules-cache == 'true' }} uses: actions/cache@v3 env: cache-name: node-modules-cache diff --git a/.github/actions/upload-cache-and-artifacts/action.yml b/.github/actions/upload-cache-and-artifacts/action.yml index 33af6b661d..02293a5ab2 100644 --- a/.github/actions/upload-cache-and-artifacts/action.yml +++ b/.github/actions/upload-cache-and-artifacts/action.yml @@ -5,14 +5,24 @@ inputs: description: 'packages to cache' required: false type: boolean - default: 'dist nxcache' + default: 'dist' + persistent-packages: + description: 'cache key' + required: false + type: string + default: 'nxcache' + cache-key-suffix: + description: 'cache key suffix' + required: false + type: string + default: '' runs: using: "composite" steps: - name: tar and upload artifacts shell: bash env: - REMOTE_PATH: "alfresco-ng2-components/build-cache/${{ github.run_id }}" + REMOTE_PATH: "alfresco-ng2-components/build-cache/${{ github.run_id }}" run: | packages=( $(echo ${{ inputs.packages }}) ) for i in "${packages[@]}"; do @@ -20,5 +30,20 @@ runs: du -h $i.tar.gz time aws s3 cp --no-progress $i.tar.gz s3://${S3_BUILD_BUCKET_SHORT_NAME}/${REMOTE_PATH}/$i.tar.gz done + - name: tar and upload persistent artifacts + shell: bash + env: + REMOTE_PATH: "alfresco-ng2-components/build-cache/${{ hashFiles('**/package-lock.json') }} }}" + run: | + if [ -n "${{ inputs.cache-key-suffix }}" ]; then + REMOTE_PATH="${REMOTE_PATH}-${{ inputs.cache-key-suffix }}" + fi + packages=( $(echo ${{ inputs.persistent-packages }}) ) + for i in "${packages[@]}"; do + time tar czf $i.tar.gz $i + du -h $i.tar.gz + echo "pushing persistent cache" + time aws s3 cp --no-progress $i.tar.gz s3://${S3_BUILD_BUCKET_SHORT_NAME}/${REMOTE_PATH}/$i.tar.gz + done diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml index b12e3b2127..7f2debae1e 100644 --- a/.github/workflows/pull-request.yml +++ b/.github/workflows/pull-request.yml @@ -120,8 +120,8 @@ jobs: - uses: ./.github/actions/setup id: setup with: - enable-global-nx-cache: false - enable-global-node-modules-cache: true + use-gh-nx-cache: false + use-gh-node-modules-cache: true - name: run ci if: steps.setup.outputs.cache-hit != 'true' run: npm ci @@ -168,13 +168,20 @@ jobs: fetch-depth: 0 # Fetch all history for all tags and branches - uses: ./.github/actions/setup with: - enable-global-nx-cache: true - enable-global-node-modules-cache: true + use-gh-nx-cache: false + use-gh-node-modules-cache: true cache-key-suffix: unit-tests-${{ matrix.unit-tests.name }} + - uses: ./.github/actions/download-cache-and-artifacts + with: + persistent-packages: 'nxcache' + cache-key-suffix: ${{ matrix.unit-tests.name }} - name: Run unit tests run: | /usr/bin/xvfb-run --auto-servernum nx affected:test $NX_CALCULATION_FLAGS --exclude=${{ matrix.unit-tests.exclude }} - + - uses: ./.github/actions/upload-cache-and-artifacts + with: + persistent-packages: 'nxcache' + cache-key-suffix: ${{ matrix.unit-tests.name }} lint: # long timeout required when cache has to be recreated timeout-minutes: 30 @@ -188,14 +195,17 @@ jobs: fetch-depth: 0 # Fetch all history for all tags and branches - uses: ./.github/actions/setup with: - enable-global-nx-cache: true - enable-global-node-modules-cache: true + use-gh-nx-cache: false + use-gh-node-modules-cache: true cache-key-suffix: lint - uses: ./.github/actions/download-cache-and-artifacts with: - packages: 'node_modules' + packages: 'node_modules' + persistent-packages: 'nxcache' - run: nx affected --target=lint $NX_CALCULATION_FLAGS - + - uses: ./.github/actions/upload-cache-and-artifacts + with: + persistent-packages: 'nxcache' build-libs: # long timeout required when cache has to be recreated timeout-minutes: 30 @@ -209,18 +219,19 @@ jobs: fetch-depth: 0 # Fetch all history for all tags and branches - uses: ./.github/actions/setup with: - enable-global-nx-cache: true - enable-global-node-modules-cache: true + use-gh-nx-cache: false + use-gh-node-modules-cache: true cache-key-suffix: build-libs - # - uses: ./.github/actions/download-cache-and-artifacts - # with: - # packages: 'node_modules' + - uses: ./.github/actions/download-cache-and-artifacts + with: + persistent-packages: 'nxcache' - run: nx affected:build $NX_CALCULATION_FLAGS --prod - run: nx build demoshell --configuration production - run: nx affected --target=build-storybook $NX_CALCULATION_FLAGS --configuration ci - uses: ./.github/actions/upload-cache-and-artifacts with: packages: 'dist node_modules' + persistent-packages: 'nxcache' e2e-storybook: timeout-minutes: 20 @@ -234,8 +245,8 @@ jobs: fetch-depth: 0 # Fetch all history for all - uses: ./.github/actions/setup with: - enable-global-nx-cache: false - enable-global-node-modules-cache: false + use-gh-nx-cache: false + use-gh-node-modules-cache: false - uses: ./.github/actions/download-cache-and-artifacts with: packages: 'dist node_modules' @@ -374,8 +385,8 @@ jobs: fetch-depth: 0 # Fetch all history for all - uses: ./.github/actions/setup with: - enable-global-nx-cache: false - enable-global-node-modules-cache: false + use-gh-nx-cache: false + use-gh-node-modules-cache: false - uses: ./.github/actions/download-cache-and-artifacts with: packages: 'dist node_modules' diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 5393e426c9..a1091d85fe 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -80,8 +80,8 @@ jobs: fetch-depth: 0 - uses: ./.github/actions/setup with: - enable-global-nx-cache: false - enable-global-node-modules-cache: false + use-gh-nx-cache: false + use-gh-node-modules-cache: false - name: install run: | npm ci @@ -108,8 +108,8 @@ jobs: dry-run-flag: ${{ inputs.dry-run-release }} - uses: ./.github/actions/setup with: - enable-global-nx-cache: false - enable-global-node-modules-cache: false + use-gh-nx-cache: false + use-gh-node-modules-cache: false - uses: ./.github/actions/download-cache-and-artifacts with: packages: 'dist nxcache node_modules' @@ -136,8 +136,8 @@ jobs: dry-run-flag: ${{ inputs.dry-run-release }} - uses: ./.github/actions/setup with: - enable-global-nx-cache: false - enable-global-node-modules-cache: false + use-gh-nx-cache: false + use-gh-node-modules-cache: false act: ${{ inputs.dry-run-release }} - uses: ./.github/actions/download-cache-and-artifacts with: @@ -160,8 +160,8 @@ jobs: fetch-depth: 0 - uses: ./.github/actions/setup with: - enable-global-nx-cache: false - enable-global-node-modules-cache: false + use-gh-nx-cache: false + use-gh-node-modules-cache: false - id: set-dryrun uses: ./.github/actions/enable-dryrun with: