new units cache strategy

.github/actions/download-cache-and-artifacts/action.yml

@@ -7,7 +7,16 @@ inputs:
     required: false
     type: string
     default: 'dist nxcache'
-
+  persistent-packages:
+    description: 'cache key'
+    required: false
+    type: string
+    default: 'nxcache'
+  cache-key-suffix:
+    description: 'cache key suffix'
+    required: false
+    type: string
+    default: ''    
 runs:
   using: "composite"
   steps:
@@ -22,6 +31,20 @@ runs:
           du -h $i.tar.gz
           time tar xzf $i.tar.gz
         done
+    - name: download and extract persistent artifacts from s3
+      shell: bash
+      env:
+        REMOTE_PATH: "alfresco-ng2-components/build-cache/${{ hashFiles('**/package-lock.json') }}"      
+      run: |
+        if [ -n "${{ inputs.cache-key-suffix }}" ]; then
+          REMOTE_PATH="${REMOTE_PATH}-${{ inputs.cache-key-suffix }}"
+        fi      
+        packages=( $(echo ${{ inputs.persistent-packages }}) )
+        for i in "${packages[@]}"; do
+          time aws s3 cp --no-progress s3://${S3_BUILD_BUCKET_SHORT_NAME}/${REMOTE_PATH}/$i.tar.gz $i.tar.gz
+          du -h $i.tar.gz
+          time tar xzf $i.tar.gz
+        done        
     - name: show files
       shell: bash
       run: |

.github/actions/setup/action.yml

@@ -6,12 +6,12 @@ inputs:
     required: false
     type: string
     default: ''
-  enable-global-nx-cache:
+  use-gh-nx-cache:
     description: 'enable caching'
     required: false
     type: boolean
     default: 'true'
-  enable-global-node-modules-cache:
+  use-gh-node-modules-cache:
     description: 'enable caching for node modules'
     required: false
     type: boolean
@@ -36,7 +36,7 @@ runs:
 # CACHE
       - name: NX cache
         id: nx-cache
-        if: ${{ inputs.enable-global-nx-cache == 'true' }}
+        if: ${{ inputs.use-gh-nx-cache == 'true' }}
         uses: actions/cache@v3
         env:
           cache-name: nx-cache
@@ -49,7 +49,7 @@ runs:
             nxcache-
       - name: Node Modules cache
         id: node-modules-cache
-        if: ${{ inputs.enable-global-node-modules-cache == 'true' }}
+        if: ${{ inputs.use-gh-node-modules-cache == 'true' }}
         uses: actions/cache@v3
         env:
           cache-name: node-modules-cache

.github/actions/upload-cache-and-artifacts/action.yml

@@ -5,14 +5,24 @@ inputs:
     description: 'packages to cache'
     required: false
     type: boolean
-    default: 'dist nxcache'
+    default: 'dist'
+  persistent-packages:
+    description: 'cache key'
+    required: false
+    type: string
+    default: 'nxcache'
+  cache-key-suffix:
+    description: 'cache key suffix'
+    required: false
+    type: string
+    default: ''
 runs:
   using: "composite"
   steps:
     - name: tar and upload artifacts
       shell: bash
       env:
-        REMOTE_PATH: "alfresco-ng2-components/build-cache/${{ github.run_id }}"      
+       REMOTE_PATH: "alfresco-ng2-components/build-cache/${{ github.run_id }}"
       run: |
         packages=( $(echo ${{ inputs.packages }}) )
         for i in "${packages[@]}"; do
@@ -20,5 +30,20 @@ runs:
           du -h $i.tar.gz
           time aws s3 cp --no-progress $i.tar.gz s3://${S3_BUILD_BUCKET_SHORT_NAME}/${REMOTE_PATH}/$i.tar.gz
         done
+    - name: tar and upload persistent artifacts
+      shell: bash
+      env:
+        REMOTE_PATH: "alfresco-ng2-components/build-cache/${{ hashFiles('**/package-lock.json') }} }}"
+      run: |
+        if [ -n "${{ inputs.cache-key-suffix }}" ]; then
+          REMOTE_PATH="${REMOTE_PATH}-${{ inputs.cache-key-suffix }}"
+        fi
+        packages=( $(echo ${{ inputs.persistent-packages }}) )
+        for i in "${packages[@]}"; do
+          time tar czf $i.tar.gz $i
+          du -h $i.tar.gz
+          echo "pushing persistent cache"
+          time aws s3 cp --no-progress $i.tar.gz s3://${S3_BUILD_BUCKET_SHORT_NAME}/${REMOTE_PATH}/$i.tar.gz
+        done
 
 

.github/workflows/pull-request.yml

@@ -120,8 +120,8 @@ jobs:
       - uses: ./.github/actions/setup
         id: setup
         with: 
-          enable-global-nx-cache: false
+          use-gh-nx-cache: false
-          enable-global-node-modules-cache: true       
+          use-gh-node-modules-cache: true       
       - name: run ci
         if: steps.setup.outputs.cache-hit != 'true'          
         run: npm ci
@@ -168,13 +168,20 @@ jobs:
           fetch-depth: 0 # Fetch all history for all tags and branches
       - uses: ./.github/actions/setup
         with: 
-          enable-global-nx-cache: true
+          use-gh-nx-cache: false
-          enable-global-node-modules-cache: true
+          use-gh-node-modules-cache: true
           cache-key-suffix: unit-tests-${{ matrix.unit-tests.name }}
+      - uses: ./.github/actions/download-cache-and-artifacts
+        with:
+          persistent-packages: 'nxcache'
+          cache-key-suffix: ${{ matrix.unit-tests.name }}
       - name: Run unit tests
         run: |
           /usr/bin/xvfb-run --auto-servernum nx affected:test $NX_CALCULATION_FLAGS --exclude=${{ matrix.unit-tests.exclude }}
-
+      - uses: ./.github/actions/upload-cache-and-artifacts
+        with:
+          persistent-packages: 'nxcache'
+          cache-key-suffix: ${{ matrix.unit-tests.name }}
   lint:
     # long timeout required when cache has to be recreated
     timeout-minutes: 30
@@ -188,14 +195,17 @@ jobs:
           fetch-depth: 0 # Fetch all history for all tags and branches
       - uses: ./.github/actions/setup
         with: 
-          enable-global-nx-cache: true
+          use-gh-nx-cache: false
-          enable-global-node-modules-cache: true
+          use-gh-node-modules-cache: true
           cache-key-suffix: lint
       - uses: ./.github/actions/download-cache-and-artifacts
         with:
-          packages: 'node_modules'      
+          packages: 'node_modules'
+          persistent-packages: 'nxcache'
       - run: nx affected --target=lint $NX_CALCULATION_FLAGS
-
+      - uses: ./.github/actions/upload-cache-and-artifacts
+        with:
+          persistent-packages: 'nxcache'
   build-libs:
     # long timeout required when cache has to be recreated
     timeout-minutes: 30
@@ -209,18 +219,19 @@ jobs:
           fetch-depth: 0 # Fetch all history for all tags and branches
       - uses: ./.github/actions/setup
         with: 
-          enable-global-nx-cache: true
+          use-gh-nx-cache: false
-          enable-global-node-modules-cache: true
+          use-gh-node-modules-cache: true
           cache-key-suffix: build-libs
-      # - uses: ./.github/actions/download-cache-and-artifacts
+      - uses: ./.github/actions/download-cache-and-artifacts
-      #   with:
+        with:
-      #     packages: 'node_modules'      
+          persistent-packages: 'nxcache'
       - run: nx affected:build $NX_CALCULATION_FLAGS --prod
       - run: nx build demoshell --configuration production
       - run: nx affected --target=build-storybook $NX_CALCULATION_FLAGS --configuration ci
       - uses: ./.github/actions/upload-cache-and-artifacts
         with:
           packages: 'dist node_modules'
+          persistent-packages: 'nxcache'
 
   e2e-storybook:
     timeout-minutes: 20
@@ -234,8 +245,8 @@ jobs:
           fetch-depth: 0 # Fetch all history for all
       - uses: ./.github/actions/setup
         with: 
-          enable-global-nx-cache: false
+          use-gh-nx-cache: false
-          enable-global-node-modules-cache: false
+          use-gh-node-modules-cache: false
       - uses: ./.github/actions/download-cache-and-artifacts
         with:
           packages: 'dist node_modules'
@@ -374,8 +385,8 @@ jobs:
           fetch-depth: 0 # Fetch all history for all
       - uses: ./.github/actions/setup
         with:
-          enable-global-nx-cache: false
+          use-gh-nx-cache: false
-          enable-global-node-modules-cache: false
+          use-gh-node-modules-cache: false
       - uses: ./.github/actions/download-cache-and-artifacts
         with:
           packages: 'dist node_modules'

.github/workflows/release.yml

@@ -80,8 +80,8 @@ jobs:
           fetch-depth: 0
       - uses: ./.github/actions/setup
         with: 
-          enable-global-nx-cache: false
+          use-gh-nx-cache: false
-          enable-global-node-modules-cache: false
+          use-gh-node-modules-cache: false
       - name: install
         run: |
           npm ci
@@ -108,8 +108,8 @@ jobs:
           dry-run-flag: ${{ inputs.dry-run-release }}
       - uses: ./.github/actions/setup
         with: 
-          enable-global-nx-cache: false
+          use-gh-nx-cache: false
-          enable-global-node-modules-cache: false
+          use-gh-node-modules-cache: false
       - uses: ./.github/actions/download-cache-and-artifacts     
         with:
           packages: 'dist nxcache node_modules'
@@ -136,8 +136,8 @@ jobs:
           dry-run-flag: ${{ inputs.dry-run-release }}
       - uses: ./.github/actions/setup
         with: 
-          enable-global-nx-cache: false
+          use-gh-nx-cache: false
-          enable-global-node-modules-cache: false
+          use-gh-node-modules-cache: false
           act: ${{ inputs.dry-run-release }}
       - uses: ./.github/actions/download-cache-and-artifacts
         with:
@@ -160,8 +160,8 @@ jobs:
           fetch-depth: 0
       - uses: ./.github/actions/setup
         with: 
-          enable-global-nx-cache: false
+          use-gh-nx-cache: false
-          enable-global-node-modules-cache: false
+          use-gh-node-modules-cache: false
       - id: set-dryrun
         uses: ./.github/actions/enable-dryrun
         with: