[AAE-11882] demoshell fix; pipeline bugfix (#8093)

* demoshell regr fix; s3 cache on s3; setup chrome; e2e headless fix

.github/actions/download-build-artifacts/action.yml

@@ -1,53 +0,0 @@
-name: "Download build artifacts"
-description: "Download build artifacts"
-runs:
-  using: "composite"
-  steps:
-    ## NXCACHE
-    - name: update NX cache folder
-      uses: actions/download-artifact@v3
-      with:
-        path: .
-        name: nxcache
-    ## DIST
-    - name: update Dist folder
-      uses: actions/download-artifact@v3
-      with:
-        path: .
-        name: dist
-    ## ADF CLI
-    - name: update Dist folder
-      uses: actions/download-artifact@v3
-      with:
-        path: .
-        name: adf-cli
-    ## ADF TESTING
-    - name: update Dist folder
-      uses: actions/download-artifact@v3
-      with:
-        path: .
-        name: adf-testing
-    - name: extract nxcache
-      run: |
-        tar xzf nxcache.tar.gz
-      shell: bash
-    - name: extract dist
-      run: |
-        tar xzf dist.tar.gz
-      shell: bash
-    - name: extract adf cli
-      run: |
-        tar xzf adf-cli.tar.gz
-        tar xzf adf-testing.tar.gz
-      shell: bash
-    - name: show files
-      shell: bash
-      run: |
-        pwd
-        ls -lha
-        echo "====DIST===="
-        find dist -maxdepth 1 -type d
-        echo "====NXCACHE===="
-        find nxcache -maxdepth 1 -type d
-        echo "====ADF===="
-        find node_modules/@alfresco/ -maxdepth 1 -type d

.github/actions/download-cache-and-artifacts/action.yml

@@ -0,0 +1,27 @@
+name: "Download build artifacts"
+description: "Download build artifacts"
+runs:
+  using: "composite"
+  steps:
+    - name: download and extract artifacts from s3
+      shell: bash
+      env:
+        REMOTE_PATH: "alfresco-ng2-components/build-cache/${{ github.run_id }}"      
+      run: |
+        packages=( dist nxcache node_modules )
+        for i in "${packages[@]}"; do
+          time aws s3 cp --no-progress s3://${S3_BUILD_BUCKET_SHORT_NAME}/${REMOTE_PATH}/$i.tar.gz $i.tar.gz
+          du -h $i.tar.gz
+          time tar xzf $i.tar.gz
+        done
+    - name: show files
+      shell: bash
+      run: |
+        pwd
+        ls -lha
+        echo "====DIST===="
+        find dist -maxdepth 1 -type d
+        echo "====NXCACHE===="
+        find nxcache -maxdepth 1 -type d
+        echo "====ADF===="
+        find node_modules/@alfresco/ -maxdepth 1 -type d

.github/actions/e2e/action.yml

@@ -129,6 +129,7 @@ runs:
         echo "NX_CALCULATION_FLAGS=$NX_CALCULATION_FLAGS" >> $GITHUB_ENV
         GIT_HASH=$(echo $GIT_HASH | sed "s@'@@g")
         echo "GIT_HASH=$GIT_HASH" >> $GITHUB_ENV
+
     - name: run test with retries
       id: retry_run
       env:
@@ -136,6 +137,7 @@ runs:
         PROVIDER: "${{ inputs.e2e-test-provider }}"
         AUTH_TYPE: "${{ inputs.e2e-test-auth }}"
         E2E_TEST_ID: "${{ inputs.e2e-test-id }}"
+        
       uses: nick-fields/retry@v2.8.2
       with:
         timeout_minutes: 40
@@ -146,8 +148,14 @@ runs:
           set -u;
           export GH_ACTION_RETRY_COUNT=$(cat ${GITHUB_OUTPUT} | grep -E '^[0-9]{1,2}$' | tail -n1)
           echo "RETRY GH_ACTION_RETRY_COUNT = <$GH_ACTION_RETRY_COUNT>"                     
-          bash ./scripts/travis/e2e/e2e.sh "$E2E_TEST_ID" || exit 1
+          if [[ ${{ inputs.e2e-test-folder }} == 'content-services/upload' ]]; then
-
+            export DISPLAY=:99
+            chromedriver --url-base=/wd/hub &
+            sudo Xvfb -ac :99 -screen 0 1280x1024x24 > /dev/null 2>&1 & # optional                           
+            bash ./scripts/travis/e2e/e2e.sh "$E2E_TEST_ID" "browser" || exit 1
+          else
+            bash ./scripts/travis/e2e/e2e.sh "$E2E_TEST_ID" || exit 1          
+          fi
     - name: upload smartrunner tests results on s3 to cache tests
       shell: bash
       if: always()
@@ -157,5 +165,3 @@ runs:
       run: |
         tar czf "${{ inputs.e2e-tar-name }}" "${SMART_RUNNER_PATH}"
         aws s3 cp "${{ inputs.e2e-tar-name }}" "${S3_BUILD_BUCKET}/adf/$REMOTE_PATH"
-
-

.github/actions/enable-dryrun/action.yml

@@ -0,0 +1,30 @@
+name: 'enable dryrun'
+description: 'check if must run pipeline in dryrun mode'
+
+inputs:
+  dry-run-flag:
+    description: 'enable dryrun'
+    required: false
+    type: boolean
+    default: true
+
+outputs:
+  dryrun:
+    description: "long sha of the tag"
+    value: ${{ steps.dryrun.outputs.dryrun }}
+
+runs:
+  using: "composite"
+
+  steps:
+    - name: set dryrun flag to TRUE
+      shell: bash
+      id: dryrun
+      run: |
+          if [[ '${{ inputs.dry-run-flag }}' == 'true' ]]; then
+            echo "dryrun=--dryrun" >> $GITHUB_OUTPUT;
+            echo "enabling dryrun"
+          else
+            echo "dryrun=" >> $GITHUB_OUTPUT;
+            echo "dryrun NOT enabled"
+          fi

.github/actions/npm-check-bundle/action.yml

@@ -0,0 +1,19 @@
+name: 'Check NPM bundle'
+description: 'check npm bundle'
+
+outputs:
+  tag_sha:
+    description: "long sha of the tag"
+    value: ${{ steps.sha_out.outputs.tag_sha }}
+
+runs:
+  using: "composite"
+
+  steps:
+    - name: check npm bundle
+      shell: bash
+      id: sha_out
+      run: |
+          ADF_VERSION=$(npm view @alfresco/adf-core@${TAG_NPM} version)
+          echo "check bundle on TAG_NPM='${TAG_NPM}' and ADF_VERSION='${ADF_VERSION}'"
+          ./scripts/travis/build/npm-check-bundles.sh -v ${ADF_VERSION}

.github/actions/setup-chrome/action.yml

@@ -0,0 +1,13 @@
+name: 'Install Google Chrome'
+description: 'Install Google Chrome'
+    
+runs:
+  using: "composite"
+  steps:
+      - name: Install google chrome
+        shell: bash
+        run: |
+            wget -q https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb
+            sudo apt install ./google-chrome-stable_current_amd64.deb
+            sudo ln -s /usr/bin/google-chrome /usr/bin/chrome
+            chrome --version

.github/actions/setup/action.yml

@@ -27,7 +27,6 @@ runs:
           cache-name: node-cache
         with:
           path: |
-            node_modules
             ~/.npm
             nxcache
             dist
@@ -36,6 +35,20 @@ runs:
             node-${{ runner.os }}-build-${{ env.cache-name }}-
             node-${{ runner.os }}-build-
             node-${{ runner.os }}-
+      - name: Node Modules cache
+        id: node-modules-cache
+        if: ${{ inputs.enable-node-modules-cache == 'true' }}
+        uses: actions/cache@v3
+        env:
+          cache-name: node-modules-cache
+        with:
+          path: |
+            node_modules
+          key: .npm-${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }}-${{ steps.tag-sha.outputs.tag_sha }}
+          restore-keys: |
+            node_modules-${{ runner.os }}-build-${{ env.cache-name }}-
+            node_modules-${{ runner.os }}-build-
+            node_modules-${{ runner.os }}-            
       - name: pip cache
         uses: actions/cache@v3
         if: ${{ inputs.enable-cache == 'true' }}

.github/actions/travis-env-var-facade/action.yml

@@ -16,6 +16,7 @@ runs:
         EVENT_TYPE=""
         TRAVIS_PULL_REQUEST_BRANCH=""
         TRAVIS_BRANCH=""
+        TAG_NPM=""
         REGEX="(repository|workflow)_dispatch"
         if [[ "${{ inputs.event_name }}" == "schedule"  ]]; then 
             EVENT_TYPE="cron";
@@ -29,6 +30,14 @@ runs:
         elif [[ "${{ inputs.event_name }}" =~ $REGEX ]]; then
             EVENT_TYPE="api";
         fi
+
+        if [[ GITHUB_REF_NAME == "master" ]]; then
+            TAG_NPM="latest"
+        fi
+        if [[ GITHUB_REF_NAME == "develop" ]]; then
+            TAG_NPM="alpha"
+        fi
+        echo "TAG_NPM=${TAG_NPM}" >> $GITHUB_ENV
         echo "TRAVIS_EVENT_TYPE=${EVENT_TYPE}" >> $GITHUB_ENV
         echo "TRAVIS_PULL_REQUEST_BRANCH=${TRAVIS_PULL_REQUEST_BRANCH}" >> $GITHUB_ENV
         echo "TRAVIS_BRANCH=${TRAVIS_BRANCH}" >> $GITHUB_ENV

.github/actions/upload-build-artifacts/action.yml

@@ -1,41 +0,0 @@
-name: "Upload build artifacts"
-description: "Upload build artifacts"
-
-runs:
-  using: "composite"
-  steps:
-    ## NXCACHE
-    - name: tar artifacts
-      run: |
-        tar czf dist.tar.gz dist
-        tar czf nxcache.tar.gz nxcache
-        tar czf adf-cli.tar.gz node_modules/@alfresco/adf-cli
-        tar czf adf-testing.tar.gz node_modules/@alfresco/adf-testing
-      shell: bash
-    - name: update NX cache folder
-      uses: actions/upload-artifact@v3
-      continue-on-error: true
-      with:
-        path: nxcache.tar.gz
-        name: nxcache
-    ## DIST
-    - name: update Dist folder
-      uses: actions/upload-artifact@v3
-      continue-on-error: true
-      with:
-        path: dist.tar.gz
-        name: dist
-    ## ADF CLI
-    - name: save ADF cli
-      uses: actions/upload-artifact@v3
-      continue-on-error: true
-      with:
-        path: adf-cli.tar.gz
-        name: adf-cli
-    ## ADF TESTING
-    - name: save ADF testing
-      uses: actions/upload-artifact@v3
-      continue-on-error: true
-      with:
-        path: adf-testing.tar.gz
-        name: adf-testing

.github/actions/upload-cache-and-artifacts/action.yml

@@ -0,0 +1,19 @@
+name: "Upload build artifacts"
+description: "Upload build artifacts"
+
+runs:
+  using: "composite"
+  steps:
+    - name: tar and upload artifacts
+      shell: bash
+      env:
+        REMOTE_PATH: "alfresco-ng2-components/build-cache/${{ github.run_id }}"      
+      run: |
+        packages=( dist nxcache node_modules )
+        for i in "${packages[@]}"; do
+          time tar czf $i.tar.gz $i
+          du -h $i.tar.gz
+          time aws s3 cp --no-progress $i.tar.gz s3://${S3_BUILD_BUCKET_SHORT_NAME}/${REMOTE_PATH}/$i.tar.gz
+        done
+
+

.github/workflows/pull-request.yml

@@ -1,6 +1,13 @@
 name: "pull-request"
 
 on:
+  workflow_call:
+    inputs:
+      dry-run-release:
+        description: 'enable dry-run on artifact push'
+        required: false
+        type: boolean
+        default: true
   pull_request:
     types: [opened, synchronize, reopened]
     branches:
@@ -49,7 +56,14 @@ env:
   SMART_RUNNER_PATH: ".protractor-smartrunner"
   S3_DBP_PATH:  ${{ secrets.S3_DBP_PATH }}
   S3_BUILD_BUCKET: ${{ secrets.S3_BUILD_BUCKET }}
+  S3_BUILD_BUCKET_SHORT_NAME: ${{ secrets.S3_BUILD_BUCKET_SHORT_NAME }}
   NODE_OPTIONS: "--max-old-space-size=5120"
+  DOCKER_REPOSITORY_DOMAIN: ${{ secrets.DOCKER_REPOSITORY_DOMAIN }}
+  DOCKER_REPOSITORY_USER: ${{ secrets.DOCKER_REPOSITORY_USER }}
+  DOCKER_REPOSITORY_PASSWORD: ${{ secrets.DOCKER_REPOSITORY_PASSWORD }}
+  DOCKER_REPOSITORY_STORYBOOK: "${{ secrets.DOCKER_REPOSITORY_DOMAIN }}/alfresco/storybook"
+  DOCKER_REPOSITORY: "${{ secrets.DOCKER_REPOSITORY_DOMAIN }}/alfresco/demo-shell"
+
 jobs:
   main:
     # long timeout required when cache has to be recreated
@@ -72,7 +86,7 @@ jobs:
       - run: nx affected:build $NX_CALCULATION_FLAGS --prod
       - run: nx build demoshell --configuration production
       - run: nx run stories:build-storybook --configuration ci
-      - uses: ./.github/actions/upload-build-artifacts
+      - uses: ./.github/actions/upload-cache-and-artifacts
 
   playwright:
     timeout-minutes: 20
@@ -85,20 +99,13 @@ jobs:
         with:
           fetch-depth: 0 # Fetch all history for all
       - uses: ./.github/actions/setup
-      - name: clean
+      - uses: ./.github/actions/download-cache-and-artifacts
-        run: |
-          rm -rf nxcache
-          rm -rf dist
-      - uses: ./.github/actions/download-build-artifacts
-      - name: build adf cli
-        run: |
-          nx build cli
       - name: Process Cloud Storybook Playwright
         run: |
           npx playwright install chromium
           sudo sysctl -w fs.inotify.max_user_watches=524288
           npx playwright test --config='e2e-playwright/playwright.config.ts' || exit 1
-      - uses: ./.github/actions/upload-build-artifacts
+      - uses: ./.github/actions/upload-cache-and-artifacts
 
   e2e:
     timeout-minutes: 90
@@ -142,7 +149,7 @@ jobs:
             auth: "BASIC"
             check-cs-env: "true"
           - description: "Content: Upload and Versioning"
-            test-id: "content-services browser"
+            test-id: "content-services"
             folder: "content-services/upload"
             provider: "ECM"
             auth: "BASIC"
@@ -230,11 +237,10 @@ jobs:
       - uses: ./.github/actions/setup
         with:
           enable-cache: "true"
-      - name: clean
+          enable-node-modules-cache: "true"
-        run: |
+      - uses: ./.github/actions/download-cache-and-artifacts
-          rm -rf nxcache
+      - name: setup chrome
-          rm -rf dist
+        uses: ./.github/actions/setup-chrome
-      - uses: ./.github/actions/download-build-artifacts
       - name: e2e
         uses: ./.github/actions/e2e
         with:

.github/workflows/release.yml

@@ -1,6 +1,13 @@
 name: "release"
 
 on:
+  workflow_call:
+    inputs:
+      dry-run-release:
+        description: 'enable dry-run on artifact push'
+        required: false
+        type: boolean
+        default: true
   pull_request:
     types: [closed]
     branches:
@@ -10,6 +17,8 @@ on:
       - master-patch*
 
 env:
+  BASE_REF: ${{ github.base_ref }}
+  HEAD_REF: ${{ github.head_ref }}
   AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
   AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
   GITHUB_BRANCH: ${{ github.ref_name }}
@@ -17,7 +26,7 @@ env:
   TRAVIS_COMMIT: ${{ github.sha }}
   BUILD_ID: ${{ github.run_id }}
   TRAVIS_RUN_NUMBER: ${{ github.run_attempt }}
-  TRAVIS_BUILD_NUMBER: github-run-${{ github.run_id }}
+  TRAVIS_BUILD_NUMBER: ${{ github.run_id }}
   JOB_ID: ${{ github.run_id }}
   PROXY_HOST_BPM: ${{ secrets.E2E_HOST }}
   E2E_HOST_APA: ${{ secrets.E2E_HOST_APA }}
@@ -44,20 +53,23 @@ env:
   PASSWORD_SUPER_ADMIN_ADF: ${{ secrets.PASSWORD_SUPER_ADMIN_ADF }}
   HR_USER: ${{ secrets.HR_USER }}
   HR_USER_PASSWORD: ${{ secrets.HR_USER_PASSWORD }}
-  NPM_REGISTRY_ADDRESS: ${{ secrets.NPM_REGISTRY_ADDRESS }}
+  SMART_RUNNER_PATH: ".protractor-smartrunner"
-  NPM_REGISTRY_TOKEN: ${{ secrets.NPM_REGISTRY_TOKEN }}
+  S3_DBP_PATH:  ${{ secrets.S3_DBP_PATH }}
+  S3_BUILD_BUCKET: ${{ secrets.S3_BUILD_BUCKET }}
+  S3_BUILD_BUCKET_SHORT_NAME: ${{ secrets.S3_BUILD_BUCKET_SHORT_NAME }}
+  NODE_OPTIONS: "--max-old-space-size=5120"
+  DOCKER_REPOSITORY_DOMAIN: ${{ secrets.DOCKER_REPOSITORY_DOMAIN }}
   DOCKER_REPOSITORY_USER: ${{ secrets.DOCKER_REPOSITORY_USER }}
   DOCKER_REPOSITORY_PASSWORD: ${{ secrets.DOCKER_REPOSITORY_PASSWORD }}
-  DOCKER_REPOSITORY_DOMAIN: ${{ secrets.DOCKER_REPOSITORY_DOMAIN }}
   DOCKER_REPOSITORY_STORYBOOK: "${{ secrets.DOCKER_REPOSITORY_DOMAIN }}/alfresco/storybook"
   DOCKER_REPOSITORY: "${{ secrets.DOCKER_REPOSITORY_DOMAIN }}/alfresco/demo-shell"
-  NODE_OPTIONS: "--max-old-space-size=5120"
+
 
 jobs:
-  build:
+  setup:
     timeout-minutes: 20
-    if: github.event.pull_request.merged == true
+    if: github.event.pull_request.merged || ${{ inputs.dry-run-release }}
-    name: "Build Components"
+    name: "Setup"
     runs-on: ubuntu-22.04
     steps:
       - name: Checkout repository
@@ -65,20 +77,17 @@ jobs:
         with:
           fetch-depth: 0
       - uses: ./.github/actions/setup
-      - run: npm -v
       - name: install
         run: |
           npm ci
           nx run cli:bundle
           nx run testing:bundle
-      - run: npx nx affected --target=lint $NX_CALCULATION_FLAGS
+      - uses: ./.github/actions/upload-cache-and-artifacts
-      - run: nx affected:build $NX_CALCULATION_FLAGS --prod --exclude="demoshell"
-      - uses: ./.github/actions/upload-build-artifacts
 
   release-demoshell:
-    needs: [release-npm]
+    needs: [setup]
     timeout-minutes: 15
-    if: github.event.pull_request.merged == true
+    if: github.event.pull_request.merged || ${{ inputs.dry-run-release }}
     runs-on: ubuntu-22.04
     steps:
       - name: Checkout repository
@@ -86,48 +95,51 @@ jobs:
         with:
           fetch-depth: 1
       - run: git fetch --all
+      - id: set-dryrun
+        uses: ./.github/actions/enable-dryrun
+        with: 
+          dry-run-flag: ${{ inputs.dry-run-release }}
       - uses: ./.github/actions/setup
         with: 
           enable-cache: false
-      - name: install nx and cli
+          enable-node-modules-cache: false
-        run: |
+      - uses: ./.github/actions/download-cache-and-artifacts     
-          npm ci
+      - name: release Demoshell docker
-          nx run cli:bundle
-      - name: push Demoshell docker image
         run: |
           nx build demoshell --configuration production
           . ./scripts/travis/release/docker-tag.sh
-          echo $TAGS
+          ./scripts/travis/release/release-demoshell-docker.sh ${{ steps.set-dryrun.outputs.dryrun }}
-          ./scripts/travis/release/release-demoshell-docker.sh
 
   release-storybook:
-    needs: [release-npm]
+    needs: [setup]
     timeout-minutes: 15
-    if: github.event.pull_request.merged == true
+    if: github.event.pull_request.merged || ${{ inputs.dry-run-release }}
     runs-on: ubuntu-22.04
     steps:
       - name: Checkout repository
         uses: actions/checkout@v3
         with:
-          fetch-depth: 0
+          fetch-depth: 1
+      - run: git fetch --all
+      - id: set-dryrun
+        uses: ./.github/actions/enable-dryrun
+        with: 
+          dry-run-flag: ${{ inputs.dry-run-release }}
       - uses: ./.github/actions/setup
         with: 
           enable-cache: false
-      - name: install nx and cli
+          enable-node-modules-cache: false
-        run: |
+      - uses: ./.github/actions/download-cache-and-artifacts
-          npm ci
+      - name: release Storybook docker
-          nx run cli:bundle
-      - name: push Storybook docker image
         run: |
           nx run stories:build-storybook --configuration ci
           . ./scripts/travis/release/docker-tag.sh  
-          echo $TAGS     
+          ./scripts/travis/release/release-storybook-docker.sh ${{ steps.set-dryrun.outputs.dryrun }}
-          ./scripts/travis/release/release-storybook-docker.sh
 
   release-npm:
-    needs: [build]
+    needs: [setup]
     timeout-minutes: 15
-    if: github.event.pull_request.merged == true
+    if: github.event.pull_request.merged == true || ${{ inputs.dry-run-release }}
     runs-on: ubuntu-22.04
     steps:
       - name: Checkout repository
@@ -137,23 +149,25 @@ jobs:
       - uses: ./.github/actions/setup
         with: 
           enable-cache: false
-      - name: install nx and cli
+          enable-node-modules-cache: false
-        run: |
+      - id: set-dryrun
-          npm ci
+        uses: ./.github/actions/enable-dryrun
-          nx run cli:bundle
+        with: 
-      - name: nx build
+          dry-run-flag: ${{ inputs.dry-run-release }}          
+      - uses: ./.github/actions/download-cache-and-artifacts
+      - name: build libraries
         run: |
           set -u;
           ./scripts/travis/build/bumpversion.sh
           nx affected:build $NX_CALCULATION_FLAGS --prod --exclude="demoshell"
           nx affected $NX_CALCULATION_FLAGS --target=pretheme
-      - name: release
+      - name: release libraries
-        run: ./scripts/travis/release/release-npm.sh
+        run: ./scripts/travis/release/release-npm.sh ${{ steps.set-dryrun.outputs.dryrun }}
 
   npm-check-bundle:
     needs: [release-npm]
     timeout-minutes: 15
-    if: github.event.pull_request.merged == true
+    if: github.event.pull_request.merged == true || ${{ inputs.dry-run-release }}
     runs-on: ubuntu-22.04
     steps:
       - name: Checkout repository