[AAE-1653] Script to update webdriver locally ADF (#8846)

* [AAE-1653] Script to update webdriver locally ADF

* update readme

* trigger ci

scripts/webdriver-update-newest/README.md

@@ -0,0 +1,37 @@
+# Run protractor with newest webdriver locally
+
+## Instruction
+To download newest driver simply run script from its directory
+`update-to-newest-webdriver.sh`
+
+Command accepts one parameter to define what OS you are using. By default its set to `mac-x64`
+Possible inputs `linux64, mac-arm64, mac-x64, win32, win64`
+
+Example `./update-to-newest-webdriver.sh win64` - will set driver for windows
+
+## How it works
+1. The script removes your existing driver files from webdriver node_modules
+2. Generates two new files (chrome_xml.js and update.js) that have updated methods needed to get the new driver
+3. Replaces browser type depending on parameter
+4. Copies and replaces the files to the webdriver node_modules
+5. Executes command to to update-webdriver using updated code
+
+## Troubleshooting
+If the script fails for any reason. You can do some of these actions manually:
+1. Find the two files (chrome_xml.js and update.js) in node_modules/webdriver-manager
+2. Replace its contents with (chrome_xml_schema.js and update_schema.js) keep the original names.
+3. Change version for specific OS in both files
+    chrome_xml.js -> ['platform'] == 'mac-x64' e.g. ['platform'] == 'win64' Line 70
+    update.js -> 'chromedriver-mac-x64' e.g 'chromedriver-win64' Line 240
+4. Run standard command to update webdriver `./node_modules/webdriver-manager/bin/webdriver-manager update --gecko=false`
+
+
+
+## Reason
+Latest ChromeDriver Binaries https://googlechromelabs.github.io/chrome-for-testing/
+
+Starting with M115 the latest Chrome + ChromeDriver releases per release channel (Stable, Beta, Dev, Canary) are available at the Chrome for Testing availability dashboard. For automated version downloading one can use the convenient JSON endpoints.
+The older releases can be found at the Downloads page.
+
+
+Note: Protractor is a depricated tool and this probably won't be fixed.

scripts/webdriver-update-newest/chrome_xml_schema.js

@@ -0,0 +1,184 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const semver = require("semver");
+const config_1 = require("../config");
+const http_utils_1 = require("../http_utils");
+const config_source_1 = require("./config_source");
+class ChromeXml extends config_source_1.XmlConfigSource {
+    constructor() {
+        super('chrome', config_1.Config.cdnUrls()['chrome']);
+        this.maxVersion = config_1.Config.binaryVersions().maxChrome;
+    }
+    getUrl(version) {
+        if (version === 'latest') {
+            return this.getLatestChromeDriverVersion();
+        }
+        else {
+            return this.getSpecificChromeDriverVersion(version);
+        }
+    }
+    /**
+     * Get a list of chrome drivers paths available for the configuration OS type and architecture.
+     */
+    getVersionList() {
+        return this.getXml().then(xml => {
+            let versionPaths = [];
+            let osType = this.getOsTypeName();
+            for (let content of xml.ListBucketResult.Contents) {
+                let contentKey = content.Key[0];
+                if (
+                // Filter for 32-bit devices, make sure x64 is not an option
+                (this.osarch.includes('64') || !contentKey.includes('64')) &&
+                    // Filter for x86 macs, make sure m1 is not an option
+                    ((this.ostype === 'Darwin' && this.osarch === 'arm64') || !contentKey.includes('m1'))) {
+                    // Filter for only the osType
+                    if (contentKey.includes(osType)) {
+                        versionPaths.push(contentKey);
+                    }
+                }
+            }
+            return versionPaths;
+        });
+    }
+    /**
+     * Helper method, gets the ostype and gets the name used by the XML
+     */
+    getOsTypeName() {
+        // Get the os type name.
+        if (this.ostype === 'Darwin') {
+            return 'mac';
+        }
+        else if (this.ostype === 'Windows_NT') {
+            return 'win';
+        }
+        else {
+            return 'linux';
+        }
+    }
+    /**
+     * Gets the latest item from the XML.
+     */
+    getLatestChromeDriverVersion() {
+        const path = require('path')
+        const fs = require('fs')
+
+        const lastKnownGoodVersionsWithDownloads_Url = 'https://googlechromelabs.github.io/chrome-for-testing/last-known-good-versions-with-downloads.json';
+        return http_utils_1.requestBody(lastKnownGoodVersionsWithDownloads_Url).then(body => {
+            const latestVersion_Body = JSON.parse(body)['channels']['Stable']
+
+            const latestVersion = latestVersion_Body['version']
+            const latestVersion_Url = latestVersion_Body['downloads']['chromedriver'].find(obj => obj['platform'] == 'mac-x64')['url']
+
+            const latestMajorVersion = latestVersion.split('.')[0]
+
+            const localVersion_FileName = fs.readdirSync(path.resolve(__dirname, '..', '..', '..', 'selenium'))
+                .find(f => f.startsWith(`chromedriver_${latestMajorVersion}`)) || ''
+
+            const localVersion = localVersion_FileName.slice(13, -4)
+            const localVersion_Url = latestVersion_Url.replace(latestVersion, localVersion)
+
+            const localMajorVersion = localVersion.split('.')[0]
+
+            if (latestMajorVersion == localMajorVersion) {
+                return Promise.resolve({
+                    url: localVersion_Url,
+                    version: localVersion,
+                })
+            } else {
+                return Promise.resolve({
+                    url: latestVersion_Url,
+                    version: latestVersion,
+                })
+            }
+        });
+    }
+    /**
+     * Gets a specific item from the XML.
+     */
+    getSpecificChromeDriverVersion(inputVersion) {
+        return this.getVersionList().then(list => {
+            const specificVersion = getValidSemver(inputVersion);
+            if (specificVersion === '') {
+                throw new Error(`version ${inputVersion} ChromeDriver does not exist`);
+            }
+            let itemFound = '';
+            for (let item of list) {
+                // Get a semantic version.
+                let version = item.split('/')[0];
+                if (semver.valid(version) == null) {
+                    const lookUpVersion = getValidSemver(version);
+                    if (semver.valid(lookUpVersion)) {
+                        // Check to see if the specified version matches.
+                        if (lookUpVersion === specificVersion) {
+                            // When item found is null, check the os arch
+                            // 64-bit version works OR not 64-bit version and the path does not have '64'
+                            if (itemFound == '') {
+                                if (this.osarch === 'x64' ||
+                                    (this.osarch !== 'x64' && !item.includes(this.getOsTypeName() + '64'))) {
+                                    itemFound = item;
+                                }
+                                if (this.osarch === 'arm64' && this.ostype === 'Darwin' && item.includes('m1')) {
+                                    itemFound = item;
+                                }
+                            }
+                            else if (this.osarch === 'x64') {
+                                // No win64 version exists, so even on x64 we need to look for win32
+                                const osTypeNameAndArch = this.getOsTypeName() + (this.getOsTypeName() === 'win' ? '32' : '64');
+                                if (item.includes(osTypeNameAndArch)) {
+                                    itemFound = item;
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+            if (itemFound == '') {
+                return { url: '', version: inputVersion };
+            }
+            else {
+                return { url: config_1.Config.cdnUrls().chrome + itemFound, version: inputVersion };
+            }
+        });
+    }
+}
+exports.ChromeXml = ChromeXml;
+/**
+ * Chromedriver is the only binary that does not conform to semantic versioning
+ * and either has too little number of digits or too many. To get this to be in
+ * semver, we will either add a '.0' at the end or chop off the last set of
+ * digits. This is so we can compare to find the latest and greatest.
+ *
+ * Example:
+ *   2.46 -> 2.46.0
+ *   75.0.3770.8 -> 75.0.3770
+ *
+ * @param version
+ */
+function getValidSemver(version) {
+    let lookUpVersion = '';
+    // This supports downloading 2.46
+    try {
+        const oldRegex = /(\d+.\d+)/g;
+        const exec = oldRegex.exec(version);
+        if (exec) {
+            lookUpVersion = exec[1] + '.0';
+        }
+    }
+    catch (_) {
+        // no-op: is this is not valid, do not throw here.
+    }
+    // This supports downloading 74.0.3729.6
+    try {
+        const newRegex = /(\d+.\d+.\d+).\d+/g;
+        const exec = newRegex.exec(version);
+        if (exec) {
+            lookUpVersion = exec[1];
+        }
+    }
+    catch (_) {
+        // no-op: if this does not work, use the other regex pattern.
+    }
+    return lookUpVersion;
+}
+exports.getValidSemver = getValidSemver;
+//# sourceMappingURL=chrome_xml.js.map

scripts/webdriver-update-newest/update-to-newest-webdriver.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+#set -x
+BROWSER_TYPE=mac-x64
+
+if [ ! -z "$1" ]; then BROWSER_TYPE=$1 ; fi
+
+PATH_TO_COMMANDS=../../node_modules/webdriver-manager/built/lib/cmds
+PATH_TO_BINARIES=../../node_modules/webdriver-manager/built/lib/binaries
+PATH_TO_SELENIUM=../../node_modules/webdriver-manager/selenium
+
+# Remove existing drivers
+rm -rf $PATH_TO_SELENIUM/selenium-server-*
+rm -rf $PATH_TO_SELENIUM/chromedriver-*
+rm -f $PATH_TO_SELENIUM/chromedriver_*
+
+# Replace browser type in file and create new file
+sed  "s/mac-x64/$BROWSER_TYPE/" chrome_xml_schema.js > chrome_xml.js
+sed  "s/mac-x64/$BROWSER_TYPE/" update_schema.js > update.js
+sleep 2
+
+# Replace webdriver files
+cp -f update.js $PATH_TO_COMMANDS/update.js
+cp -f chrome_xml.js $PATH_TO_BINARIES/chrome_xml.js
+
+rm -f update.js
+rm -f chrome_xml.js
+
+#$(npm bin)/webdriver-manager update --gecko=false
+node ../../node_modules/webdriver-manager/bin/webdriver-manager update --gecko=false

scripts/webdriver-update-newest/update_schema.js

@@ -0,0 +1,304 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const AdmZip = require("adm-zip");
+const child_process = require("child_process");
+const fs = require("fs");
+const minimist = require("minimist");
+const path = require("path");
+const q = require("q");
+const rimraf = require("rimraf");
+const binaries_1 = require("../binaries");
+const cli_1 = require("../cli");
+const config_1 = require("../config");
+const files_1 = require("../files");
+const http_utils_1 = require("../http_utils");
+const utils_1 = require("../utils");
+const Opt = require("./");
+const initialize_1 = require("./initialize");
+const opts_1 = require("./opts");
+config_1.Config.runCommand = 'update';
+let logger = new cli_1.Logger('update');
+let prog = new cli_1.Program()
+    .command('update', 'install or update selected binaries')
+    .action(update)
+    .addOption(opts_1.Opts[Opt.OUT_DIR])
+    .addOption(opts_1.Opts[Opt.VERBOSE])
+    .addOption(opts_1.Opts[Opt.IGNORE_SSL])
+    .addOption(opts_1.Opts[Opt.PROXY])
+    .addOption(opts_1.Opts[Opt.ALTERNATE_CDN])
+    .addOption(opts_1.Opts[Opt.STANDALONE])
+    .addOption(opts_1.Opts[Opt.CHROME])
+    .addOption(opts_1.Opts[Opt.GECKO])
+    .addOption(opts_1.Opts[Opt.ANDROID])
+    .addOption(opts_1.Opts[Opt.ANDROID_API_LEVELS])
+    .addOption(opts_1.Opts[Opt.ANDROID_ARCHITECTURES])
+    .addOption(opts_1.Opts[Opt.ANDROID_PLATFORMS])
+    .addOption(opts_1.Opts[Opt.ANDROID_ACCEPT_LICENSES]);
+if (config_1.Config.osType() === 'Darwin') {
+    prog.addOption(opts_1.Opts[Opt.IOS]);
+}
+if (config_1.Config.osType() === 'Windows_NT') {
+    prog.addOption(opts_1.Opts[Opt.IE]).addOption(opts_1.Opts[Opt.IE32]).addOption(opts_1.Opts[Opt.IE64]);
+}
+prog.addOption(opts_1.Opts[Opt.VERSIONS_STANDALONE])
+    .addOption(opts_1.Opts[Opt.VERSIONS_CHROME])
+    .addOption(opts_1.Opts[Opt.VERSIONS_APPIUM])
+    .addOption(opts_1.Opts[Opt.VERSIONS_ANDROID])
+    .addOption(opts_1.Opts[Opt.VERSIONS_GECKO]);
+if (config_1.Config.osType() === 'Windows_NT') {
+    prog.addOption(opts_1.Opts[Opt.VERSIONS_IE]);
+}
+exports.program = prog;
+// stand alone runner
+let argv = minimist(process.argv.slice(2), prog.getMinimistOptions());
+if (argv._[0] === 'update-run') {
+    prog.run(JSON.parse(JSON.stringify(argv)));
+}
+else if (argv._[0] === 'update-help') {
+    prog.printHelp();
+}
+let browserFile;
+/**
+ * Parses the options and downloads binaries if they do not exist.
+ * @param options
+ */
+function update(options) {
+    let promises = [];
+    let standalone = options[Opt.STANDALONE].getBoolean();
+    let chrome = options[Opt.CHROME].getBoolean();
+    let gecko = options[Opt.GECKO].getBoolean();
+    let ie32 = false;
+    let ie64 = false;
+    if (options[Opt.IE]) {
+        ie32 = ie32 || options[Opt.IE].getBoolean();
+    }
+    if (options[Opt.IE32]) {
+        ie32 = ie32 || options[Opt.IE32].getBoolean();
+    }
+    if (options[Opt.IE64]) {
+        ie64 = options[Opt.IE64].getBoolean();
+    }
+    let android = options[Opt.ANDROID].getBoolean();
+    let ios = false;
+    if (options[Opt.IOS]) {
+        ios = options[Opt.IOS].getBoolean();
+    }
+    let outputDir = options[Opt.OUT_DIR].getString();
+    try {
+        browserFile =
+            JSON.parse(fs.readFileSync(path.resolve(outputDir, 'update-config.json')).toString());
+    }
+    catch (err) {
+        browserFile = {};
+    }
+    let android_api_levels = options[Opt.ANDROID_API_LEVELS].getString().split(',');
+    let android_architectures = options[Opt.ANDROID_ARCHITECTURES].getString().split(',');
+    let android_platforms = options[Opt.ANDROID_PLATFORMS].getString().split(',');
+    let android_accept_licenses = options[Opt.ANDROID_ACCEPT_LICENSES].getBoolean();
+    if (options[Opt.OUT_DIR].getString()) {
+        if (path.isAbsolute(options[Opt.OUT_DIR].getString())) {
+            outputDir = options[Opt.OUT_DIR].getString();
+        }
+        else {
+            outputDir = path.resolve(config_1.Config.getBaseDir(), options[Opt.OUT_DIR].getString());
+        }
+        files_1.FileManager.makeOutputDirectory(outputDir);
+    }
+    let ignoreSSL = options[Opt.IGNORE_SSL].getBoolean();
+    let proxy = options[Opt.PROXY].getString();
+    http_utils_1.HttpUtils.assignOptions({ ignoreSSL, proxy });
+    let verbose = options[Opt.VERBOSE].getBoolean();
+    // setup versions for binaries
+    let binaries = files_1.FileManager.setupBinaries(options[Opt.ALTERNATE_CDN].getString());
+    binaries[binaries_1.Standalone.id].versionCustom = options[Opt.VERSIONS_STANDALONE].getString();
+    binaries[binaries_1.ChromeDriver.id].versionCustom = options[Opt.VERSIONS_CHROME].getString();
+    if (options[Opt.VERSIONS_IE]) {
+        binaries[binaries_1.IEDriver.id].versionCustom = options[Opt.VERSIONS_IE].getString();
+    }
+    if (options[Opt.VERSIONS_GECKO]) {
+        binaries[binaries_1.GeckoDriver.id].versionCustom = options[Opt.VERSIONS_GECKO].getString();
+    }
+    binaries[binaries_1.AndroidSDK.id].versionCustom = options[Opt.VERSIONS_ANDROID].getString();
+    binaries[binaries_1.Appium.id].versionCustom = options[Opt.VERSIONS_APPIUM].getString();
+    // if the file has not been completely downloaded, download it
+    // else if the file has already been downloaded, unzip the file, rename it, and give it
+    // permissions
+    if (standalone) {
+        let binary = binaries[binaries_1.Standalone.id];
+        promises.push(files_1.FileManager.downloadFile(binary, outputDir)
+            .then((downloaded) => {
+            if (!downloaded) {
+                logger.info(binary.name + ': file exists ' +
+                    path.resolve(outputDir, binary.filename()));
+                logger.info(binary.name + ': ' + binary.filename() + ' up to date');
+            }
+        })
+            .then(() => {
+            updateBrowserFile(binary, outputDir);
+        }));
+    }
+    if (chrome) {
+        let binary = binaries[binaries_1.ChromeDriver.id];
+        promises.push(updateBinary(binary, outputDir, proxy, ignoreSSL).then(() => {
+            return Promise.resolve(updateBrowserFile(binary, outputDir));
+        }));
+    }
+    if (gecko) {
+        let binary = binaries[binaries_1.GeckoDriver.id];
+        promises.push(updateBinary(binary, outputDir, proxy, ignoreSSL).then(() => {
+            return Promise.resolve(updateBrowserFile(binary, outputDir));
+        }));
+    }
+    if (ie64) {
+        let binary = binaries[binaries_1.IEDriver.id];
+        binary.osarch = config_1.Config.osArch(); // Win32 or x64
+        promises.push(updateBinary(binary, outputDir, proxy, ignoreSSL).then(() => {
+            return Promise.resolve(updateBrowserFile(binary, outputDir));
+        }));
+    }
+    if (ie32) {
+        let binary = binaries[binaries_1.IEDriver.id];
+        binary.osarch = 'Win32';
+        promises.push(updateBinary(binary, outputDir, proxy, ignoreSSL).then(() => {
+            return Promise.resolve(updateBrowserFile(binary, outputDir));
+        }));
+    }
+    if (android) {
+        let binary = binaries[binaries_1.AndroidSDK.id];
+        let sdk_path = path.resolve(outputDir, binary.executableFilename());
+        let oldAVDList;
+        updateBrowserFile(binary, outputDir);
+        promises.push(q.nfcall(fs.readFile, path.resolve(sdk_path, 'available_avds.json'))
+            .then((oldAVDs) => {
+            oldAVDList = oldAVDs;
+        }, () => {
+            oldAVDList = '[]';
+        })
+            .then(() => {
+            return updateBinary(binary, outputDir, proxy, ignoreSSL);
+        })
+            .then(() => {
+            initialize_1.android(path.resolve(outputDir, binary.executableFilename()), android_api_levels, android_architectures, android_platforms, android_accept_licenses, binaries[binaries_1.AndroidSDK.id].versionCustom, JSON.parse(oldAVDList), logger, verbose);
+        }));
+    }
+    if (ios) {
+        initialize_1.iOS(logger);
+    }
+    if (android || ios) {
+        installAppium(binaries[binaries_1.Appium.id], outputDir);
+        updateBrowserFile(binaries[binaries_1.Appium.id], outputDir);
+    }
+    return Promise.all(promises).then(() => {
+        writeBrowserFile(outputDir);
+    });
+}
+function updateBinary(binary, outputDir, proxy, ignoreSSL) {
+    return files_1.FileManager
+        .downloadFile(binary, outputDir, (binary, outputDir, fileName) => {
+        unzip(binary, outputDir, fileName);
+    })
+        .then(downloaded => {
+        if (!downloaded) {
+            // The file did not have to download, we should unzip it.
+            logger.info(binary.name + ': file exists ' + path.resolve(outputDir, binary.filename()));
+            let fileName = binary.filename();
+            unzip(binary, outputDir, fileName);
+            logger.info(binary.name + ': ' + binary.executableFilename() + ' up to date');
+        }
+    });
+}
+function unzip(binary, outputDir, fileName) {
+    // remove the previously saved file and unzip it
+    let osType = config_1.Config.osType();
+    let mv = path.resolve(outputDir, binary.executableFilename());
+    try {
+        fs.unlinkSync(mv);
+    }
+    catch (err) {
+        try {
+            rimraf.sync(mv);
+        }
+        catch (err2) {
+        }
+    }
+    // unzip the file
+    logger.info(binary.name + ': unzipping ' + fileName);
+    if (fileName.slice(-4) == '.zip') {
+        try {
+            let zip = new AdmZip(path.resolve(outputDir, fileName));
+            zip.extractAllTo(outputDir, true);
+        }
+        catch (e) {
+            throw new Error(`Invalid filename: ${path.resolve(outputDir, fileName)}`);
+        }
+    }
+    else {
+        // We will only ever get .tar files on linux
+        child_process.spawnSync('tar', ['zxvf', path.resolve(outputDir, fileName), '-C', outputDir]);
+    }
+    // rename
+    if (fileName.indexOf('chromedriver_') != -1) {
+        fs.renameSync(path.resolve(outputDir, 'chromedriver-mac-x64', binary.zipContentName()), mv)
+    } else {
+        fs.renameSync(path.resolve(outputDir, binary.zipContentName()), mv);
+    }
+    // set permissions
+    if (osType !== 'Windows_NT') {
+        logger.info(binary.name + ': setting permissions to 0755 for ' + mv);
+        if (binary.id() !== binaries_1.AndroidSDK.id) {
+            fs.chmodSync(mv, '0755');
+        }
+        else {
+            fs.chmodSync(path.resolve(mv, 'tools', 'android'), '0755');
+            fs.chmodSync(path.resolve(mv, 'tools', 'emulator'), '0755');
+            // TODO(sjelin): get 64 bit versions working
+        }
+    }
+}
+function installAppium(binary, outputDir) {
+    logger.info('appium: installing appium');
+    let folder = path.resolve(outputDir, binary.filename());
+    try {
+        rimraf.sync(folder);
+    }
+    catch (err) {
+    }
+    fs.mkdirSync(folder);
+    fs.writeFileSync(path.resolve(folder, 'package.json'), JSON.stringify({ scripts: { appium: 'appium' } }));
+    utils_1.spawn('npm', ['install', 'appium@' + binary.version()], null, { cwd: folder });
+}
+function updateBrowserFile(binary, outputDir) {
+    let currentDownload = path.resolve(outputDir, binary.executableFilename());
+    // if browserFile[id] exists, we should update it
+    if (browserFile[binary.id()]) {
+        let binaryPath = browserFile[binary.id()];
+        if (binaryPath.last === currentDownload) {
+            return;
+        }
+        else {
+            binaryPath.last = currentDownload;
+            for (let bin of binaryPath.all) {
+                if (bin === currentDownload) {
+                    return;
+                }
+            }
+            binaryPath.all.push(currentDownload);
+        }
+    }
+    else {
+        // The browserFile[id] does not exist / has not been downloaded previously.
+        // We should create the entry.
+        let binaryPath = { last: currentDownload, all: [currentDownload] };
+        browserFile[binary.id()] = binaryPath;
+    }
+}
+function writeBrowserFile(outputDir) {
+    let filePath = path.resolve(outputDir, 'update-config.json');
+    fs.writeFileSync(filePath, JSON.stringify(browserFile));
+}
+// for testing
+function clearBrowserFile() {
+    browserFile = {};
+}
+exports.clearBrowserFile = clearBrowserFile;
+//# sourceMappingURL=update.js.map