[ADF-4252] Added doc link fixing tool (#4461)

* [ADF-4252] Added basic fixer for link and image URLs

* [ADF-4252] Added full check to link fixing tool

* [ADF-4252] Manually replaced http Github doc URLs

* [ADF-4252] Fixed rel note doc links with tool

tools/doc/tools/gqIndex.js

@@ -8,14 +8,13 @@ var frontMatter = require("remark-frontmatter");
 var replaceZone = require("mdast-zone");
 var graphql_1 = require("graphql");
 var MQ = require("../mqDefs");
-var libNamesRegex = /content-services|core|extensions|insights|process-services|process-services-cloud/;
 var libNamesList = [
     'content-services', 'core', 'extensions',
     'insights', 'process-services', 'process-services-cloud'
 ];
 var query = "\n    query libIndex($libName: String) {\n        documents(idFilter: $libName) {\n            title: metadata(key: \"Title\")\n            status: metadata(key: \"Status\")\n            id\n            classType: folder(depth: 2)\n            heading {\n                link {\n                    url\n                }\n            }\n            paragraph {\n                plaintext\n            }\n        }\n    }\n";
 function processDocs(mdCache, aggData, _errorMessages) {
-    var docset = new GQDocset(mdCache);
+    var docset = new MQ.Docset(mdCache);
     var templateFilePath = path.resolve(__dirname, '..', 'templates', 'gqIndex.ejs');
     var templateSource = fs.readFileSync(templateFilePath, 'utf8');
     var template = ejs.compile(templateSource);
@@ -49,30 +48,3 @@ function processDocs(mdCache, aggData, _errorMessages) {
     });
 }
 exports.processDocs = processDocs;
-var GQDocset = /** @class */ (function () {
-    function GQDocset(mdCache) {
-        var _this = this;
-        this.docs = [];
-        var pathnames = Object.keys(mdCache);
-        pathnames.forEach(function (pathname) {
-            if (!pathname.match(/README/) &&
-                pathname.match(libNamesRegex)) {
-                var doc = new MQ.Root(mdCache[pathname].mdInTree);
-                doc.id = pathname.replace(/\\/g, '/');
-                _this.docs.push(doc);
-            }
-        });
-    }
-    GQDocset.prototype.documents = function (args) {
-        if (args['idFilter'] === '') {
-            return this.docs;
-        }
-        else {
-            return this.docs.filter(function (doc) { return doc.id.indexOf(args['idFilter'] + '/') !== -1; });
-        }
-    };
-    GQDocset.prototype.size = function () {
-        return this.docs.length;
-    };
-    return GQDocset;
-}());

tools/doc/tools/gqIndex.ts

@@ -11,7 +11,6 @@ import { graphql, buildSchema } from 'graphql';
 
 import * as MQ from '../mqDefs';
 
-let libNamesRegex = /content-services|core|extensions|insights|process-services|process-services-cloud/;
 let libNamesList = [
     'content-services', 'core', 'extensions',
     'insights', 'process-services', 'process-services-cloud'
@@ -39,7 +38,7 @@ let query = `
 
 
 export function processDocs(mdCache, aggData, _errorMessages) {
-    let docset: GQDocset = new GQDocset(mdCache);
+    let docset: MQ.Docset = new MQ.Docset(mdCache);
 
     let templateFilePath = path.resolve(__dirname, '..', 'templates', 'gqIndex.ejs');
     let templateSource = fs.readFileSync(templateFilePath, 'utf8');
@@ -82,35 +81,3 @@ export function processDocs(mdCache, aggData, _errorMessages) {
 }
 
 
-class GQDocset {
-    public docs: MQ.Root[];
-
-    constructor(mdCache) {
-        this.docs = [];
-
-        let pathnames = Object.keys(mdCache);
-
-        pathnames.forEach(pathname => {
-
-            if (!pathname.match(/README/) &&
-                pathname.match(libNamesRegex)
-            ) {
-                let doc = new MQ.Root(mdCache[pathname].mdInTree);
-                doc.id = pathname.replace(/\\/g, '/');
-                this.docs.push(doc);
-            }
-        });
-    }
-
-    documents(args): MQ.Root[] {
-        if (args['idFilter'] === '') {
-            return this.docs;
-        } else {
-            return this.docs.filter(doc => doc.id.indexOf(args['idFilter'] + '/') !== -1);
-        }
-    }
-
-    size(): number {
-        return this.docs.length;
-    }
-}

tools/doc/tools/linkFixer.js

@@ -0,0 +1,96 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+var path = require("path");
+var fs = require("fs");
+var unist_util_select_1 = require("unist-util-select");
+var angFilenameRegex = /([a-zA-Z0-9\-]+)\.((component)|(dialog)|(directive)|(interface)|(model)|(pipe)|(service)|(widget))/;
+var suffixesNotToCheck = /\.ts/;
+function processDocs(mdCache, aggData, errorMessages) {
+    var pathnames = Object.keys(mdCache);
+    var linkSet = new LinkSet(pathnames);
+    var imageFolderPath = path.resolve(aggData['rootFolder'], 'docs', 'docassets', 'images');
+    var imageSet = new LinkSet(getImagePaths(imageFolderPath));
+    pathnames.forEach(function (pathname) {
+        var fileBaseName = path.basename(pathname, '.md');
+        /*
+        if (!fileBaseName.match(angFilenameRegex)) {
+            return;
+        }
+        */
+        var tree = mdCache[pathname].mdOutTree;
+        fixUrls(tree, pathname, linkSet, 'link');
+        fixUrls(tree, pathname, imageSet, 'image');
+    });
+}
+exports.processDocs = processDocs;
+function fixUrls(tree, docFilePath, linkSet, selector) {
+    var linksInDoc = unist_util_select_1.selectAll(selector, tree);
+    var errors = [];
+    linksInDoc.forEach(function (linkElem) {
+        var origFullUrlPath = path.resolve(path.dirname(docFilePath), linkElem.url);
+        var hashPos = origFullUrlPath.indexOf('#');
+        var anchor = '';
+        if (hashPos !== -1) {
+            anchor = origFullUrlPath.substring(hashPos);
+            origFullUrlPath = origFullUrlPath.substring(0, hashPos);
+        }
+        if (!linkElem.url.match(/http:|https:|ftp:|mailto:/) &&
+            !path.extname(origFullUrlPath).match(suffixesNotToCheck) &&
+            (origFullUrlPath !== '') &&
+            !fs.existsSync(origFullUrlPath)) {
+            var newUrl = linkSet.update(origFullUrlPath) || origFullUrlPath;
+            linkElem.url = path.relative(path.dirname(docFilePath), newUrl).replace(/\\/g, '/') + anchor;
+            errors.push("Bad link: " + origFullUrlPath + "\nReplacing with " + linkElem.url);
+        } /*else {
+            console.log(`Link OK: ${origFullUrlPath}`);
+        }
+        */
+    });
+    if (errors.length > 0) {
+        showMessages("File: " + docFilePath + ":", errors);
+    }
+}
+function showMessages(groupName, messages) {
+    console.group(groupName);
+    messages.forEach(function (message) {
+        console.log(message);
+    });
+    console.groupEnd();
+}
+function getImagePaths(imageFolderPath) {
+    return fs.readdirSync(imageFolderPath)
+        .map(function (imageFileName) { return path.resolve(imageFolderPath, imageFileName); });
+}
+var LinkSet = /** @class */ (function () {
+    function LinkSet(urls) {
+        var _this = this;
+        this.links = new Map();
+        urls.forEach(function (url) {
+            var fileName = path.basename(url);
+            if (_this.links.has(fileName)) {
+                var item = _this.links.get(fileName);
+                item.push(url);
+            }
+            else {
+                _this.links.set(fileName, [url]);
+            }
+        });
+    }
+    LinkSet.prototype.update = function (oldUrl) {
+        var oldFileName = path.basename(oldUrl);
+        if (!this.links.has(oldFileName)) {
+            return '';
+        }
+        else {
+            var candidates = this.links.get(oldFileName);
+            if (candidates.length === 1) {
+                return candidates[0];
+            }
+            else {
+                console.log("Multiple candidates for " + oldUrl);
+                return '';
+            }
+        }
+    };
+    return LinkSet;
+}());

tools/doc/tools/linkFixer.ts

@@ -0,0 +1,122 @@
+import * as path from 'path';
+import * as fs from 'fs';
+
+import { selectAll } from 'unist-util-select';
+import { MDAST } from 'mdast';
+
+
+const angFilenameRegex = /([a-zA-Z0-9\-]+)\.((component)|(dialog)|(directive)|(interface)|(model)|(pipe)|(service)|(widget))/;
+const suffixesNotToCheck = /\.ts/;
+
+export function processDocs(mdCache, aggData, errorMessages) {
+    var pathnames = Object.keys(mdCache);
+
+    let linkSet = new LinkSet(pathnames);
+    
+    let imageFolderPath = path.resolve(aggData['rootFolder'], 'docs', 'docassets', 'images');
+
+    let imageSet = new LinkSet(getImagePaths(imageFolderPath));
+
+    pathnames.forEach(pathname => {
+        let fileBaseName = path.basename(pathname, '.md');
+
+        /*
+        if (!fileBaseName.match(angFilenameRegex)) {
+            return;
+        }
+        */
+        let tree = mdCache[pathname].mdOutTree;
+        
+        fixUrls(tree, pathname, linkSet, 'link');
+        fixUrls(tree, pathname, imageSet, 'image');
+    });
+}
+
+
+function fixUrls(tree: MDAST.Root, docFilePath: string, linkSet: LinkSet, selector: string) {
+    let linksInDoc = selectAll(selector, tree);
+
+    let errors: string[] = [];
+
+    linksInDoc.forEach(linkElem => {
+        let origFullUrlPath = path.resolve(path.dirname(docFilePath), linkElem.url);
+
+        let hashPos = origFullUrlPath.indexOf('#');
+        let anchor = '';
+
+        if (hashPos !== -1) {
+            anchor = origFullUrlPath.substring(hashPos);
+            origFullUrlPath = origFullUrlPath.substring(0, hashPos);
+        }
+
+        if (!linkElem.url.match(/http:|https:|ftp:|mailto:/) &&
+            !path.extname(origFullUrlPath).match(suffixesNotToCheck) &&
+            (origFullUrlPath !== '') &&
+            !fs.existsSync(origFullUrlPath)
+        ) {
+            let newUrl = linkSet.update(origFullUrlPath) || origFullUrlPath; 
+            linkElem.url = path.relative(path.dirname(docFilePath), newUrl).replace(/\\/g,'/') + anchor;
+            errors.push(`Bad link: ${origFullUrlPath}\nReplacing with ${linkElem.url}`);
+        } /*else {
+            console.log(`Link OK: ${origFullUrlPath}`);
+        }
+        */
+    });
+
+    if (errors.length > 0) {
+        showMessages(`File: ${docFilePath}:`, errors);
+    }
+}
+
+
+function showMessages(groupName: string, messages: string[]) {
+    console.group(groupName);
+
+    messages.forEach(message => {
+        console.log(message);
+    });
+
+    console.groupEnd();
+}
+
+function getImagePaths(imageFolderPath: string): string[] {
+    return fs.readdirSync(imageFolderPath)
+    .map(imageFileName => path.resolve(imageFolderPath, imageFileName));
+}
+
+
+class LinkSet {
+    links: Map<string, string[]>;
+
+    constructor(urls: string[]) {
+        this.links = new Map();
+
+        urls.forEach(url => {
+            let fileName = path.basename(url);
+            
+            if (this.links.has(fileName)) {
+                let item = this.links.get(fileName);
+                item.push(url);
+            } else {
+                this.links.set(fileName, [url]);
+            }
+        }); 
+    }
+
+    update(oldUrl: string): string {
+        let oldFileName = path.basename(oldUrl);
+
+        if (!this.links.has(oldFileName)) {
+            return '';
+        } else {
+            let candidates = this.links.get(oldFileName);
+
+            if (candidates.length === 1) {
+                return candidates[0];
+            } else {
+                console.log(`Multiple candidates for ${oldUrl}`);
+                return '';
+            }
+        }
+    }
+}