[ADF-3230] Changes to stop doc tools from updating files unnecessarily (#3519)

* [ADF-3230] Added basic change detector to doc tools

* [ADF-3230] Updates to type linker to fix before/after inconsistencies

* [ADF-3230] Fixed comparison error caused by adjacent text blocks

* [ADF-3230] Added basic change detector to doc tools

* [ADF-3230] Updates to type linker to fix before/after inconsistencies

* [ADF-3230] Fixed comparison error caused by adjacent text blocks

* [ADF-3230] Modified props tool to remove spaces from union types

* [ADF-3230] Made ToC tool before/after state consistent

package.json

@@ -143,6 +143,7 @@
     "loader-utils": "1.1.0",
     "markdown-toc": "1.1.0",
     "markdownlint-cli": "^0.3.1",
+    "mdast-util-compact": "^1.0.1",
     "mdast-util-heading-range": "^2.1.0",
     "mdast-util-toc": "^2.0.1",
     "mdast-zone": "^3.0.1",

tools/doc/docProcessor.js

@@ -1,11 +1,13 @@
 var fs = require("fs");
 var path = require("path");
 var program = require("commander");
+var lodash = require("lodash");
 
 var remark = require("remark");
 var parse = require("remark-parse");
 var stringify = require("remark-stringify");
 var frontMatter = require("remark-frontmatter");
+var mdCompact = require("mdast-util-compact");
 
 
 // "Aggregate" data collected over the whole file set.
@@ -51,14 +53,16 @@ function updatePhase(filenames, aggData) {
 
     for (var i = 0; i < filenames.length; i++) {
         errorMessages = [];
-        var pathname = filenames[i]; // path.resolve(srcFolder, filenames[i]);
+        var pathname = filenames[i];
         
         if (program.verbose) {
-            console.log(pathname);
+            console.log("Reading " + pathname);
         }
         
         var src = fs.readFileSync(pathname);
-        var tree = remark().use(frontMatter, ["yaml"]).parse(src)
+        var tree = remark().use(frontMatter, ["yaml"]).parse(src);
+
+        var original = minimiseTree(tree);
 
         var modified = false;
 
@@ -70,17 +74,45 @@ function updatePhase(filenames, aggData) {
             showErrors(pathname, errorMessages);
         }
 
+        tree = minimiseTree(tree);
+       
         if (program.json) {
+            let filename = path.basename(pathname);
+
+            console.log(`\nFile "${filename}" before processing:`);
+            console.log(JSON.stringify(original));
+            console.log(`\nFile "${filename}" after processing:`);
             console.log(JSON.stringify(tree));
         }
         
+        modified = !lodash.isEqual(tree, original);
+
         if (modified) {
+            if (program.verbose) {
+                console.log(`Modified: ${pathname}`);
+            }
+
             fs.writeFileSync(filenames[i], remark().use(frontMatter, {type: 'yaml', fence: '---'}).data("settings", {paddedTable: false, gfm: false}).stringify(tree));
         }
     }
 }
 
 
+function deepCopy(obj) {
+    // Despite how it looks, this technique is apparently quite efficient
+    // because the JSON routines are implemented in C code and faster
+    // than the equivalent JavaScript loops ;-)
+    return JSON.parse(JSON.stringify(obj));
+}
+
+
+function minimiseTree(tree) {
+    let minPropsTree = JSON.parse(JSON.stringify(tree, (key, value) => key === "position" ? undefined : value));
+    mdCompact(minPropsTree);
+    return minPropsTree;
+}
+
+
 function showErrors(filename, errorMessages) {
     console.log(filename);
 

tools/doc/doctool.config.json

@@ -19,7 +19,7 @@
             "toc"
         ],
         "dev": [
-            "index"
+            "toc"
         ]
     },
     "statusIcons": {

tools/doc/mdNav.js

@@ -61,6 +61,13 @@ var MDNav = /** @class */ (function () {
             return h.type === "heading" && test(h);
         }, index);
     };
+    MDNav.prototype.headings = function (test, index) {
+        if (test === void 0) { test = function () { return true; }; }
+        if (index === void 0) { index = 0; }
+        return this.findAll(function (h) {
+            return h.type === "heading" && test(h);
+        }, index);
+    };
     MDNav.prototype.html = function (test, index) {
         if (test === void 0) { test = function () { return true; }; }
         if (index === void 0) { index = 0; }
@@ -171,6 +178,28 @@ var MDNav = /** @class */ (function () {
         enumerable: true,
         configurable: true
     });
+    Object.defineProperty(MDNav.prototype, "textValue", {
+        get: function () {
+            if (this.item) {
+                if (this.item["value"]) {
+                    return this.item.value;
+                }
+                else if (this.item.children &&
+                    (this.item.children.length > 0) &&
+                    (this.item.children[0].type === "text")) {
+                    return this.item.children[0].value;
+                }
+                else {
+                    return "";
+                }
+            }
+            else {
+                return "";
+            }
+        },
+        enumerable: true,
+        configurable: true
+    });
     return MDNav;
 }());
 exports.MDNav = MDNav;

tools/doc/mdNav.ts

@@ -65,6 +65,13 @@ export class MDNav {
     }
 
 
+    headings(test: (element: any) => boolean = () => true, index: number = 0): MDNav[] {
+        return this.findAll((h) => {
+            return h.type === "heading" && test(h);
+        }, index);
+    }
+
+
     html(test: (element: any) => boolean = () => true, index: number = 0): MDNav {
         return this.find((h) => {
             return h.type === "html" && test(h);
@@ -160,4 +167,23 @@ export class MDNav {
             return "";
         }
     }
+
+
+    get textValue() : string {
+        if (this.item) { 
+            if (this.item["value"]) {
+                return this.item.value;
+            } else if (
+                this.item.children &&
+                (this.item.children.length > 0) &&
+                (this.item.children[0].type === "text")
+            ){
+                return this.item.children[0].value;
+            } else {
+                return "";
+            }
+        } else {
+            return "";
+        }
+    }
 }

tools/doc/templates/toc.ejs

@@ -0,0 +1,3 @@
+<% headings.forEach(function(heading) { -%>
+<%= "  ".repeat(heading.level); %>- [<%= heading.title %>](<%= heading.anchor %>)
+<% }); %>

tools/doc/tools/toc.js

@@ -1,15 +1,21 @@
 var path = require("path");
+var fs = require("fs");
 
-//var remark = require("remark");
+var remark = require("remark");
-var tocGenerator = require("mdast-util-toc");
+//var tocGenerator = require("mdast-util-toc");
 var replaceSection = require("mdast-util-heading-range");
 
+var ejs = require("ejs");
+
 var unist = require("../unistHelpers");
+var mdNav = require("../mdNav");
 
 const contentsHeading = "Contents";
 const minHeadingsForToc = 8;
 const maxTocHeadingDepth = 3;
 
+var templateFolder = path.resolve("tools", "doc", "templates");
+
 module.exports = {
     "initPhase": initPhase,
     "readPhase": readPhase,
@@ -83,10 +89,10 @@ function updatePhase(tree, pathname, aggData) {
     var numTocHeadings = establishContentsSection(tree);
 
     if (numTocHeadings >= minHeadingsForToc) {
-        var newToc = tocGenerator(tree, {heading: contentsHeading, maxDepth: 3});
+        var newToc = makeToc(tree); //tocGenerator(tree, {heading: contentsHeading, maxDepth: 3});
 
         replaceSection(tree, contentsHeading, function(before, oldSection, after) {
-            return [before, newToc.map, after];
+            return [before, newToc, after];
         });
     } else {
         // Otherwise, we don't need one, so remove any existing one.
@@ -97,3 +103,33 @@ function updatePhase(tree, pathname, aggData) {
 
     return true;
 }
+
+
+function makeToc(tree) {
+    var nav = new mdNav.MDNav(tree);
+
+    var headings = nav.headings(h => 
+        (h.depth > 1) &&
+        (h.depth <= maxTocHeadingDepth) &&
+        !((h.children[0].type === "text") && (h.children[0].value === "Contents"))
+    );
+
+    var context = {headings: []};
+
+    headings.forEach(heading => {
+        context.headings.push({
+            "level": heading.item.depth - 2,
+            "title": heading.textValue,
+            "anchor": "#" + heading.textValue.toLowerCase().replace(/ /g, "-").replace(/[\.,]/g ,"")
+        })
+    });
+
+    var templateName = path.resolve(templateFolder, "toc.ejs");
+    var templateSource = fs.readFileSync(templateName, "utf8");
+    var template = ejs.compile(templateSource);
+
+    var mdText = template(context);
+    var newMD = remark().parse(mdText);
+
+    return newMD.children[0];
+}

tools/doc/tools/tsInfo.js

@@ -27,7 +27,7 @@ var PropInfo = /** @class */ (function () {
         this.docText = this.docText.replace(/[\n\r]+/g, " ").trim();
         this.defaultValue = rawProp.defaultValue || "";
         this.defaultValue = this.defaultValue.replace(/\|/, "\\|");
-        this.type = rawProp.type ? rawProp.type.toString() : "";
+        this.type = rawProp.type ? rawProp.type.toString().replace(/\s/g, "") : "";
         this.type = this.type.replace(/\|/, "\\|");
         this.isDeprecated = rawProp.comment && rawProp.comment.hasTag("deprecated");
         if (this.isDeprecated) {
@@ -69,7 +69,7 @@ var PropInfo = /** @class */ (function () {
 var ParamInfo = /** @class */ (function () {
     function ParamInfo(rawParam) {
         this.name = rawParam.name;
-        this.type = rawParam.type.toString();
+        this.type = rawParam.type.toString().replace(/\s/g, "");
         this.defaultValue = rawParam.defaultValue;
         this.docText = rawParam.comment ? rawParam.comment.text : "";
         this.docText = this.docText.replace(/[\n\r]+/g, " ").trim();
@@ -88,7 +88,7 @@ var MethodSigInfo = /** @class */ (function () {
         var _this = this;
         this.errorMessages = [];
         this.name = rawSig.name;
-        this.returnType = rawSig.type ? rawSig.type.toString() : "";
+        this.returnType = rawSig.type ? rawSig.type.toString().replace(/\s/g, "") : "";
         this.returnsSomething = this.returnType != "void";
         if (rawSig.hasComment()) {
             this.docText = rawSig.comment.shortText + rawSig.comment.text;
@@ -220,7 +220,7 @@ function updatePhase(tree, pathname, aggData, errorMessages) {
         var template = ejs.compile(templateSource);
         var mdText = template(compData);
         mdText = mdText.replace(/^ +\|/mg, "|");
-        var newSection_1 = remark().data("settings", { paddedTable: false, gfm: false }).parse(mdText.trim()).children;
+        var newSection_1 = remark().parse(mdText.trim()).children;
         replaceSection(tree, "Class members", function (before, section, after) {
             newSection_1.unshift(before);
             newSection_1.push(after);

tools/doc/tools/tsInfo.ts

@@ -59,7 +59,7 @@ class PropInfo {
         this.docText = this.docText.replace(/[\n\r]+/g, " ").trim();
         this.defaultValue = rawProp.defaultValue || "";
         this.defaultValue = this.defaultValue.replace(/\|/, "\\|");
-        this.type = rawProp.type ? rawProp.type.toString() : "";
+        this.type = rawProp.type ? rawProp.type.toString().replace(/\s/g, "") : "";
         this.type = this.type.replace(/\|/, "\\|");
 
         this.isDeprecated = rawProp.comment && rawProp.comment.hasTag("deprecated");
@@ -113,7 +113,7 @@ class ParamInfo {
 
     constructor(rawParam: ParameterReflection) {
         this.name = rawParam.name;
-        this.type = rawParam.type.toString();
+        this.type = rawParam.type.toString().replace(/\s/g, "");
         this.defaultValue = rawParam.defaultValue;
         this.docText = rawParam.comment ? rawParam.comment.text : "";
         this.docText = this.docText.replace(/[\n\r]+/g, " ").trim();
@@ -148,7 +148,7 @@ class MethodSigInfo {
     constructor(rawSig: SignatureReflection) {
         this.errorMessages = [];
         this.name = rawSig.name;
-        this.returnType = rawSig.type ? rawSig.type.toString() : "";
+        this.returnType = rawSig.type ? rawSig.type.toString().replace(/\s/g, "") : "";
         this.returnsSomething = this.returnType != "void";
 
         if (rawSig.hasComment()) {
@@ -316,7 +316,7 @@ export function updatePhase(tree, pathname, aggData, errorMessages) {
         let mdText = template(compData);
         mdText = mdText.replace(/^ +\|/mg, "|");
 
-        let newSection = remark().data("settings", {paddedTable: false, gfm: false}).parse(mdText.trim()).children;
+        let newSection = remark().parse(mdText.trim()).children;
 
         replaceSection(tree, "Class members", (before, section, after) => {
             newSection.unshift(before);

tools/doc/tools/typeLinker.js

@@ -49,13 +49,16 @@ function updatePhase(tree, pathname, aggData) {
         if (!includedNodeTypes.includes(node.type)) {
             return;
         }
-        if (node.type === "inlineCode") {
+        /*if (node.type === "inlineCode") {
-            var link = resolveTypeLink(aggData, node.value);
+            console.log(`Link text: ${node.value}`);
+            let link = resolveTypeLink(aggData, node.value);
+
             if (link) {
                 convertNodeToTypeLink(node, node.value, link);
             }
-        }
+            
-        else if (node.type === "link") {
+        } else */
+        if (node.type === "link") {
             if (node.children && ((node.children[0].type === "inlineCode") ||
                 (node.children[0].type === "text"))) {
                 var link = resolveTypeLink(aggData, node.children[0].value);
@@ -64,7 +67,7 @@ function updatePhase(tree, pathname, aggData) {
                 }
             }
         }
-        else if ((node.type === "paragraph")) {
+        else if ((node.children) && (node.type !== "heading")) {
             node.children.forEach(function (child, index) {
                 if ((child.type === "text") || (child.type === "inlineCode")) {
                     var newNodes = handleLinksInBodyText(aggData, child.value, child.type === 'inlineCode');
@@ -75,12 +78,12 @@ function updatePhase(tree, pathname, aggData) {
                 }
                 var _a;
             });
-        }
+        } /*else if (node.children) {
-        else if (node.children) {
+            node.children.forEach(child => {
-            node.children.forEach(function (child) {
                 traverseMDTree(child);
             });
         }
+        */
     }
 }
 exports.updatePhase = updatePhase;
@@ -310,9 +313,11 @@ function isLinkable(kind) {
         (kind === typedoc_1.ReflectionKind.Enum) ||
         (kind === typedoc_1.ReflectionKind.TypeAlias);
 }
-function convertNodeToTypeLink(node, text, url) {
+function convertNodeToTypeLink(node, text, url, title) {
+    if (title === void 0) { title = null; }
     var linkDisplayText = unist.makeInlineCode(text);
     node.type = "link";
+    node.title = title;
     node.url = url;
     node.children = [linkDisplayText];
 }

tools/doc/tools/typeLinker.ts

@@ -32,29 +32,7 @@ const includedNodeTypes = [
 const docFolder = path.resolve("docs");
 const adfLibNames = ["core", "content-services", "insights", "process-services"];
 
-
-<<<<<<< HEAD
-const externalTypes = {
-    'Blob': 'https://developer.mozilla.org/en-US/docs/Web/API/Blob',
-    'EventEmitter': 'https://angular.io/api/core/EventEmitter',
-    'MatSnackBarRef': 'https://material.angular.io/components/snack-bar/overview',
-    'TemplateRef': 'https://angular.io/api/core/TemplateRef',
-    'Observable': 'http://reactivex.io/documentation/observable.html',
-    'Subject': 'http://reactivex.io/documentation/subject.html',
-    'AppDefinitionRepresentation': 'https://github.com/Alfresco/alfresco-js-api/blob/master/src/alfresco-activiti-rest-api/docs/AppDefinitionRepresentation.md',
-    'DeletedNodesPaging': 'https://github.com/Alfresco/alfresco-js-api/blob/master/src/alfresco-core-rest-api/docs/DeletedNodesPaging.md',
-    'MinimalNodeEntity': '../content-services/document-library.model.md',
-    'MinimalNodeEntryEntity': '../content-services/document-library.model.md',
-    'NodeEntry': 'https://github.com/Alfresco/alfresco-js-api/blob/master/src/alfresco-core-rest-api/docs/NodeEntry.md',
-    'ProcessInstanceFilterRepresentation': 'https://github.com/Alfresco/alfresco-js-api/blob/master/src/alfresco-activiti-rest-api/docs/ProcessInstanceFilterRepresentation.md',
-    'RelatedContentRepresentation': 'https://github.com/Alfresco/alfresco-js-api/blob/master/src/alfresco-activiti-rest-api/docs/RelatedContentRepresentation.md',
-    'SiteEntry': 'https://github.com/Alfresco/alfresco-js-api/blob/master/src/alfresco-core-rest-api/docs/SiteEntry.md',
-    'SitePaging': 'https://github.com/Alfresco/alfresco-js-api/blob/master/src/alfresco-core-rest-api/docs/SitePaging.md'
-};
-
-=======
 let externalNameLinks;
->>>>>>> [ADF-3150] Moved config to doctools.config.json and removed obsolete scripts
 
 export function initPhase(aggData) {
     externalNameLinks = aggData.config.externalNameLinks;
@@ -104,14 +82,16 @@ export function updatePhase(tree, pathname, aggData) {
             return;
         }
     
-        if (node.type === "inlineCode") {
+        /*if (node.type === "inlineCode") {
+            console.log(`Link text: ${node.value}`);
             let link = resolveTypeLink(aggData, node.value);
 
             if (link) {
                 convertNodeToTypeLink(node, node.value, link);
             }
             
-        } else if (node.type === "link") {
+        } else */
+        if (node.type === "link") {
             if (node.children && (
                 (node.children[0].type === "inlineCode") ||
                 (node.children[0].type === "text")
@@ -122,7 +102,7 @@ export function updatePhase(tree, pathname, aggData) {
                     convertNodeToTypeLink(node, node.children[0].value, link);
                 }
             }
-        } else if ((node.type === "paragraph")) {
+        } else if ((node.children) && (node.type !== "heading")) { //((node.type === "paragraph") || (node.type === "tableCell")) {
             node.children.forEach((child, index) => {
                 if ((child.type === "text") || (child.type === "inlineCode")) {
                     let newNodes = handleLinksInBodyText(aggData, child.value, child.type === 'inlineCode');
@@ -131,11 +111,12 @@ export function updatePhase(tree, pathname, aggData) {
                     traverseMDTree(child);
                 }
             });
-        } else if (node.children) {
+        } /*else if (node.children) {
             node.children.forEach(child => {
                 traverseMDTree(child);
             });
         }
+        */
     }
 }
 
@@ -404,9 +385,10 @@ function isLinkable(kind: ReflectionKind) {
     (kind === ReflectionKind.TypeAlias);
 }
 
-function convertNodeToTypeLink(node, text, url) {
+function convertNodeToTypeLink(node, text, url, title = null) {
     let linkDisplayText = unist.makeInlineCode(text);
     node.type = "link";
+    node.title = title;
     node.url = url;
     node.children = [linkDisplayText];
 }

tools/doc/unistHelpers.js

@@ -36,9 +36,10 @@ module.exports = {
         };
     },
 
-    makeLink: function (caption, url) {
+    makeLink: function (caption, url, title = null) {
         return {
             "type": "link",
+            "title": title,
             "url": url,
             "children": [ caption ]
         };