[ADF-3279] Initial refactoring of doc tools (#3541)

* [ADF-3279] Added timing option

* [ADF-3279] Updated to keep md files in memory during processing

* [ADF-3279] Changed read phase to use stored Markdown trees

* [ADF-3279] Renamed tool functions and removed obsolete ones

* [ADF-3279] Added progress bar and better error message handling

tools/doc/tools/index.js

@@ -12,10 +12,7 @@ var searchLibraryRecursive = require("../libsearch");
 var mdNav = require("../mdNav");
 
 module.exports = {
-    "initPhase": initPhase,
-    "readPhase": readPhase,
-    "aggPhase": aggPhase,
-    "updatePhase": updatePhase
+    "processDocs": processDocs
 }
 
 var angFilenameRegex = /([a-zA-Z0-9\-]+)\.((component)|(directive)|(model)|(pipe)|(service)|(widget))\.ts/;
@@ -34,6 +31,14 @@ var adfLibNames = ["core", "content-services", "insights", "process-services"];
 
 var statusIcons;
 
+
+function processDocs(mdCache, aggData, _errorMessages) {
+    initPhase(aggData);
+    readPhase(mdCache, aggData);
+    aggPhase(aggData);
+}
+
+
 function initPhase(aggData) {
     statusIcons = aggData.config["statusIcons"] || {};
     aggData.stoplist = makeStoplist(aggData.config);    
@@ -45,7 +50,16 @@ function initPhase(aggData) {
 }
 
 
-function readPhase(tree, pathname, aggData) {
+function readPhase(mdCache, aggData) {
+    var pathnames = Object.keys(mdCache);
+
+    pathnames.forEach(pathname => {
+        getFileData(mdCache[pathname].mdInTree, pathname, aggData);
+    });
+}
+
+
+function getFileData(tree, pathname, aggData) {
     var itemName = path.basename(pathname, ".md");
 
     // Look for the first paragraph in the file by skipping other items.
@@ -149,11 +163,6 @@ function aggPhase(aggData) {
 }
 
 
-function updatePhase(tree, pathname, aggData) {
-    return false;
-}
-
-
 // Create a stoplist of regular expressions.
 function makeStoplist(config) {
     var listExpressions = config.undocStoplist;
@@ -281,7 +290,7 @@ function makeMDDocumentedTableRow(docItem, forSubFolder) {
     }
 
     var srcFileLink = unist.makeLink(unist.makeText("Source"), srcPath);
-    var desc = docItem.briefDesc;
+    var desc = JSON.parse(JSON.stringify(docItem.briefDesc));
 
     removeBriefDescLinks(desc);
 
@@ -401,5 +410,6 @@ function removeBriefDescLinks(desc) {
     links.forEach(link => {
         link.item.type = "text";
         link.item.value = link.item.children[0].value;
+        link.item.children = null;
     });
 }

tools/doc/tools/toc.js

@@ -17,15 +17,18 @@ const maxTocHeadingDepth = 3;
 var templateFolder = path.resolve("tools", "doc", "templates");
 
 module.exports = {
-    "initPhase": initPhase,
-    "readPhase": readPhase,
-    "aggPhase": aggPhase,
-    "updatePhase": updatePhase
+    "processDocs": processDocs
+}
+
+
+function processDocs(mdCache, aggData, errorMessages) {
+    var pathnames = Object.keys(mdCache);
+
+    pathnames.forEach(pathname => {
+        updateFile(mdCache[pathname].mdOutTree, pathname, aggData, errorMessages);
+    });
 }
 
-function initPhase(aggData) {}
-function readPhase(tree, pathname, aggData) {}
-function aggPhase(aggData) {}
 
 
 // Find an existing Contents section or add a new empty one if needed.
@@ -80,7 +83,11 @@ function establishContentsSection(mdTree) {
     return numTocHeadings;
 }
 
-function updatePhase(tree, pathname, aggData) {
+
+
+
+
+function updateFile(tree, pathname, _aggData, _errorMessages) {
     if (path.basename(pathname, ".md").match(/README|versionIndex/)) {
         return false;
     }

tools/doc/tools/tsInfo.js

@@ -18,6 +18,26 @@ var nameExceptions;
 var undocMethodNames = {
     "ngOnChanges": 1
 };
+function processDocs(mdCache, aggData, _errorMessages) {
+    initPhase(aggData);
+    var pathnames = Object.keys(mdCache);
+    var internalErrors;
+    pathnames.forEach(function (pathname) {
+        internalErrors = [];
+        updateFile(mdCache[pathname].mdOutTree, pathname, aggData, internalErrors);
+        if (internalErrors.length > 0) {
+            showErrors(pathname, internalErrors);
+        }
+    });
+}
+exports.processDocs = processDocs;
+function showErrors(filename, errorMessages) {
+    console.log(filename);
+    errorMessages.forEach(function (message) {
+        console.log("    " + message);
+    });
+    console.log("");
+}
 var PropInfo = /** @class */ (function () {
     function PropInfo(rawProp) {
         var _this = this;
@@ -189,14 +209,7 @@ function initPhase(aggData) {
     }));
     aggData.projData = app.convert(sources);
 }
-exports.initPhase = initPhase;
-function readPhase(tree, pathname, aggData) {
-}
-exports.readPhase = readPhase;
-function aggPhase(aggData) {
-}
-exports.aggPhase = aggPhase;
-function updatePhase(tree, pathname, aggData, errorMessages) {
+function updateFile(tree, pathname, aggData, errorMessages) {
     var compName = angNameToClassName(path.basename(pathname, ".md"));
     var classRef = aggData.projData.findReflectionByName(compName);
     if (!classRef) {
@@ -232,7 +245,6 @@ function updatePhase(tree, pathname, aggData, errorMessages) {
     }
     return true;
 }
-exports.updatePhase = updatePhase;
 function initialCap(str) {
     return str[0].toUpperCase() + str.substr(1);
 }

tools/doc/tools/tsInfo.ts

@@ -40,6 +40,33 @@ let undocMethodNames = {
 };
 
 
+export function processDocs(mdCache, aggData, _errorMessages) {
+    initPhase(aggData);
+    
+    let pathnames = Object.keys(mdCache);
+    let internalErrors;
+
+    pathnames.forEach(pathname => {
+        internalErrors = [];
+        updateFile(mdCache[pathname].mdOutTree, pathname, aggData, internalErrors);
+
+        if (internalErrors.length > 0) {
+            showErrors(pathname, internalErrors);
+        }
+    });
+}
+
+
+function showErrors(filename, errorMessages) {
+    console.log(filename);
+
+    errorMessages.forEach(message => {
+        console.log("    " + message);
+    });
+
+    console.log("");
+}
+
 class PropInfo {
     name: string;
     type: string;
@@ -258,7 +285,7 @@ class ComponentInfo {
 }
 
 
-export function initPhase(aggData) {
+function initPhase(aggData) {
     nameExceptions = aggData.config.typeNameExceptions;
 
     let app = new Application({
@@ -276,15 +303,9 @@ export function initPhase(aggData) {
 }
 
 
-export function readPhase(tree, pathname, aggData) {
-}
 
 
-export function aggPhase(aggData) {
-}
-
-
-export function updatePhase(tree, pathname, aggData, errorMessages) {
+function updateFile(tree, pathname, aggData, errorMessages) {
     let compName = angNameToClassName(path.basename(pathname, ".md"));
     let classRef = aggData.projData.findReflectionByName(compName);
 

tools/doc/tools/tutorialIndex.js

@@ -11,10 +11,10 @@ var unist = require("../unistHelpers");
 var tutFolder = path.resolve("docs", "tutorials");
 var templateFolder = path.resolve("tools", "doc", "templates");
 var userGuideFolder = path.resolve("docs", "user-guide");
-function initPhase(aggData) { }
-exports.initPhase = initPhase;
-function readPhase(tree, pathname, aggData) { }
-exports.readPhase = readPhase;
+function processDocs(tree, pathname, aggData, errorMessages) {
+    aggPhase(aggData);
+}
+exports.processDocs = processDocs;
 function aggPhase(aggData) {
     var indexDocData = getIndexDocData();
     var templateName = path.resolve(templateFolder, "tutIndex.ejs");
@@ -33,11 +33,6 @@ function aggPhase(aggData) {
     });
     fs.writeFileSync(tutIndexFile, remark().use(frontMatter, { type: 'yaml', fence: '---' }).data("settings", { paddedTable: false, gfm: false }).stringify(tutIndexMD));
 }
-exports.aggPhase = aggPhase;
-function updatePhase(tree, pathname, aggData) {
-    return false;
-}
-exports.updatePhase = updatePhase;
 function getIndexDocData() {
     var indexFile = path.resolve(userGuideFolder, "summary.json");
     var summaryArray = JSON.parse(fs.readFileSync(indexFile, "utf8"));

tools/doc/tools/tutorialIndex.ts

@@ -15,12 +15,12 @@ const templateFolder = path.resolve("tools", "doc", "templates");
 const userGuideFolder = path.resolve("docs", "user-guide");
 
 
-export function initPhase(aggData) {}
-
-export function readPhase(tree, pathname, aggData) {}
+export function processDocs(tree, pathname, aggData, errorMessages) {
+    aggPhase(aggData);
+}
 
 
-export function aggPhase(aggData) {
+function aggPhase(aggData) {
     let indexDocData = getIndexDocData();
 
     let templateName = path.resolve(templateFolder, "tutIndex.ejs");
@@ -46,10 +46,6 @@ export function aggPhase(aggData) {
 }
 
 
-export function updatePhase(tree, pathname, aggData) {
-    return false;
-}
-
 
 function getIndexDocData() {
     let indexFile = path.resolve(userGuideFolder, "summary.json");

tools/doc/tools/typeLinker.js

@@ -3,6 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 var path = require("path");
 var fs = require("fs");
 var typedoc_1 = require("typedoc");
+var ProgressBar = require("progress");
 var unist = require("../unistHelpers");
 var ngHelpers = require("../ngHelpers");
 var includedNodeTypes = [
@@ -13,6 +14,21 @@ var includedNodeTypes = [
 var docFolder = path.resolve("docs");
 var adfLibNames = ["core", "content-services", "insights", "process-services"];
 var externalNameLinks;
+function processDocs(mdCache, aggData, errorMessages) {
+    initPhase(aggData);
+    var pathnames = Object.keys(mdCache);
+    var progress = new ProgressBar("Processing: [:bar] (:current/:total)", {
+        total: pathnames.length,
+        width: 50,
+        clear: true
+    });
+    pathnames.forEach(function (pathname) {
+        updateFile(mdCache[pathname].mdOutTree, pathname, aggData, errorMessages);
+        progress.tick();
+        progress.render();
+    });
+}
+exports.processDocs = processDocs;
 function initPhase(aggData) {
     externalNameLinks = aggData.config.externalNameLinks;
     aggData.docFiles = {};
@@ -36,13 +52,7 @@ function initPhase(aggData) {
     });
     //console.log(JSON.stringify(aggData.nameLookup));
 }
-exports.initPhase = initPhase;
-function readPhase(tree, pathname, aggData) { }
-exports.readPhase = readPhase;
-function aggPhase(aggData) {
-}
-exports.aggPhase = aggPhase;
-function updatePhase(tree, pathname, aggData) {
+function updateFile(tree, pathname, aggData, errorMessages) {
     traverseMDTree(tree);
     return true;
     function traverseMDTree(node) {
@@ -86,7 +96,6 @@ function updatePhase(tree, pathname, aggData) {
         */
     }
 }
-exports.updatePhase = updatePhase;
 var SplitNameNode = /** @class */ (function () {
     function SplitNameNode(key, value) {
         if (key === void 0) { key = ""; }

tools/doc/tools/typeLinker.ts

@@ -18,6 +18,8 @@ import {
  } from "typedoc";
 import { CommentTag } from "typedoc/dist/lib/models";
 
+import * as ProgressBar from "progress";
+
 import * as unist from "../unistHelpers";
 import * as ngHelpers from "../ngHelpers";
 import { match } from "minimatch";
@@ -34,7 +36,26 @@ const adfLibNames = ["core", "content-services", "insights", "process-services"]
 
 let externalNameLinks;
 
-export function initPhase(aggData) {
+export function processDocs(mdCache, aggData, errorMessages) {
+    initPhase(aggData);
+
+    var pathnames = Object.keys(mdCache);
+
+    let progress = new ProgressBar("Processing: [:bar] (:current/:total)", {
+        total: pathnames.length,
+        width: 50,
+        clear: true
+    });
+
+    pathnames.forEach(pathname => {
+        updateFile(mdCache[pathname].mdOutTree, pathname, aggData, errorMessages);
+        progress.tick();
+        progress.render();
+    });
+}
+
+
+function initPhase(aggData) {
     externalNameLinks = aggData.config.externalNameLinks;
     aggData.docFiles = {};
     aggData.nameLookup = new SplitNameLookup();
@@ -64,15 +85,10 @@ export function initPhase(aggData) {
     //console.log(JSON.stringify(aggData.nameLookup));
 }
 
-export function readPhase(tree, pathname, aggData) {}
 
 
-export function aggPhase(aggData) {
 
-}
-
-
-export function updatePhase(tree, pathname, aggData) {
+function updateFile(tree, pathname, aggData, errorMessages) {
     traverseMDTree(tree);
     return true;
 

tools/doc/tools/versionIndex.js

@@ -14,10 +14,7 @@ var ngHelpers = require("../ngHelpers");
 
 
 module.exports = {
-    "initPhase": initPhase,
-    "readPhase": readPhase,
-    "aggPhase": aggPhase,
-    "updatePhase": updatePhase
+    "processDocs": processDocs
 }
 
 var angFilenameRegex = /([a-zA-Z0-9\-]+)\.((component)|(directive)|(model)|(pipe)|(service)|(widget))\.ts/;
@@ -30,12 +27,29 @@ var initialVersion = "v2.0.0";
 
 var templateFolder = path.resolve("tools", "doc", "templates");
 
+
+function processDocs(mdCache, aggData, errorMessages) {
+    initPhase(aggData);
+    readPhase(mdCache, aggData);
+    aggPhase(aggData);
+}
+
+
 function initPhase(aggData) {
     aggData.versions = { "v2.0.0":[] };
 }
 
 
-function readPhase(tree, pathname, aggData) {
+function readPhase(mdCache, aggData) {
+    var pathnames = Object.keys(mdCache);
+
+    pathnames.forEach(pathname => {
+        getFileData(mdCache[pathname].mdInTree, pathname, aggData);
+    });
+}
+
+
+function getFileData(tree, pathname, aggData) {
     var compName = pathname;
     var angNameRegex = /([a-zA-Z0-9\-]+)\.((component)|(directive)|(model)|(pipe)|(service)|(widget))/;
 
@@ -121,8 +135,3 @@ function aggPhase(aggData) {
 
     fs.writeFileSync(histFilePath, remark().use(frontMatter, {type: 'yaml', fence: '---'}).data("settings", {paddedTable: false, gfm: false}).stringify(histFileTree));
 }
-
-
-function updatePhase(tree, pathname, aggData) {
-    return false;
-}