[ADF-3279] Initial refactoring of doc tools (#3541)

* [ADF-3279] Added timing option

* [ADF-3279] Updated to keep md files in memory during processing

* [ADF-3279] Changed read phase to use stored Markdown trees

* [ADF-3279] Renamed tool functions and removed obsolete ones

* [ADF-3279] Added progress bar and better error message handling

tools/doc/docProcessor.js

@@ -18,6 +18,7 @@ var configFileName = "doctool.config.json";
 var defaultFolder = path.resolve("docs");
 
 
+/*
 function initPhase(aggData) {
     toolList.forEach(toolName => {
         toolModules[toolName].initPhase(aggData);
@@ -25,19 +26,10 @@ function initPhase(aggData) {
 }
 
 
-function readPhase(filenames, aggData) {
-    for (var i = 0; i < filenames.length; i++) {
-        var pathname = filenames[i];//path.resolve(srcFolder, filenames[i]);
-        
-        var src = fs.readFileSync(pathname);
-        var tree = remark().use(frontMatter, ["yaml"]).parse(src);
-
-        toolList.forEach(toolName => {
-            toolModules[toolName].readPhase(tree, pathname, aggData);
-        });
-    }
-
-    //console.log(JSON.stringify(aggData.mdFileData));
+function readPhase(mdCache, aggData) {
+    toolList.forEach(toolName => {
+        toolModules[toolName].readPhase(mdCache, aggData);
+    });
 }
 
 
@@ -46,36 +38,25 @@ function aggPhase(aggData) {
         toolModules[toolName].aggPhase(aggData);
     });
 }
+*/
 
-
-function updatePhase(filenames, aggData) {
+function updatePhase(mdCache, aggData) {
     var errorMessages;
 
-    for (var i = 0; i < filenames.length; i++) {
+    toolList.forEach(toolName => {
         errorMessages = [];
+        console.log(`Tool: ${toolName}`);
+        toolModules[toolName].processDocs(mdCache, aggData, errorMessages);
+    });
+
+    var filenames = Object.keys(mdCache);
+    
+
+    for (var i = 0; i < filenames.length; i++) {
         var pathname = filenames[i];
-        
-        if (program.verbose) {
-            console.log("Reading " + pathname);
-        }
-        
-        var src = fs.readFileSync(pathname);
-        var tree = remark().use(frontMatter, ["yaml"]).parse(src);
+        var tree = mdCache[pathname].mdOutTree;
+        var original = mdCache[pathname].mdInTree;
 
-        var original = minimiseTree(tree);
-
-        var modified = false;
-
-        toolList.forEach(toolName => {
-            modified |= toolModules[toolName].updatePhase(tree, pathname, aggData, errorMessages);
-        });
-        
-        if (errorMessages.length > 0) {
-            showErrors(pathname, errorMessages);
-        }
-
-        tree = minimiseTree(tree);
-       
         if (program.json) {
             let filename = path.basename(pathname);
 
@@ -84,10 +65,8 @@ function updatePhase(filenames, aggData) {
             console.log(`\nFile "${filename}" after processing:`);
             console.log(JSON.stringify(tree));
         }
-        
-        modified = !lodash.isEqual(tree, original);
 
-        if (modified) {
+        if (!lodash.isEqual(tree, original)) {
             if (program.verbose) {
                 console.log(`Modified: ${pathname}`);
             }
@@ -113,17 +92,6 @@ function minimiseTree(tree) {
 }
 
 
-function showErrors(filename, errorMessages) {
-    console.log(filename);
-
-    errorMessages.forEach(message => {
-        console.log("    " + message);
-    });
-
-    console.log("");
-}
-
-
 function loadToolModules() {
     var mods = {};
     var toolsFolderPath = path.resolve(__dirname, toolsFolderName);
@@ -164,13 +132,37 @@ function getAllDocFilePaths(docFolder, files) {
 }
 
 
+function initMdCache(filenames) {
+    var mdCache = {};
+
+    for (var i = 0; i < filenames.length; i++) {
+        var pathname = filenames[i];
+        mdCache[pathname] = {};
+
+        var src = fs.readFileSync(pathname);
+        var tree = remark().use(frontMatter, ["yaml"]).parse(src);
+        mdCache[pathname].mdInTree = minimiseTree(tree);
+        mdCache[pathname].mdOutTree = minimiseTree(tree);
+    }
+
+    return mdCache;
+}
+
+
 program
 .usage("[options] <source>")
 .option("-p, --profile [profileName]", "Select named config profile", "default")
 .option("-j, --json", "Output JSON data for Markdown syntax tree")
 .option("-v, --verbose", "Log doc files as they are processed")
+.option("-t, --timing", "Output time taken for run")
 .parse(process.argv);
 
+var startTime;
+
+if (program.timing) {
+    startTime = process.hrtime();
+}
+
 var sourcePath;
 
 if (program.args.length === 0) {
@@ -206,23 +198,29 @@ if (sourceInfo.isDirectory()) {
 }
 
 files = files.filter(filename => 
+    (filename !== undefined) &&
     (path.extname(filename) === ".md") &&
     (filename !== "README.md")
 );
 
-//files.forEach(element => console.log(element));
 
+var mdCache = initMdCache(files);
+
+/*
 console.log("Initialising...");
 initPhase(aggData);
 
 console.log("Analysing Markdown files...");
-readPhase(files, aggData);
+readPhase(mdCache, aggData);
 
 console.log("Computing aggregate data...");
 aggPhase(aggData);
+*/
 
 console.log("Updating Markdown files...");
-updatePhase(files, aggData);
-
-
+updatePhase(mdCache, aggData);
 
+if (program.timing) {
+    var endTime = process.hrtime(startTime);
+    console.log(`Run complete in ${endTime[0]} sec`);
+}

tools/doc/doctool.config.json

@@ -19,7 +19,8 @@
             "toc"
         ],
         "dev": [
-            "toc"
+            "tsInfo",
+            "typeLinker"
         ]
     },
     "statusIcons": {

tools/doc/tools/index.js

@@ -12,10 +12,7 @@ var searchLibraryRecursive = require("../libsearch");
 var mdNav = require("../mdNav");
 
 module.exports = {
-    "initPhase": initPhase,
-    "readPhase": readPhase,
-    "aggPhase": aggPhase,
-    "updatePhase": updatePhase
+    "processDocs": processDocs
 }
 
 var angFilenameRegex = /([a-zA-Z0-9\-]+)\.((component)|(directive)|(model)|(pipe)|(service)|(widget))\.ts/;
@@ -34,6 +31,14 @@ var adfLibNames = ["core", "content-services", "insights", "process-services"];
 
 var statusIcons;
 
+
+function processDocs(mdCache, aggData, _errorMessages) {
+    initPhase(aggData);
+    readPhase(mdCache, aggData);
+    aggPhase(aggData);
+}
+
+
 function initPhase(aggData) {
     statusIcons = aggData.config["statusIcons"] || {};
     aggData.stoplist = makeStoplist(aggData.config);    
@@ -45,7 +50,16 @@ function initPhase(aggData) {
 }
 
 
-function readPhase(tree, pathname, aggData) {
+function readPhase(mdCache, aggData) {
+    var pathnames = Object.keys(mdCache);
+
+    pathnames.forEach(pathname => {
+        getFileData(mdCache[pathname].mdInTree, pathname, aggData);
+    });
+}
+
+
+function getFileData(tree, pathname, aggData) {
     var itemName = path.basename(pathname, ".md");
 
     // Look for the first paragraph in the file by skipping other items.
@@ -149,11 +163,6 @@ function aggPhase(aggData) {
 }
 
 
-function updatePhase(tree, pathname, aggData) {
-    return false;
-}
-
-
 // Create a stoplist of regular expressions.
 function makeStoplist(config) {
     var listExpressions = config.undocStoplist;
@@ -281,7 +290,7 @@ function makeMDDocumentedTableRow(docItem, forSubFolder) {
     }
 
     var srcFileLink = unist.makeLink(unist.makeText("Source"), srcPath);
-    var desc = docItem.briefDesc;
+    var desc = JSON.parse(JSON.stringify(docItem.briefDesc));
 
     removeBriefDescLinks(desc);
 
@@ -401,5 +410,6 @@ function removeBriefDescLinks(desc) {
     links.forEach(link => {
         link.item.type = "text";
         link.item.value = link.item.children[0].value;
+        link.item.children = null;
     });
 }

tools/doc/tools/toc.js

@@ -17,15 +17,18 @@ const maxTocHeadingDepth = 3;
 var templateFolder = path.resolve("tools", "doc", "templates");
 
 module.exports = {
-    "initPhase": initPhase,
-    "readPhase": readPhase,
-    "aggPhase": aggPhase,
-    "updatePhase": updatePhase
+    "processDocs": processDocs
+}
+
+
+function processDocs(mdCache, aggData, errorMessages) {
+    var pathnames = Object.keys(mdCache);
+
+    pathnames.forEach(pathname => {
+        updateFile(mdCache[pathname].mdOutTree, pathname, aggData, errorMessages);
+    });
 }
 
-function initPhase(aggData) {}
-function readPhase(tree, pathname, aggData) {}
-function aggPhase(aggData) {}
 
 
 // Find an existing Contents section or add a new empty one if needed.
@@ -80,7 +83,11 @@ function establishContentsSection(mdTree) {
     return numTocHeadings;
 }
 
-function updatePhase(tree, pathname, aggData) {
+
+
+
+
+function updateFile(tree, pathname, _aggData, _errorMessages) {
     if (path.basename(pathname, ".md").match(/README|versionIndex/)) {
         return false;
     }

tools/doc/tools/tsInfo.js

@@ -18,6 +18,26 @@ var nameExceptions;
 var undocMethodNames = {
     "ngOnChanges": 1
 };
+function processDocs(mdCache, aggData, _errorMessages) {
+    initPhase(aggData);
+    var pathnames = Object.keys(mdCache);
+    var internalErrors;
+    pathnames.forEach(function (pathname) {
+        internalErrors = [];
+        updateFile(mdCache[pathname].mdOutTree, pathname, aggData, internalErrors);
+        if (internalErrors.length > 0) {
+            showErrors(pathname, internalErrors);
+        }
+    });
+}
+exports.processDocs = processDocs;
+function showErrors(filename, errorMessages) {
+    console.log(filename);
+    errorMessages.forEach(function (message) {
+        console.log("    " + message);
+    });
+    console.log("");
+}
 var PropInfo = /** @class */ (function () {
     function PropInfo(rawProp) {
         var _this = this;
@@ -189,14 +209,7 @@ function initPhase(aggData) {
     }));
     aggData.projData = app.convert(sources);
 }
-exports.initPhase = initPhase;
-function readPhase(tree, pathname, aggData) {
-}
-exports.readPhase = readPhase;
-function aggPhase(aggData) {
-}
-exports.aggPhase = aggPhase;
-function updatePhase(tree, pathname, aggData, errorMessages) {
+function updateFile(tree, pathname, aggData, errorMessages) {
     var compName = angNameToClassName(path.basename(pathname, ".md"));
     var classRef = aggData.projData.findReflectionByName(compName);
     if (!classRef) {
@@ -232,7 +245,6 @@ function updatePhase(tree, pathname, aggData, errorMessages) {
     }
     return true;
 }
-exports.updatePhase = updatePhase;
 function initialCap(str) {
     return str[0].toUpperCase() + str.substr(1);
 }

tools/doc/tools/tsInfo.ts

@@ -40,6 +40,33 @@ let undocMethodNames = {
 };
 
 
+export function processDocs(mdCache, aggData, _errorMessages) {
+    initPhase(aggData);
+    
+    let pathnames = Object.keys(mdCache);
+    let internalErrors;
+
+    pathnames.forEach(pathname => {
+        internalErrors = [];
+        updateFile(mdCache[pathname].mdOutTree, pathname, aggData, internalErrors);
+
+        if (internalErrors.length > 0) {
+            showErrors(pathname, internalErrors);
+        }
+    });
+}
+
+
+function showErrors(filename, errorMessages) {
+    console.log(filename);
+
+    errorMessages.forEach(message => {
+        console.log("    " + message);
+    });
+
+    console.log("");
+}
+
 class PropInfo {
     name: string;
     type: string;
@@ -258,7 +285,7 @@ class ComponentInfo {
 }
 
 
-export function initPhase(aggData) {
+function initPhase(aggData) {
     nameExceptions = aggData.config.typeNameExceptions;
 
     let app = new Application({
@@ -276,15 +303,9 @@ export function initPhase(aggData) {
 }
 
 
-export function readPhase(tree, pathname, aggData) {
-}
 
 
-export function aggPhase(aggData) {
-}
-
-
-export function updatePhase(tree, pathname, aggData, errorMessages) {
+function updateFile(tree, pathname, aggData, errorMessages) {
     let compName = angNameToClassName(path.basename(pathname, ".md"));
     let classRef = aggData.projData.findReflectionByName(compName);
 

tools/doc/tools/tutorialIndex.js

@@ -11,10 +11,10 @@ var unist = require("../unistHelpers");
 var tutFolder = path.resolve("docs", "tutorials");
 var templateFolder = path.resolve("tools", "doc", "templates");
 var userGuideFolder = path.resolve("docs", "user-guide");
-function initPhase(aggData) { }
-exports.initPhase = initPhase;
-function readPhase(tree, pathname, aggData) { }
-exports.readPhase = readPhase;
+function processDocs(tree, pathname, aggData, errorMessages) {
+    aggPhase(aggData);
+}
+exports.processDocs = processDocs;
 function aggPhase(aggData) {
     var indexDocData = getIndexDocData();
     var templateName = path.resolve(templateFolder, "tutIndex.ejs");
@@ -33,11 +33,6 @@ function aggPhase(aggData) {
     });
     fs.writeFileSync(tutIndexFile, remark().use(frontMatter, { type: 'yaml', fence: '---' }).data("settings", { paddedTable: false, gfm: false }).stringify(tutIndexMD));
 }
-exports.aggPhase = aggPhase;
-function updatePhase(tree, pathname, aggData) {
-    return false;
-}
-exports.updatePhase = updatePhase;
 function getIndexDocData() {
     var indexFile = path.resolve(userGuideFolder, "summary.json");
     var summaryArray = JSON.parse(fs.readFileSync(indexFile, "utf8"));

tools/doc/tools/tutorialIndex.ts

@@ -15,12 +15,12 @@ const templateFolder = path.resolve("tools", "doc", "templates");
 const userGuideFolder = path.resolve("docs", "user-guide");
 
 
-export function initPhase(aggData) {}
-
-export function readPhase(tree, pathname, aggData) {}
+export function processDocs(tree, pathname, aggData, errorMessages) {
+    aggPhase(aggData);
+}
 
 
-export function aggPhase(aggData) {
+function aggPhase(aggData) {
     let indexDocData = getIndexDocData();
 
     let templateName = path.resolve(templateFolder, "tutIndex.ejs");
@@ -46,10 +46,6 @@ export function aggPhase(aggData) {
 }
 
 
-export function updatePhase(tree, pathname, aggData) {
-    return false;
-}
-
 
 function getIndexDocData() {
     let indexFile = path.resolve(userGuideFolder, "summary.json");

tools/doc/tools/typeLinker.js

@@ -3,6 +3,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 var path = require("path");
 var fs = require("fs");
 var typedoc_1 = require("typedoc");
+var ProgressBar = require("progress");
 var unist = require("../unistHelpers");
 var ngHelpers = require("../ngHelpers");
 var includedNodeTypes = [
@@ -13,6 +14,21 @@ var includedNodeTypes = [
 var docFolder = path.resolve("docs");
 var adfLibNames = ["core", "content-services", "insights", "process-services"];
 var externalNameLinks;
+function processDocs(mdCache, aggData, errorMessages) {
+    initPhase(aggData);
+    var pathnames = Object.keys(mdCache);
+    var progress = new ProgressBar("Processing: [:bar] (:current/:total)", {
+        total: pathnames.length,
+        width: 50,
+        clear: true
+    });
+    pathnames.forEach(function (pathname) {
+        updateFile(mdCache[pathname].mdOutTree, pathname, aggData, errorMessages);
+        progress.tick();
+        progress.render();
+    });
+}
+exports.processDocs = processDocs;
 function initPhase(aggData) {
     externalNameLinks = aggData.config.externalNameLinks;
     aggData.docFiles = {};
@@ -36,13 +52,7 @@ function initPhase(aggData) {
     });
     //console.log(JSON.stringify(aggData.nameLookup));
 }
-exports.initPhase = initPhase;
-function readPhase(tree, pathname, aggData) { }
-exports.readPhase = readPhase;
-function aggPhase(aggData) {
-}
-exports.aggPhase = aggPhase;
-function updatePhase(tree, pathname, aggData) {
+function updateFile(tree, pathname, aggData, errorMessages) {
     traverseMDTree(tree);
     return true;
     function traverseMDTree(node) {
@@ -86,7 +96,6 @@ function updatePhase(tree, pathname, aggData) {
         */
     }
 }
-exports.updatePhase = updatePhase;
 var SplitNameNode = /** @class */ (function () {
     function SplitNameNode(key, value) {
         if (key === void 0) { key = ""; }

tools/doc/tools/typeLinker.ts

@@ -18,6 +18,8 @@ import {
  } from "typedoc";
 import { CommentTag } from "typedoc/dist/lib/models";
 
+import * as ProgressBar from "progress";
+
 import * as unist from "../unistHelpers";
 import * as ngHelpers from "../ngHelpers";
 import { match } from "minimatch";
@@ -34,7 +36,26 @@ const adfLibNames = ["core", "content-services", "insights", "process-services"]
 
 let externalNameLinks;
 
-export function initPhase(aggData) {
+export function processDocs(mdCache, aggData, errorMessages) {
+    initPhase(aggData);
+
+    var pathnames = Object.keys(mdCache);
+
+    let progress = new ProgressBar("Processing: [:bar] (:current/:total)", {
+        total: pathnames.length,
+        width: 50,
+        clear: true
+    });
+
+    pathnames.forEach(pathname => {
+        updateFile(mdCache[pathname].mdOutTree, pathname, aggData, errorMessages);
+        progress.tick();
+        progress.render();
+    });
+}
+
+
+function initPhase(aggData) {
     externalNameLinks = aggData.config.externalNameLinks;
     aggData.docFiles = {};
     aggData.nameLookup = new SplitNameLookup();
@@ -64,15 +85,10 @@ export function initPhase(aggData) {
     //console.log(JSON.stringify(aggData.nameLookup));
 }
 
-export function readPhase(tree, pathname, aggData) {}
 
 
-export function aggPhase(aggData) {
 
-}
-
-
-export function updatePhase(tree, pathname, aggData) {
+function updateFile(tree, pathname, aggData, errorMessages) {
     traverseMDTree(tree);
     return true;
 

tools/doc/tools/versionIndex.js

@@ -14,10 +14,7 @@ var ngHelpers = require("../ngHelpers");
 
 
 module.exports = {
-    "initPhase": initPhase,
-    "readPhase": readPhase,
-    "aggPhase": aggPhase,
-    "updatePhase": updatePhase
+    "processDocs": processDocs
 }
 
 var angFilenameRegex = /([a-zA-Z0-9\-]+)\.((component)|(directive)|(model)|(pipe)|(service)|(widget))\.ts/;
@@ -30,12 +27,29 @@ var initialVersion = "v2.0.0";
 
 var templateFolder = path.resolve("tools", "doc", "templates");
 
+
+function processDocs(mdCache, aggData, errorMessages) {
+    initPhase(aggData);
+    readPhase(mdCache, aggData);
+    aggPhase(aggData);
+}
+
+
 function initPhase(aggData) {
     aggData.versions = { "v2.0.0":[] };
 }
 
 
-function readPhase(tree, pathname, aggData) {
+function readPhase(mdCache, aggData) {
+    var pathnames = Object.keys(mdCache);
+
+    pathnames.forEach(pathname => {
+        getFileData(mdCache[pathname].mdInTree, pathname, aggData);
+    });
+}
+
+
+function getFileData(tree, pathname, aggData) {
     var compName = pathname;
     var angNameRegex = /([a-zA-Z0-9\-]+)\.((component)|(directive)|(model)|(pipe)|(service)|(widget))/;
 
@@ -121,8 +135,3 @@ function aggPhase(aggData) {
 
     fs.writeFileSync(histFilePath, remark().use(frontMatter, {type: 'yaml', fence: '---'}).data("settings", {paddedTable: false, gfm: false}).stringify(histFileTree));
 }
-
-
-function updatePhase(tree, pathname, aggData) {
-    return false;
-}