[ADF-4152] Updated doc folder structure and index tool (#4403)

* [ADF-4152] Initial GraphQL implementation

* [ADF-4152] Schema updates

* [ADF-4152] Rounded out basic fields

* [ADF-4152] Added basic template functionality

* [ADF-4152] Added full template generation

* [ADF-4152] Moved proc services doc files to new folders

* [ADF-4152] Updated README.md with section from new template

* [ADF-4152] Fixed another problem with relative URLs

* [ADF-4152] Fixed links and some more bugs

* [ADF-4152] Removed proc services folder README file
This commit is contained in:
Andy Stark
2019-03-06 20:23:31 +00:00
committed by Eugenio Romano
parent 75b90c5e08
commit e74f545aa8
48 changed files with 1180 additions and 461 deletions

View File

@@ -0,0 +1,75 @@
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
var fs = require("fs");
var path = require("path");
var ejs = require("ejs");
var remark = require("remark");
var frontMatter = require("remark-frontmatter");
var replaceZone = require("mdast-zone");
var graphql_1 = require("graphql");
var MQ = require("../mqDefs");
var libNamesRegex = /content-services|core|extensions|insights|process-services|process-services-cloud/;
var libNamesList = ['process-services'];
var query = "\n query libIndex($libName: String) {\n documents(idFilter: $libName) {\n title: metadata(key: \"Title\")\n status: metadata(key: \"Status\")\n id\n classType: folder(depth: 2)\n heading {\n link {\n url\n }\n }\n paragraph {\n plaintext\n }\n }\n }\n";
function processDocs(mdCache, aggData, _errorMessages) {
var docset = new GQDocset(mdCache);
var templateFilePath = path.resolve(__dirname, '..', 'templates', 'gqIndex.ejs');
var templateSource = fs.readFileSync(templateFilePath, 'utf8');
var template = ejs.compile(templateSource);
var indexFilePath = path.resolve(aggData['rootFolder'], 'README.md');
var indexFileText = fs.readFileSync(indexFilePath, 'utf8');
var indexMD = remark()
.use(frontMatter, ["yaml"])
.parse(indexFileText);
var schema = graphql_1.buildSchema(MQ.schema);
libNamesList.forEach(function (libName) {
graphql_1.graphql(schema, query, docset, null, { 'libName': libName })
.then(function (response) {
if (!response['data']) {
console.log(JSON.stringify(response));
}
else {
//console.log(template(response['data']));
var newSection_1 = remark().parse(template(response['data'])).children;
replaceZone(indexMD, libName, function (start, _oldZone, end) {
newSection_1.unshift(start);
newSection_1.push(end);
return newSection_1;
});
var outText = remark()
.use(frontMatter, { type: 'yaml', fence: '---' })
.data("settings", { paddedTable: false, gfm: false })
.stringify(indexMD);
fs.writeFileSync(indexFilePath, outText);
}
});
});
}
exports.processDocs = processDocs;
var GQDocset = /** @class */ (function () {
function GQDocset(mdCache) {
var _this = this;
this.docs = [];
var pathnames = Object.keys(mdCache);
pathnames.forEach(function (pathname) {
if (!pathname.match(/README/) &&
pathname.match(libNamesRegex)) {
var doc = new MQ.Root(mdCache[pathname].mdInTree);
doc.id = pathname.replace(/\\/g, '/');
_this.docs.push(doc);
}
});
}
GQDocset.prototype.documents = function (args) {
if (args['idFilter'] === '') {
return this.docs;
}
else {
return this.docs.filter(function (doc) { return doc.id.indexOf(args['idFilter'] + '/') !== -1; });
}
};
GQDocset.prototype.size = function () {
return this.docs.length;
};
return GQDocset;
}());

113
tools/doc/tools/gqIndex.ts Normal file
View File

@@ -0,0 +1,113 @@
import * as fs from 'fs';
import * as path from 'path';
import * as ejs from 'ejs';
import * as remark from 'remark';
import * as frontMatter from 'remark-frontmatter';
import * as replaceZone from 'mdast-zone';
import { graphql, buildSchema } from 'graphql';
import * as MQ from '../mqDefs';
let libNamesRegex = /content-services|core|extensions|insights|process-services|process-services-cloud/;
let libNamesList = ['process-services'];
let query = `
query libIndex($libName: String) {
documents(idFilter: $libName) {
title: metadata(key: "Title")
status: metadata(key: "Status")
id
classType: folder(depth: 2)
heading {
link {
url
}
}
paragraph {
plaintext
}
}
}
`;
export function processDocs(mdCache, aggData, _errorMessages) {
let docset: GQDocset = new GQDocset(mdCache);
let templateFilePath = path.resolve(__dirname, '..', 'templates', 'gqIndex.ejs');
let templateSource = fs.readFileSync(templateFilePath, 'utf8');
let template = ejs.compile(templateSource);
let indexFilePath = path.resolve(aggData['rootFolder'], 'docs', 'README.md');
let indexFileText = fs.readFileSync(indexFilePath, 'utf8');
let indexMD = remark()
.use(frontMatter, ["yaml"])
.parse(indexFileText);
let schema = buildSchema(MQ.schema);
libNamesList.forEach(libName => {
graphql(schema, query, docset, null, {'libName': libName})
.then((response) => {
if (!response['data']) {
console.log(JSON.stringify(response));
} else {
//console.log(template(response['data']));
let newSection = remark().parse(template(response['data'])).children;
replaceZone(indexMD, libName, (start, _oldZone, end) => {
newSection.unshift(start);
newSection.push(end);
return newSection;
});
let outText = remark()
.use(frontMatter, {type: 'yaml', fence: '---'})
.data("settings", {paddedTable: false, gfm: false})
.stringify(indexMD);
fs.writeFileSync(indexFilePath, outText);
}
});
});
}
class GQDocset {
public docs: MQ.Root[];
constructor(mdCache) {
this.docs = [];
let pathnames = Object.keys(mdCache);
pathnames.forEach(pathname => {
if (!pathname.match(/README/) &&
pathname.match(libNamesRegex)
) {
let doc = new MQ.Root(mdCache[pathname].mdInTree);
doc.id = pathname.replace(/\\/g, '/');
this.docs.push(doc);
}
});
}
documents(args): MQ.Root[] {
if (args['idFilter'] === '') {
return this.docs;
} else {
return this.docs.filter(doc => doc.id.indexOf(args['idFilter'] + '/') !== -1);
}
}
size(): number {
return this.docs.length;
}
}

View File

@@ -0,0 +1,8 @@
import * as path from "path";
import * as fs from "fs";
import { select, selectAll } from "unist-util-select";
export function processDocs(mdCache, aggData, errorMessages) {
}

View File

@@ -1,5 +1,5 @@
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.__esModule = true;
var path = require("path");
var unist_util_select_1 = require("unist-util-select");
var ngHelpers = require("../ngHelpers");
@@ -16,20 +16,30 @@ function processDocs(mdCache, aggData, errorMessages) {
var classInfo = aggData.classInfo[className];
var sourcePath = classInfo ? classInfo.sourcePath : '';
var titleHeading = unist_util_select_1.select('heading[depth=1]:first-of-type', tree);
var relDocPath = pathname.substring(pathname.indexOf('docs'));
var srcUrl = fixRelSrcUrl(relDocPath, sourcePath);
if (titleHeading.children[0].type === "text") {
var titleText = titleHeading.children[0];
titleHeading.children[0] = {
type: 'link',
url: "../../" + sourcePath,
url: srcUrl,
title: "Defined in " + path.basename(sourcePath),
children: [titleText]
};
}
else if ((titleHeading.children[0].type === "link") && sourcePath) {
var linkElem = titleHeading.children[0];
linkElem.url = "../../" + sourcePath;
linkElem.title = "Defined in " + path.basename(sourcePath);
linkElem.url = srcUrl, //`../../${sourcePath}`;
linkElem.title = "Defined in " + path.basename(sourcePath);
}
});
}
exports.processDocs = processDocs;
function fixRelSrcUrl(docPath, srcPath) {
var docPathSegments = docPath.split(/[\\\/]/);
var dotPathPart = '';
for (var i = 0; i < (docPathSegments.length - 1); i++) {
dotPathPart += '../';
}
return dotPathPart + srcPath;
}

View File

@@ -23,20 +23,33 @@ export function processDocs(mdCache, aggData, errorMessages) {
let classInfo = aggData.classInfo[className];
let sourcePath = classInfo ? classInfo.sourcePath : '';
let titleHeading = select('heading[depth=1]:first-of-type', tree);
let relDocPath = pathname.substring(pathname.indexOf('docs'));
let srcUrl = fixRelSrcUrl(relDocPath, sourcePath);
if (titleHeading.children[0].type === "text") {
let titleText = titleHeading.children[0];
titleHeading.children[0] = {
type: 'link',
url: `../../${sourcePath}`,
url: srcUrl,//`../../${sourcePath}`,
title: `Defined in ${path.basename(sourcePath)}`,
children: [titleText]
}
} else if ((titleHeading.children[0].type === "link") && sourcePath) {
let linkElem = titleHeading.children[0];
linkElem.url = `../../${sourcePath}`;
linkElem.url = srcUrl, //`../../${sourcePath}`;
linkElem.title = `Defined in ${path.basename(sourcePath)}`;
}
});
}
function fixRelSrcUrl(docPath: string, srcPath: string) {
let docPathSegments = docPath.split(/[\\\/]/);
let dotPathPart = '';
for (let i = 0; i < (docPathSegments.length - 1); i++) {
dotPathPart += '../';
}
return dotPathPart + srcPath;
}

View File

@@ -1,22 +1,6 @@
"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.__esModule = true;
var path = require("path");
var fs = require("fs");
/*
import {
Application,
ProjectReflection,
Reflection,
DeclarationReflection,
SignatureReflection,
ParameterReflection,
ReflectionKind,
TraverseProperty,
Decorator
} from "typedoc";
import { CommentTag } from "typedoc/dist/lib/models";
*/
var ProgressBar = require("progress");
var unist = require("../unistHelpers");
var ngHelpers = require("../ngHelpers");
var includedNodeTypes = [
@@ -28,44 +12,38 @@ var docFolder = path.resolve("docs");
var adfLibNames = ["core", "content-services", "insights", "process-services", "process-services-cloud", "extensions"];
var externalNameLinks;
function processDocs(mdCache, aggData, errorMessages) {
initPhase(aggData);
initPhase(aggData, mdCache);
var pathnames = Object.keys(mdCache);
var progress = new ProgressBar("Processing: [:bar] (:current/:total)", {
total: pathnames.length,
width: 50,
clear: true
});
pathnames.forEach(function (pathname) {
updateFile(mdCache[pathname].mdOutTree, pathname, aggData, errorMessages);
progress.tick();
progress.render();
});
}
exports.processDocs = processDocs;
function initPhase(aggData) {
function initPhase(aggData, mdCache) {
externalNameLinks = aggData.config.externalNameLinks;
aggData.docFiles = {};
aggData.nameLookup = new SplitNameLookup();
adfLibNames.forEach(function (libName) {
var libFolderPath = path.resolve(docFolder, libName);
var files = fs.readdirSync(libFolderPath);
files.forEach(function (file) {
/*
adfLibNames.forEach(libName => {
let libFolderPath = path.resolve(docFolder, libName);
let files = fs.readdirSync(libFolderPath);
files.forEach(file => {
if (path.extname(file) === ".md") {
var relPath = libFolderPath.substr(libFolderPath.indexOf("docs") + 5).replace(/\\/, "/") + "/" + file;
var compName = path.basename(file, ".md");
let relPath = libFolderPath.substr(libFolderPath.indexOf("docs") + 5).replace(/\\/, "/") + "/" + file;
let compName = path.basename(file, ".md");
aggData.docFiles[compName] = relPath;
}
});
});
/*
let classes = aggData.projData.getReflectionsByKind(ReflectionKind.Class);
classes.forEach(currClass => {
if (currClass.name.match(/(Component|Directive|Interface|Model|Pipe|Service|Widget)$/)) {
aggData.nameLookup.addName(currClass.name);
}
});
*/
var docFilePaths = Object.keys(mdCache);
docFilePaths.forEach(function (docFilePath) {
var relPath = docFilePath.substring(docFilePath.indexOf('docs') + 5).replace(/\\/g, "/");
var compName = path.basename(relPath, ".md");
aggData.docFiles[compName] = relPath;
});
var classNames = Object.keys(aggData.classInfo);
classNames.forEach(function (currClassName) {
if (currClassName.match(/(Component|Directive|Interface|Model|Pipe|Service|Widget)$/)) {
@@ -81,34 +59,25 @@ function updateFile(tree, pathname, aggData, _errorMessages) {
if (!includedNodeTypes.includes(node.type)) {
return;
}
/*if (node.type === "inlineCode") {
console.log(`Link text: ${node.value}`);
let link = resolveTypeLink(aggData, node.value);
if (link) {
convertNodeToTypeLink(node, node.value, link);
}
} else */
if (node.type === "link") {
if (node.children && ((node.children[0].type === "inlineCode") ||
(node.children[0].type === "text"))) {
var link = resolveTypeLink(aggData, node.children[0].value);
var link = resolveTypeLink(aggData, pathname, node.children[0].value);
if (link) {
convertNodeToTypeLink(node, node.children[0].value, link);
}
}
}
else if ((node.children) && (node.type !== "heading")) { //((node.type === "paragraph") || (node.type === "tableCell")) {
else if ((node.children) && (node.type !== "heading")) {
node.children.forEach(function (child, index) {
var _a;
if ((child.type === "text") || (child.type === "inlineCode")) {
var newNodes = handleLinksInBodyText(aggData, child.value, child.type === 'inlineCode');
var newNodes = handleLinksInBodyText(aggData, pathname, child.value, child.type === 'inlineCode');
(_a = node.children).splice.apply(_a, [index, 1].concat(newNodes));
}
else {
traverseMDTree(child);
}
var _a;
});
} /*else if (node.children) {
node.children.forEach(child => {
@@ -250,7 +219,7 @@ var WordScanner = /** @class */ (function () {
};
return WordScanner;
}());
function handleLinksInBodyText(aggData, text, wrapInlineCode) {
function handleLinksInBodyText(aggData, docFilePath, text, wrapInlineCode) {
if (wrapInlineCode === void 0) { wrapInlineCode = false; }
var result = [];
var currTextStart = 0;
@@ -260,12 +229,12 @@ function handleLinksInBodyText(aggData, text, wrapInlineCode) {
.replace(/'s$/, "")
.replace(/^[;:,\."']+/g, "")
.replace(/[;:,\."']+$/g, "");
var link = resolveTypeLink(aggData, word);
var link = resolveTypeLink(aggData, docFilePath, word);
var matchStart = void 0;
if (!link) {
var match_1 = matcher.nextWord(word.toLowerCase(), scanner.index);
if (match_1 && match_1[0]) {
link = resolveTypeLink(aggData, match_1[0].value);
link = resolveTypeLink(aggData, docFilePath, match_1[0].value);
matchStart = match_1[0].startPos;
}
}
@@ -307,7 +276,7 @@ function handleLinksInBodyText(aggData, text, wrapInlineCode) {
}
return result;
}
function resolveTypeLink(aggData, text) {
function resolveTypeLink(aggData, docFilePath, text) {
var possTypeName = cleanTypeName(text);
if (possTypeName === 'constructor') {
return "";
@@ -320,10 +289,11 @@ function resolveTypeLink(aggData, text) {
if (classInfo) {
var kebabName = ngHelpers.kebabifyClassName(possTypeName);
var possDocFile = aggData.docFiles[kebabName];
//let url = "../../lib/" + ref.sources[0].fileName;
var url = "../../" + classInfo.sourcePath; //"../../lib/" + classInfo.items[0].source.path;
//let url = "../../" + classInfo.sourcePath;
var url = fixRelSrcUrl(docFilePath, classInfo.sourcePath);
if (possDocFile) {
url = "../" + possDocFile;
//url = "../" + possDocFile;
url = fixRelDocUrl(docFilePath, possDocFile);
}
return url;
}
@@ -334,6 +304,25 @@ function resolveTypeLink(aggData, text) {
return "";
}
}
function fixRelSrcUrl(docPath, srcPath) {
var relDocPath = docPath.substring(docPath.indexOf('docs'));
var docPathSegments = relDocPath.split(/[\\\/]/);
var dotPathPart = '';
for (var i = 0; i < (docPathSegments.length - 1); i++) {
dotPathPart += '../';
}
return dotPathPart + srcPath;
}
function fixRelDocUrl(docPathFrom, docPathTo) {
var relDocPathFrom = docPathFrom.substring(docPathFrom.indexOf('docs'));
var docPathSegments = relDocPathFrom.split(/[\\\/]/);
var dotPathPart = '';
console.log("Fixing: " + docPathFrom + " " + docPathTo);
for (var i = 0; i < (docPathSegments.length - 2); i++) {
dotPathPart += '../';
}
return dotPathPart + docPathTo;
}
function cleanTypeName(text) {
var matches = text.match(/[a-zA-Z0-9_]+<([a-zA-Z0-9_]+)(\[\])?>/);
if (matches) {

View File

@@ -1,28 +1,10 @@
import * as path from "path";
import * as fs from "fs";
import * as remark from "remark";
import * as stringify from "remark-stringify";
import * as frontMatter from "remark-frontmatter";
/*
import {
Application,
ProjectReflection,
Reflection,
DeclarationReflection,
SignatureReflection,
ParameterReflection,
ReflectionKind,
TraverseProperty,
Decorator
} from "typedoc";
import { CommentTag } from "typedoc/dist/lib/models";
*/
import * as ProgressBar from "progress";
import * as unist from "../unistHelpers";
import * as ngHelpers from "../ngHelpers";
import { match } from "minimatch";
@@ -40,29 +22,22 @@ const adfLibNames = ["core", "content-services", "insights", "process-services",
let externalNameLinks;
export function processDocs(mdCache, aggData, errorMessages) {
initPhase(aggData);
initPhase(aggData, mdCache);
var pathnames = Object.keys(mdCache);
let progress = new ProgressBar("Processing: [:bar] (:current/:total)", {
total: pathnames.length,
width: 50,
clear: true
});
pathnames.forEach(pathname => {
updateFile(mdCache[pathname].mdOutTree, pathname, aggData, errorMessages);
progress.tick();
progress.render();
});
}
function initPhase(aggData) {
function initPhase(aggData, mdCache) {
externalNameLinks = aggData.config.externalNameLinks;
aggData.docFiles = {};
aggData.nameLookup = new SplitNameLookup();
/*
adfLibNames.forEach(libName => {
let libFolderPath = path.resolve(docFolder, libName);
@@ -76,17 +51,16 @@ function initPhase(aggData) {
}
});
});
/*
let classes = aggData.projData.getReflectionsByKind(ReflectionKind.Class);
classes.forEach(currClass => {
if (currClass.name.match(/(Component|Directive|Interface|Model|Pipe|Service|Widget)$/)) {
aggData.nameLookup.addName(currClass.name);
}
});
*/
let docFilePaths = Object.keys(mdCache);
docFilePaths.forEach(docFilePath => {
let relPath = docFilePath.substring(docFilePath.indexOf('docs') + 5).replace(/\\/g, "/");
let compName = path.basename(relPath, ".md");
aggData.docFiles[compName] = relPath;
});
let classNames = Object.keys(aggData.classInfo);
classNames.forEach(currClassName => {
@@ -110,21 +84,12 @@ function updateFile(tree, pathname, aggData, _errorMessages) {
return;
}
/*if (node.type === "inlineCode") {
console.log(`Link text: ${node.value}`);
let link = resolveTypeLink(aggData, node.value);
if (link) {
convertNodeToTypeLink(node, node.value, link);
}
} else */
if (node.type === "link") {
if (node.children && (
(node.children[0].type === "inlineCode") ||
(node.children[0].type === "text")
)) {
let link = resolveTypeLink(aggData, node.children[0].value);
let link = resolveTypeLink(aggData, pathname, node.children[0].value);
if (link) {
convertNodeToTypeLink(node, node.children[0].value, link);
@@ -133,7 +98,7 @@ function updateFile(tree, pathname, aggData, _errorMessages) {
} else if ((node.children) && (node.type !== "heading")) { //((node.type === "paragraph") || (node.type === "tableCell")) {
node.children.forEach((child, index) => {
if ((child.type === "text") || (child.type === "inlineCode")) {
let newNodes = handleLinksInBodyText(aggData, child.value, child.type === 'inlineCode');
let newNodes = handleLinksInBodyText(aggData, pathname, child.value, child.type === 'inlineCode');
node.children.splice(index, 1, ...newNodes);
} else {
traverseMDTree(child);
@@ -302,7 +267,7 @@ class WordScanner {
}
function handleLinksInBodyText(aggData, text: string, wrapInlineCode: boolean = false): Node[] {
function handleLinksInBodyText(aggData, docFilePath: string, text: string, wrapInlineCode: boolean = false): Node[] {
let result = [];
let currTextStart = 0;
let matcher = new SplitNameMatcher(aggData.nameLookup.root);
@@ -313,14 +278,14 @@ function handleLinksInBodyText(aggData, text: string, wrapInlineCode: boolean =
.replace(/^[;:,\."']+/g, "")
.replace(/[;:,\."']+$/g, "");
let link = resolveTypeLink(aggData, word);
let link = resolveTypeLink(aggData, docFilePath, word);
let matchStart;
if (!link) {
let match = matcher.nextWord(word.toLowerCase(), scanner.index);
if (match && match[0]) {
link = resolveTypeLink(aggData, match[0].value);
link = resolveTypeLink(aggData, docFilePath, match[0].value);
matchStart = match[0].startPos;
}
} else {
@@ -368,7 +333,7 @@ function handleLinksInBodyText(aggData, text: string, wrapInlineCode: boolean =
}
function resolveTypeLink(aggData, text): string {
function resolveTypeLink(aggData, docFilePath, text): string {
let possTypeName = cleanTypeName(text);
if (possTypeName === 'constructor') {
@@ -384,12 +349,14 @@ function resolveTypeLink(aggData, text): string {
if (classInfo) {
let kebabName = ngHelpers.kebabifyClassName(possTypeName);
let possDocFile = aggData.docFiles[kebabName];
//let url = "../../lib/" + ref.sources[0].fileName;
let url = "../../" + classInfo.sourcePath; //"../../lib/" + classInfo.items[0].source.path;
//let url = "../../" + classInfo.sourcePath;
let url = fixRelSrcUrl(docFilePath, classInfo.sourcePath);
if (possDocFile) {
url = "../" + possDocFile;
//url = "../" + possDocFile;
url = fixRelDocUrl(docFilePath, possDocFile);
}
return url;
@@ -400,6 +367,31 @@ function resolveTypeLink(aggData, text): string {
}
}
function fixRelSrcUrl(docPath: string, srcPath: string) {
let relDocPath = docPath.substring(docPath.indexOf('docs'));
let docPathSegments = relDocPath.split(/[\\\/]/);
let dotPathPart = '';
for (let i = 0; i < (docPathSegments.length - 1); i++) {
dotPathPart += '../';
}
return dotPathPart + srcPath;
}
function fixRelDocUrl(docPathFrom: string, docPathTo: string) {
let relDocPathFrom = docPathFrom.substring(docPathFrom.indexOf('docs'));
let docPathSegments = relDocPathFrom.split(/[\\\/]/);
let dotPathPart = '';
console.log(`Fixing: ${docPathFrom} ${docPathTo}`);
for (let i = 0; i < (docPathSegments.length - 2); i++) {
dotPathPart += '../';
}
return dotPathPart + docPathTo;
}
function cleanTypeName(text) {
let matches = text.match(/[a-zA-Z0-9_]+<([a-zA-Z0-9_]+)(\[\])?>/);