diff --git a/lib/config/DocProcessor/ReviewCheckerGuide.md b/lib/config/DocProcessor/ReviewCheckerGuide.md new file mode 100644 index 0000000000..ba890ece76 --- /dev/null +++ b/lib/config/DocProcessor/ReviewCheckerGuide.md @@ -0,0 +1,27 @@ +# Review checker guide + +The review checker tool queries the Github repo to look for recent +commits to the component source files. The dates of these commits +are compared against against a review date stored in the Markdown doc +file for each component. The time and the number of commits since the +last review are then combined into a "score" that gives an indication +of how urgently the doc file needs a review. + +## Review date metadata + +The review date is kept in the YAML metadata section at the top of each +Markdown file. The key is "Last reviewed" and the date is in the form +YYYY-MM-DD. + +## Commit message stoplist + +The checker will ignore any commits that match regular expressions stored +in the `commitStoplist.json` file in the `DocProcessor` folder. You could +use this, for example, to filter out JIRA tasks that don't involve any +changes in functionality (and therefore don't need documenting). + +## Output format + +The script sends comma-separated text to the command line. You can copy/paste +this into a spreadsheet or redirect the output to a text file with a ".csv" +suffix. \ No newline at end of file diff --git a/lib/config/DocProcessor/commitStoplist.json b/lib/config/DocProcessor/commitStoplist.json new file mode 100644 index 0000000000..cd9793875a --- /dev/null +++ b/lib/config/DocProcessor/commitStoplist.json @@ -0,0 +1 @@ +["ADF-1769"] \ No newline at end of file diff --git a/lib/config/DocProcessor/libsearch.js b/lib/config/DocProcessor/libsearch.js new file mode 100644 index 0000000000..ba6c63a5db --- /dev/null +++ b/lib/config/DocProcessor/libsearch.js @@ -0,0 +1,31 @@ +var fs = require("fs"); +var path = require("path"); + +module.exports = searchLibraryRecursive; + +var angFilenameRegex = /([a-zA-Z0-9\-]+)\.((component)|(directive)|(model)|(pipe)|(service)|(widget))\.ts/; +var searchFolderOmitRegex = /(config)|(mock)|(i18n)|(assets)|(styles)/; + +// Search source folders for .ts files to discover all components, directives, etc. +function searchLibraryRecursive(srcData, folderPath) { + var items = fs.readdirSync(folderPath); + + for (var i = 0; i < items.length; i++) { + var itemPath = path.resolve(folderPath, items[i]); + var info = fs.statSync(itemPath); + + if (info.isFile() && (items[i].match(angFilenameRegex))) { + var nameNoSuffix = path.basename(items[i], '.ts'); + + var displayPath = itemPath.replace(/\\/g, '/'); + displayPath = displayPath.substr(displayPath.indexOf("lib") + 4); + + // Type == "component", "directive", etc. + var itemType = nameNoSuffix.split('.')[1]; + + srcData[nameNoSuffix] = { "path": displayPath, "type": itemType }; + } else if (info.isDirectory() && !items[i].match(searchFolderOmitRegex)) { + searchLibraryRecursive(srcData, itemPath); + } + } +} \ No newline at end of file diff --git a/lib/config/DocProcessor/reviewChecker.js b/lib/config/DocProcessor/reviewChecker.js new file mode 100644 index 0000000000..028ee3f15a --- /dev/null +++ b/lib/config/DocProcessor/reviewChecker.js @@ -0,0 +1,95 @@ +"use strict"; +exports.__esModule = true; +var path = require("path"); +var fs = require("fs"); +var process = require("process"); +var graphql_request_1 = require("graphql-request"); +var remark = require("remark"); +var frontMatter = require("remark-frontmatter"); +var yaml = require("js-yaml"); +var moment = require("moment"); +var Rx_1 = require("rxjs/Rx"); +var libsearch = require("./libsearch"); +var stoplist_1 = require("./stoplist"); +var adf20StartDate = "2017-11-20"; +var commitWeight = 0.1; +var scoreTimeBase = 60; +var rootFolder = "."; +var stoplistFilePath = path.resolve("config", "DocProcessor", "commitStoplist.json"); +var angFilePattern = /(component)|(directive)|(model)|(pipe)|(service)|(widget)/; +var srcData = {}; +var stoplist = new stoplist_1.Stoplist(stoplistFilePath); +var docsFolderPath = path.resolve("..", "docs"); +libsearch(srcData, path.resolve(rootFolder)); +/* +let keys = Object.keys(srcData); + +for (let i = 0; i < keys.length; i++) { + console.log(keys[i]); +} +*/ +var authToken = process.env.graphAuthToken; +var client = new graphql_request_1.GraphQLClient('https://api.github.com/graphql', { + headers: { + Authorization: 'Bearer ' + authToken + } +}); +var query = "query commitHistory($path: String) {\n repository(name: \"alfresco-ng2-components\", owner: \"alfresco\") {\n ref(qualifiedName: \"development\") {\n target {\n ... on Commit {\n history(first: 15, path: $path) {\n nodes {\n pushedDate\n message\n }\n }\n }\n }\n }\n }\n}"; +var docFiles = getDocFileNames(docsFolderPath); +var docNames = Rx_1.Observable.from(docFiles); +console.log("'Name','Review date','Commits since review','Score'"); +docNames.subscribe(function (x) { + var key = path.basename(x, ".md"); + if (!srcData[key]) + return; + var vars = { + "path": "lib/" + srcData[key].path + }; + client.request(query, vars).then(function (data) { + var nodes = data["repository"].ref.target.history.nodes; + var lastReviewDate = getDocReviewDate(key + ".md"); + var numUsefulCommits = extractCommitInfo(nodes, lastReviewDate, stoplist); + var dateString = lastReviewDate.format("YYYY-MM-DD"); + var score = priorityScore(lastReviewDate, numUsefulCommits).toPrecision(3); + console.log("'" + key + "','" + dateString + "','" + numUsefulCommits + "','" + score + "'"); + }); +}); +function priorityScore(reviewDate, numCommits) { + var daysSinceReview = moment().diff(reviewDate, 'days'); + var commitScore = 2 + numCommits * commitWeight; + return Math.pow(commitScore, daysSinceReview / scoreTimeBase); +} +function getDocReviewDate(docFileName) { + var mdFilePath = path.resolve(docsFolderPath, docFileName); + var mdText = fs.readFileSync(mdFilePath); + var tree = remark().use(frontMatter, ["yaml"]).parse(mdText); + var lastReviewDate = moment(adf20StartDate); + if (tree.children[0].type == "yaml") { + var metadata = yaml.load(tree.children[0].value); + if (metadata["Last reviewed"]) + lastReviewDate = moment(metadata["Last reviewed"]); + } + return lastReviewDate; +} +function extractCommitInfo(commitNodes, cutOffDate, stoplist) { + var numUsefulCommits = 0; + commitNodes.forEach(function (element) { + if (!stoplist.isRejected(element.message)) { + var abbr = element.message.substr(0, 15); + var commitDate = moment(element.pushedDate); + if (commitDate.isAfter(cutOffDate)) { + numUsefulCommits++; + } + } + }); + return numUsefulCommits; +} +function getDocFileNames(folderPath) { + var files = fs.readdirSync(folderPath); + files = files.filter(function (filename) { + return (path.extname(filename) === ".md") && + (filename !== "README.md") && + (filename.match(angFilePattern)); + }); + return files; +} diff --git a/lib/config/DocProcessor/reviewChecker.ts b/lib/config/DocProcessor/reviewChecker.ts new file mode 100644 index 0000000000..d15db187d2 --- /dev/null +++ b/lib/config/DocProcessor/reviewChecker.ts @@ -0,0 +1,152 @@ +import * as path from "path"; +import * as fs from "fs"; +import * as process from "process" + +import { GraphQLClient } from "graphql-request"; +import * as remark from "remark"; +import * as frontMatter from "remark-frontmatter"; +import * as yaml from "js-yaml"; +import * as moment from "moment"; +import { Observable } from 'rxjs/Rx'; + +import * as libsearch from "./libsearch"; +import { Stoplist } from "./stoplist"; +import { last } from "rxjs/operator/last"; + + +const adf20StartDate = "2017-11-20"; + +const commitWeight = 0.1; +const scoreTimeBase = 60; + +const rootFolder = "."; +const stoplistFilePath = path.resolve("config", "DocProcessor", "commitStoplist.json"); + +const angFilePattern = /(component)|(directive)|(model)|(pipe)|(service)|(widget)/; + +let srcData = {}; +let stoplist = new Stoplist(stoplistFilePath); + +let docsFolderPath = path.resolve("..", "docs"); + +libsearch(srcData, path.resolve(rootFolder)); + +/* +let keys = Object.keys(srcData); + +for (let i = 0; i < keys.length; i++) { + console.log(keys[i]); +} +*/ + +const authToken = process.env.graphAuthToken; + +const client = new GraphQLClient('https://api.github.com/graphql', { + headers: { + Authorization: 'Bearer ' + authToken + } +}); + +const query = `query commitHistory($path: String) { + repository(name: "alfresco-ng2-components", owner: "alfresco") { + ref(qualifiedName: "development") { + target { + ... on Commit { + history(first: 15, path: $path) { + nodes { + pushedDate + message + } + } + } + } + } + } +}`; + +let docFiles = getDocFileNames(docsFolderPath); + +let docNames = Observable.from(docFiles); + +console.log("'Name','Review date','Commits since review','Score'"); + +docNames.subscribe(x => { + let key = path.basename(x, ".md"); + + if (!srcData[key]) + return; + + let vars = { + "path": "lib/" + srcData[key].path + }; + + client.request(query, vars).then(data => { + let nodes = data["repository"].ref.target.history.nodes; + + let lastReviewDate = getDocReviewDate(key + ".md"); + + let numUsefulCommits = extractCommitInfo(nodes, lastReviewDate, stoplist); + let dateString = lastReviewDate.format("YYYY-MM-DD"); + let score = priorityScore(lastReviewDate, numUsefulCommits).toPrecision(3); + + console.log(`'${key}','${dateString}','${numUsefulCommits}','${score}'`); + }); +}); + + +function priorityScore(reviewDate, numCommits) { + let daysSinceReview = moment().diff(reviewDate, 'days'); + let commitScore = 2 + numCommits * commitWeight; + return Math.pow(commitScore, daysSinceReview / scoreTimeBase); +} + + +function getDocReviewDate(docFileName) { + let mdFilePath = path.resolve(docsFolderPath, docFileName); + + let mdText = fs.readFileSync(mdFilePath); + let tree = remark().use(frontMatter, ["yaml"]).parse(mdText); + + let lastReviewDate = moment(adf20StartDate); + + if (tree.children[0].type == "yaml") { + let metadata = yaml.load(tree.children[0].value); + + if (metadata["Last reviewed"]) + lastReviewDate = moment(metadata["Last reviewed"]); + } + + return lastReviewDate; +} + + +function extractCommitInfo(commitNodes, cutOffDate, stoplist) { + let numUsefulCommits = 0; + + commitNodes.forEach(element => { + if (!stoplist.isRejected(element.message)) { + let abbr = element.message.substr(0, 15); + + let commitDate = moment(element.pushedDate); + + if (commitDate.isAfter(cutOffDate)) { + numUsefulCommits++; + } + } + }); + + return numUsefulCommits; +} + + +function getDocFileNames(folderPath) { + let files = fs.readdirSync(folderPath); + + files = files.filter(filename => + (path.extname(filename) === ".md") && + (filename !== "README.md") && + (filename.match(angFilePattern)) + ); + + return files; +} \ No newline at end of file diff --git a/lib/config/DocProcessor/stoplist.js b/lib/config/DocProcessor/stoplist.js new file mode 100644 index 0000000000..188a4e1e1d --- /dev/null +++ b/lib/config/DocProcessor/stoplist.js @@ -0,0 +1,29 @@ +"use strict"; +exports.__esModule = true; +var fs = require("fs"); +/* "Stoplist" of regular expressions to match against strings. */ +var Stoplist = /** @class */ (function () { + function Stoplist(slFilePath) { + var listExpressions = JSON.parse(fs.readFileSync(slFilePath, 'utf8')); + this.regexes = []; + if (listExpressions) { + for (var i = 0; i < listExpressions.length; i++) { + this.regexes.push(new RegExp(listExpressions[i])); + } + } + else { + this.regexes = []; + } + } + // Check if an item is covered by the stoplist and reject it if so. + Stoplist.prototype.isRejected = function (itemName) { + for (var i = 0; i < this.regexes.length; i++) { + if (this.regexes[i].test(itemName)) { + return true; + } + } + return false; + }; + return Stoplist; +}()); +exports.Stoplist = Stoplist; diff --git a/lib/config/DocProcessor/stoplist.ts b/lib/config/DocProcessor/stoplist.ts new file mode 100644 index 0000000000..84798f315e --- /dev/null +++ b/lib/config/DocProcessor/stoplist.ts @@ -0,0 +1,31 @@ +import * as fs from "fs"; + +/* "Stoplist" of regular expressions to match against strings. */ +export class Stoplist { + regexes: RegExp[]; + + constructor(slFilePath: string) { + let listExpressions = JSON.parse(fs.readFileSync(slFilePath, 'utf8')); + this.regexes = []; + + if (listExpressions) { + for (var i = 0; i < listExpressions.length; i++) { + this.regexes.push(new RegExp(listExpressions[i])); + } + } else { + this.regexes = []; + } + } + + // Check if an item is covered by the stoplist and reject it if so. + isRejected(itemName: string) { + for (var i = 0; i < this.regexes.length; i++) { + if (this.regexes[i].test(itemName)) { + return true; + } + } + + return false; + } + +} \ No newline at end of file diff --git a/lib/config/DocProcessor/tools/index.js b/lib/config/DocProcessor/tools/index.js index dcb011823c..1b010edcde 100644 --- a/lib/config/DocProcessor/tools/index.js +++ b/lib/config/DocProcessor/tools/index.js @@ -8,7 +8,7 @@ var yaml = require("js-yaml"); var unist = require("../unistHelpers"); var ngHelpers = require("../ngHelpers"); - +var searchLibraryRecursive = require("../libsearch"); module.exports = { "initPhase": initPhase, @@ -147,31 +147,6 @@ function rejectItemViaStoplist(stoplist, itemName) { } -// Search source folders for .ts files to discover all components, directives, etc. -function searchLibraryRecursive(srcData, folderPath) { - var items = fs.readdirSync(folderPath); - - for (var i = 0; i < items.length; i++) { - var itemPath = path.resolve(folderPath, items[i]); - var info = fs.statSync(itemPath); - - if (info.isFile() && (items[i].match(angFilenameRegex))) { - var nameNoSuffix = path.basename(items[i], '.ts'); - - var displayPath = itemPath.replace(/\\/g, '/'); - displayPath = displayPath.substr(displayPath.indexOf("lib") + 4); - - // Type == "component", "directive", etc. - var itemType = nameNoSuffix.split('.')[1]; - - srcData[nameNoSuffix] = { "path": displayPath, "type": itemType }; - } else if (info.isDirectory() && !items[i].match(searchFolderOmitRegex)) { - searchLibraryRecursive(srcData, itemPath); - } - } -} - - function prepareIndexSections(aggData) { var srcNames = Object.keys(aggData.srcData); var sections = initEmptySections();