[ADF-2447] Added review checker tool (#3039)

lib/config/DocProcessor/ReviewCheckerGuide.md

@@ -0,0 +1,27 @@
+# Review checker guide
+
+The review checker tool queries the Github repo to look for recent
+commits to the component source files. The dates of these commits
+are compared against against a review date stored in the Markdown doc
+file for each component. The time and the number of commits since the
+last review are then combined into a "score" that gives an indication
+of how urgently the doc file needs a review.
+
+## Review date metadata
+
+The review date is kept in the YAML metadata section at the top of each
+Markdown file. The key is "Last reviewed" and the date is in the form
+YYYY-MM-DD.
+
+## Commit message stoplist
+
+The checker will ignore any commits that match regular expressions stored
+in the `commitStoplist.json` file in the `DocProcessor` folder. You could
+use this, for example, to filter out JIRA tasks that don't involve any
+changes in functionality (and therefore don't need documenting).
+
+## Output format
+
+The script sends comma-separated text to the command line. You can copy/paste
+this into a spreadsheet or redirect the output to a text file with a ".csv"
+suffix.

lib/config/DocProcessor/commitStoplist.json

@@ -0,0 +1 @@
+["ADF-1769"]

lib/config/DocProcessor/libsearch.js

@@ -0,0 +1,31 @@
+var fs = require("fs");
+var path = require("path");
+
+module.exports = searchLibraryRecursive;
+
+var angFilenameRegex = /([a-zA-Z0-9\-]+)\.((component)|(directive)|(model)|(pipe)|(service)|(widget))\.ts/;
+var searchFolderOmitRegex = /(config)|(mock)|(i18n)|(assets)|(styles)/;
+
+//  Search source folders for .ts files to discover all components, directives, etc.
+function searchLibraryRecursive(srcData, folderPath) {
+    var items = fs.readdirSync(folderPath);
+    
+    for (var i = 0; i < items.length; i++) {
+        var itemPath = path.resolve(folderPath, items[i]);
+        var info = fs.statSync(itemPath);
+        
+        if (info.isFile() && (items[i].match(angFilenameRegex))) {
+            var nameNoSuffix = path.basename(items[i], '.ts');
+            
+            var displayPath = itemPath.replace(/\\/g, '/');
+            displayPath = displayPath.substr(displayPath.indexOf("lib") + 4);
+
+            // Type == "component", "directive", etc.
+            var itemType = nameNoSuffix.split('.')[1];
+            
+            srcData[nameNoSuffix] = { "path": displayPath, "type": itemType };            
+        } else if (info.isDirectory() && !items[i].match(searchFolderOmitRegex)) {
+            searchLibraryRecursive(srcData, itemPath);
+        }
+    }
+}

lib/config/DocProcessor/reviewChecker.js

@@ -0,0 +1,95 @@
+"use strict";
+exports.__esModule = true;
+var path = require("path");
+var fs = require("fs");
+var process = require("process");
+var graphql_request_1 = require("graphql-request");
+var remark = require("remark");
+var frontMatter = require("remark-frontmatter");
+var yaml = require("js-yaml");
+var moment = require("moment");
+var Rx_1 = require("rxjs/Rx");
+var libsearch = require("./libsearch");
+var stoplist_1 = require("./stoplist");
+var adf20StartDate = "2017-11-20";
+var commitWeight = 0.1;
+var scoreTimeBase = 60;
+var rootFolder = ".";
+var stoplistFilePath = path.resolve("config", "DocProcessor", "commitStoplist.json");
+var angFilePattern = /(component)|(directive)|(model)|(pipe)|(service)|(widget)/;
+var srcData = {};
+var stoplist = new stoplist_1.Stoplist(stoplistFilePath);
+var docsFolderPath = path.resolve("..", "docs");
+libsearch(srcData, path.resolve(rootFolder));
+/*
+let keys = Object.keys(srcData);
+
+for (let i = 0; i < keys.length; i++) {
+  console.log(keys[i]);
+}
+*/
+var authToken = process.env.graphAuthToken;
+var client = new graphql_request_1.GraphQLClient('https://api.github.com/graphql', {
+    headers: {
+        Authorization: 'Bearer ' + authToken
+    }
+});
+var query = "query commitHistory($path: String) {\n  repository(name: \"alfresco-ng2-components\", owner: \"alfresco\") {\n    ref(qualifiedName: \"development\") {\n      target {\n        ... on Commit {\n          history(first: 15, path: $path) {\n            nodes {\n              pushedDate\n              message\n            }\n          }\n        }\n      }\n    }\n  }\n}";
+var docFiles = getDocFileNames(docsFolderPath);
+var docNames = Rx_1.Observable.from(docFiles);
+console.log("'Name','Review date','Commits since review','Score'");
+docNames.subscribe(function (x) {
+    var key = path.basename(x, ".md");
+    if (!srcData[key])
+        return;
+    var vars = {
+        "path": "lib/" + srcData[key].path
+    };
+    client.request(query, vars).then(function (data) {
+        var nodes = data["repository"].ref.target.history.nodes;
+        var lastReviewDate = getDocReviewDate(key + ".md");
+        var numUsefulCommits = extractCommitInfo(nodes, lastReviewDate, stoplist);
+        var dateString = lastReviewDate.format("YYYY-MM-DD");
+        var score = priorityScore(lastReviewDate, numUsefulCommits).toPrecision(3);
+        console.log("'" + key + "','" + dateString + "','" + numUsefulCommits + "','" + score + "'");
+    });
+});
+function priorityScore(reviewDate, numCommits) {
+    var daysSinceReview = moment().diff(reviewDate, 'days');
+    var commitScore = 2 + numCommits * commitWeight;
+    return Math.pow(commitScore, daysSinceReview / scoreTimeBase);
+}
+function getDocReviewDate(docFileName) {
+    var mdFilePath = path.resolve(docsFolderPath, docFileName);
+    var mdText = fs.readFileSync(mdFilePath);
+    var tree = remark().use(frontMatter, ["yaml"]).parse(mdText);
+    var lastReviewDate = moment(adf20StartDate);
+    if (tree.children[0].type == "yaml") {
+        var metadata = yaml.load(tree.children[0].value);
+        if (metadata["Last reviewed"])
+            lastReviewDate = moment(metadata["Last reviewed"]);
+    }
+    return lastReviewDate;
+}
+function extractCommitInfo(commitNodes, cutOffDate, stoplist) {
+    var numUsefulCommits = 0;
+    commitNodes.forEach(function (element) {
+        if (!stoplist.isRejected(element.message)) {
+            var abbr = element.message.substr(0, 15);
+            var commitDate = moment(element.pushedDate);
+            if (commitDate.isAfter(cutOffDate)) {
+                numUsefulCommits++;
+            }
+        }
+    });
+    return numUsefulCommits;
+}
+function getDocFileNames(folderPath) {
+    var files = fs.readdirSync(folderPath);
+    files = files.filter(function (filename) {
+        return (path.extname(filename) === ".md") &&
+            (filename !== "README.md") &&
+            (filename.match(angFilePattern));
+    });
+    return files;
+}

lib/config/DocProcessor/reviewChecker.ts

@@ -0,0 +1,152 @@
+import * as path from "path";
+import * as fs from "fs";
+import * as process from "process"
+
+import { GraphQLClient } from "graphql-request";
+import * as remark from "remark";
+import * as frontMatter from "remark-frontmatter";
+import * as yaml from "js-yaml";
+import * as moment from "moment";
+import { Observable } from 'rxjs/Rx';
+
+import * as libsearch from "./libsearch";
+import { Stoplist } from "./stoplist";
+import { last } from "rxjs/operator/last";
+
+
+const adf20StartDate = "2017-11-20";
+
+const commitWeight = 0.1;
+const scoreTimeBase = 60;
+
+const rootFolder = ".";
+const stoplistFilePath = path.resolve("config", "DocProcessor", "commitStoplist.json");
+
+const angFilePattern = /(component)|(directive)|(model)|(pipe)|(service)|(widget)/;
+
+let srcData = {};
+let stoplist = new Stoplist(stoplistFilePath);
+
+let docsFolderPath = path.resolve("..", "docs");
+
+libsearch(srcData, path.resolve(rootFolder));
+
+/*
+let keys = Object.keys(srcData);
+
+for (let i = 0; i < keys.length; i++) {
+  console.log(keys[i]);
+}
+*/
+
+const authToken = process.env.graphAuthToken;
+
+const client = new GraphQLClient('https://api.github.com/graphql', {
+  headers: {
+    Authorization: 'Bearer ' + authToken
+  }
+});
+
+const query = `query commitHistory($path: String) {
+  repository(name: "alfresco-ng2-components", owner: "alfresco") {
+    ref(qualifiedName: "development") {
+      target {
+        ... on Commit {
+          history(first: 15, path: $path) {
+            nodes {
+              pushedDate
+              message
+            }
+          }
+        }
+      }
+    }
+  }
+}`;
+
+let docFiles = getDocFileNames(docsFolderPath);
+
+let docNames = Observable.from(docFiles);
+
+console.log("'Name','Review date','Commits since review','Score'");
+
+docNames.subscribe(x => {
+  let key = path.basename(x, ".md");
+
+  if (!srcData[key])
+    return;
+
+  let vars = {
+    "path": "lib/" + srcData[key].path
+  };
+
+  client.request(query, vars).then(data => {
+      let nodes = data["repository"].ref.target.history.nodes;
+
+      let lastReviewDate = getDocReviewDate(key + ".md");
+
+      let numUsefulCommits = extractCommitInfo(nodes, lastReviewDate, stoplist);
+      let dateString = lastReviewDate.format("YYYY-MM-DD");
+      let score = priorityScore(lastReviewDate, numUsefulCommits).toPrecision(3);
+
+      console.log(`'${key}','${dateString}','${numUsefulCommits}','${score}'`);
+  });
+});
+
+
+function priorityScore(reviewDate, numCommits) {
+  let daysSinceReview = moment().diff(reviewDate, 'days');
+  let commitScore = 2 + numCommits * commitWeight;
+  return Math.pow(commitScore, daysSinceReview / scoreTimeBase);
+}
+
+
+function getDocReviewDate(docFileName) {
+  let mdFilePath = path.resolve(docsFolderPath, docFileName);
+
+  let mdText = fs.readFileSync(mdFilePath);
+  let tree = remark().use(frontMatter, ["yaml"]).parse(mdText);
+
+  let lastReviewDate = moment(adf20StartDate);
+
+  if (tree.children[0].type == "yaml") {
+    let metadata = yaml.load(tree.children[0].value);
+
+    if (metadata["Last reviewed"])
+      lastReviewDate = moment(metadata["Last reviewed"]);
+  }
+
+  return lastReviewDate;
+}
+
+
+function extractCommitInfo(commitNodes, cutOffDate, stoplist) {
+  let numUsefulCommits = 0;
+
+  commitNodes.forEach(element => {
+    if (!stoplist.isRejected(element.message)) {
+      let abbr = element.message.substr(0, 15);
+
+      let commitDate = moment(element.pushedDate);
+
+      if (commitDate.isAfter(cutOffDate)) {
+        numUsefulCommits++;
+      }
+    }
+  });
+
+  return numUsefulCommits;
+}
+
+
+function getDocFileNames(folderPath) {
+  let files = fs.readdirSync(folderPath);
+
+  files = files.filter(filename => 
+    (path.extname(filename) === ".md") &&
+    (filename !== "README.md") &&
+    (filename.match(angFilePattern))
+  );
+
+  return files;
+}

lib/config/DocProcessor/stoplist.js

@@ -0,0 +1,29 @@
+"use strict";
+exports.__esModule = true;
+var fs = require("fs");
+/* "Stoplist" of regular expressions to match against strings. */
+var Stoplist = /** @class */ (function () {
+    function Stoplist(slFilePath) {
+        var listExpressions = JSON.parse(fs.readFileSync(slFilePath, 'utf8'));
+        this.regexes = [];
+        if (listExpressions) {
+            for (var i = 0; i < listExpressions.length; i++) {
+                this.regexes.push(new RegExp(listExpressions[i]));
+            }
+        }
+        else {
+            this.regexes = [];
+        }
+    }
+    // Check if an item is covered by the stoplist and reject it if so.
+    Stoplist.prototype.isRejected = function (itemName) {
+        for (var i = 0; i < this.regexes.length; i++) {
+            if (this.regexes[i].test(itemName)) {
+                return true;
+            }
+        }
+        return false;
+    };
+    return Stoplist;
+}());
+exports.Stoplist = Stoplist;

lib/config/DocProcessor/stoplist.ts

@@ -0,0 +1,31 @@
+import * as fs from "fs";
+
+/* "Stoplist" of regular expressions to match against strings. */
+export class Stoplist {
+    regexes: RegExp[];
+
+    constructor(slFilePath: string) {
+        let listExpressions = JSON.parse(fs.readFileSync(slFilePath, 'utf8'));
+        this.regexes = [];
+        
+        if (listExpressions) {
+            for (var i = 0; i < listExpressions.length; i++) {
+                this.regexes.push(new RegExp(listExpressions[i]));
+            }
+        } else {
+            this.regexes = [];
+        }
+    }
+
+    // Check if an item is covered by the stoplist and reject it if so.
+    isRejected(itemName: string) {
+        for (var i = 0; i < this.regexes.length; i++) {
+            if (this.regexes[i].test(itemName)) {
+                return true;
+            }
+        }
+        
+        return false;
+    }
+
+}

lib/config/DocProcessor/tools/index.js

@@ -8,7 +8,7 @@ var yaml = require("js-yaml");
 
 var unist = require("../unistHelpers");
 var ngHelpers = require("../ngHelpers");
-
+var searchLibraryRecursive = require("../libsearch");
 
 module.exports = {
     "initPhase": initPhase,
@@ -147,31 +147,6 @@ function rejectItemViaStoplist(stoplist, itemName) {
 }
 
 
-//  Search source folders for .ts files to discover all components, directives, etc.
-function searchLibraryRecursive(srcData, folderPath) {
-    var items = fs.readdirSync(folderPath);
-    
-    for (var i = 0; i < items.length; i++) {
-        var itemPath = path.resolve(folderPath, items[i]);
-        var info = fs.statSync(itemPath);
-        
-        if (info.isFile() && (items[i].match(angFilenameRegex))) {
-            var nameNoSuffix = path.basename(items[i], '.ts');
-            
-            var displayPath = itemPath.replace(/\\/g, '/');
-            displayPath = displayPath.substr(displayPath.indexOf("lib") + 4);
-
-            // Type == "component", "directive", etc.
-            var itemType = nameNoSuffix.split('.')[1];
-            
-            srcData[nameNoSuffix] = { "path": displayPath, "type": itemType };            
-        } else if (info.isDirectory() && !items[i].match(searchFolderOmitRegex)) {
-            searchLibraryRecursive(srcData, itemPath);
-        }
-    }
-}
-
-
 function prepareIndexSections(aggData) {
     var srcNames = Object.keys(aggData.srcData);
     var sections = initEmptySections();