update license tool

.rat-excludes

+node_modules
+\.git
+\.github
+\.editorconfig
+\.gitignore
+\.jshintrc
+\.jshintrc-dist
+\.npmignore
+\.rat-excludes
+\.DS_Store
+__MAC_OS
+README.md
+DISCLAIMER
+NOTICE
+KEYS
+LICENSE
+licenses
+benchmark/dep
+map/js
+map/json
+test/ut/lib
+test/data$
+test/lib/esl\.js$
+test/lib/perlin\.js$
+jquery
+configure
+IAxisPointer$
+ICoordinateSystem$
+.+\.json$
+.+\.map$
+.+\.gexf$
+.+\.jar$
+.+\.bin$
+.+\.csv$
+.+\.png$
+.+\.PNG$
+.+\.jpg$
+.+\.JPG$
+.+\.jpeg$
+.+\.JPEG$
+.+\.gif$
+.+\.GIF$

build/addHeader.js

@@ -17,77 +17,159 @@
 * under the License.
 */
 
-const glob = require('glob');
 const fs = require('fs');
-const headerStr = require('./preamble');
-
-const lists = [
-    '../src/**/*.js',
-    '../build/*.js',
-    '../benchmark/src/*.js',
-    '../benchmark/src/gulpfile.js',
-    '../extension-src/**/*.js',
-    '../extension/**/*.js',
-    '../map/js/**/*.js',
-    '../test/build/**/*.js',
-    '../test/node/**/*.js',
-    '../test/ut/core/*.js',
-    '../test/ut/spe/*.js',
-    '../test/ut/ut.js',
-    '../test/*.js',
-    '../theme/*.js',
-    '../theme/tool/**/*.js',
-    '../echarts.all.js',
-    '../echarts.blank.js',
-    '../echarts.common.js',
-    '../echarts.simple.js',
-    '../index.js',
-    '../index.common.js',
-    '../index.simple.js'
-];
-
-function extractLicense(str) {
-    str = str.trim();
-    const regex = new RegExp('/\\*[\\S\\s]*?\\*/', 'm');
-    const res = regex.exec(str);
-    const commentText = res && res[0];
-    if (commentText) {
-        if(commentText.toLowerCase().includes('apache license') || commentText.toLowerCase().includes('apache commons')) {
-            return 'Apache';
-        }
-        else if(commentText.toUpperCase().includes('BSD')) {
-            return 'BSD';
-        }
-        else if(commentText.toUpperCase().includes('LGPL')) {
-            return 'LGPL';
-        }
-        else if(commentText.toUpperCase().includes('GPL')) {
-            return 'GPL';
-        }
-        else if(commentText.toLowerCase().includes('mozilla public')) {
-            return 'Mozilla';
-        }
-        else if(commentText.toLowerCase().includes('mit license')) {
-            return 'MIT';
+const preamble = require('./preamble');
+const pathTool = require('path');
+const {color} = require('zrender/build/helper');
+// For consistency, we use `rat-excludes` for both Apache Rat and this tool.
+// Each item is a string of RegExp
+const excludesPath = pathTool.join(__dirname, '../.rat-excludes');
+const ecBasePath = pathTool.join(__dirname, '../');
+
+// const lists = [
+//     '../src/**/*.js',
+//     '../build/*.js',
+//     '../benchmark/src/*.js',
+//     '../benchmark/src/gulpfile.js',
+//     '../extension-src/**/*.js',
+//     '../extension/**/*.js',
+//     '../map/js/**/*.js',
+//     '../test/build/**/*.js',
+//     '../test/node/**/*.js',
+//     '../test/ut/core/*.js',
+//     '../test/ut/spe/*.js',
+//     '../test/ut/ut.js',
+//     '../test/*.js',
+//     '../theme/*.js',
+//     '../theme/tool/**/*.js',
+//     '../echarts.all.js',
+//     '../echarts.blank.js',
+//     '../echarts.common.js',
+//     '../echarts.simple.js',
+//     '../index.js',
+//     '../index.common.js',
+//     '../index.simple.js'
+// ];
+
+function run() {
+    const updatedFiles = [];
+    const passFiles = [];
+    const pendingFiles = [];
+
+    eachFile(function (absolutePath, fileExt) {
+        const fileStr = fs.readFileSync(absolutePath, 'utf-8');
+
+        const existLicense = preamble.extractLicense(fileStr, fileExt);
+        // !existLicense && console.log(absolutePath);
+
+        if (existLicense) {
+            passFiles.push(absolutePath);
+            return;
         }
+
+        // Conside binary files, only add for files with known ext.
+        if (!preamble.hasPreamble(fileExt)) {
+            pendingFiles.push(absolutePath);
+            return;
         }
-}
 
-lists.forEach(function (pattern) {
-    glob(pattern, function (err, fileList) {
-        if (err) {
-            throw new Error();
+        fs.writeFileSync(absolutePath, preamble.addPreamble(fileStr, fileExt), 'utf-8');
+        updatedFiles.push(absolutePath);
+    });
+
+    if (passFiles.length) {
+        console.log('\n\n');
+        console.log('----------------------------');
+        console.log(' Files that exists license: ');
+        console.log('----------------------------');
+        passFiles.forEach(function (path) {
+            console.log(color('fgGreen', 'dim')(path));
+        });
     }
-        fileList.forEach(function (fileUrl) {
-            const str = fs.readFileSync(fileUrl, 'utf-8');
 
-            const existLicense = extractLicense(str);
-            if (existLicense) {
-                console.log('File ' + fileUrl + ' already have license ' + existLicense);
+    if (updatedFiles.length) {
+        console.log('\n\n');
+        console.log('--------------------');
+        console.log(' License added for: ');
+        console.log('--------------------');
+        updatedFiles.forEach(function (path) {
+            console.log(color('fgGreen', 'bright')(path));
+        });
+    }
+
+    if (pendingFiles.length) {
+        console.log('\n\n');
+        console.log('----------------');
+        console.log(' Pending files: ');
+        console.log('----------------');
+        pendingFiles.forEach(function (path) {
+            console.log(color('fgRed', 'dim')(path));
+        });
+    }
+
+    console.log('\n Done.');
+}
+
+function eachFile(visit) {
+
+    const excludePatterns = [];
+    const extReg = /\.([a-zA-Z0-9_-]+)$/;
+
+    prepareExcludePatterns();
+    travel('./');
+
+    function travel(relativePath) {
+        if (isExclude(relativePath)) {
             return;
         }
 
-            fs.writeFileSync(fileUrl, headerStr + str, 'utf-8');
+        const absolutePath = pathTool.join(ecBasePath, relativePath);
+        const stat = fs.statSync(absolutePath);
+
+        if (stat.isFile()) {
+            visit(absolutePath, getExt(absolutePath));
+        }
+        else if (stat.isDirectory()) {
+            fs.readdirSync(relativePath).forEach(function (file) {
+                travel(pathTool.join(relativePath, file));
             });
+        }
+    }
+
+    function prepareExcludePatterns() {
+        const content = fs.readFileSync(excludesPath, {encoding: 'utf-8'});
+        content.replace(/\r/g, '\n').split('\n').forEach(function (line) {
+            line = line.trim();
+            if (line) {
+                excludePatterns.push(new RegExp(line));
+            }
         });
-});
\ No newline at end of file
+    }
+
+    // In Apache Rat, the ecludes file is check against the relative path
+    // (based on the base directory specified by the "--dir")
+    // See <https://github.com/sonatype/plexus-utils/blob/master/src/main/java/org/codehaus/plexus/util/DirectoryScanner.java#L400>
+    // Here we assume that the base directory is the `ecBasePath`.
+    function isExclude(relativePath) {
+        for (let i = 0; i < excludePatterns.length; i++) {
+            if (excludePatterns[i].test(relativePath)) {
+                return true;
+            }
+        }
+    }
+
+    function getExt(path) {
+        if (path) {
+            const mathResult = path.match(extReg);
+            return mathResult && mathResult[1];
+        }
+    }
+}
+
+// function assert(cond, msg) {
+//     if (!cond) {
+//         throw new Error(msg);
+//     }
+// }
+
+run();

build/config.js

@@ -61,7 +61,7 @@ function getPlugins({min, lang, sourcemap, removeDev, addBundleVersion}) {
             'dead_code': true
         },
         output: {
-            preamble: preamble
+            preamble: preamble.js
         }
     }));
 

build/pre-publish.js

@@ -60,17 +60,17 @@ module.exports = function () {
     prePulishSrc({
         inputPath: path.resolve(ecDir, 'echarts.all.js'),
         outputPath: path.resolve(ecDir, 'index.js'),
-        preamble: preamble
+        preamble: preamble.js
     });
     prePulishSrc({
         inputPath: path.resolve(ecDir, 'echarts.common.js'),
         outputPath: path.resolve(ecDir, 'index.common.js'),
-        preamble: preamble
+        preamble: preamble.js
     });
     prePulishSrc({
         inputPath: path.resolve(ecDir, 'echarts.simple.js'),
         outputPath: path.resolve(ecDir, 'index.simple.js'),
-        preamble: preamble
+        preamble: preamble.js
     });
 
     function transform({code, inputPath, outputPath}) {

build/preamble.js

-module.exports = `
+/*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*   http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing,
+* software distributed under the License is distributed on an
+* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+* KIND, either express or implied.  See the License for the
+* specific language governing permissions and limitations
+* under the License.
+*/
+
+const cStyleComment = `
 /*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
@@ -19,3 +38,144 @@ module.exports = `
 */
 
 `;
+
+const hashComment = `
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+`;
+
+const mlComment = `
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+`;
+
+function hasPreamble(fileExt) {
+    return fileExt && preambleMap[fileExt];
+}
+
+function addPreamble(fileStr, fileExt) {
+    if (fileStr && fileExt) {
+        const addFn = addFns[fileExt];
+        const headStr = preambleMap[fileExt];
+        return addFn && headStr && addFn(headStr, fileStr);
+    }
+}
+
+const addFns = {
+
+    js: function (headStr, fileStr) {
+        return headStr + fileStr;
+    },
+
+    css: function (headStr, fileStr) {
+        return headStr + fileStr;
+    },
+
+    sh: function (headStr, fileStr) {
+        // Git diff enables manual check.
+        if (/^#\!/.test(fileStr)) {
+            const lines = fileStr.split('\n');
+            lines.splice(1, 0, headStr);
+            return lines.join('\n');
+        }
+        else {
+            return headStr + fileStr;
+        }
+    },
+
+    html: function (headStr, fileStr) {
+        // Git diff enables manual check.
+        let resultStr = fileStr.replace(/^\s*<!DOCTYPE\s[^<>]+>/i, '$&' + headStr);
+        // If no doctype
+        if (resultStr.length === fileStr.length) {
+            resultStr = headStr + fileStr;
+        }
+
+        return resultStr;
+    }
+};
+
+const preambleMap = {
+    js: cStyleComment,
+    css: cStyleComment,
+    sh: hashComment,
+    html: mlComment
+};
+
+const licenseReg = [
+    {name: 'Apache', reg: /apache (license|commons)/i},
+    {name: 'BSD', reg: /BSD/},
+    {name: 'LGPL', reg: /LGPL/},
+    {name: 'GPL', reg: /GPL/},
+    {name: 'Mozilla', reg: /mozilla public/i},
+    {name: 'MIT', reg: /mit license/i}
+];
+
+function extractLicense(fileStr, fileExt) {
+    let commentText = extractComment(fileStr.trim(), fileExt);
+    if (!commentText) {
+        return;
+    }
+    for (let i = 0; i < licenseReg.length; i++) {
+        if (licenseReg[i].reg.test(commentText)) {
+            return licenseReg[i].name;
+        }
+    }
+}
+
+const cStyleCommentReg = /\/\*[\S\s]*?\*\//;
+const hashCommentReg = /^\s*#.*$/m;
+const mlCommentReg = /<\!\-\-[\S\s]*?\-\->/;
+const commentReg = {
+    js: cStyleCommentReg,
+    css: cStyleCommentReg,
+    sh: hashCommentReg,
+    html: mlCommentReg
+};
+
+function extractComment(str, fileExt) {
+    const reg = commentReg[fileExt];
+    if (!fileExt || !reg || !str) {
+        return;
+    }
+    let result = cStyleCommentReg.exec(str);
+    return result && result[0];
+}
+
+module.exports = Object.assign({
+    extractLicense,
+    hasPreamble,
+    addPreamble
+}, preambleMap);

build/rat.sh

+basePath=$(cd `dirname $0`; pwd)
+ecPath=${basePath}/..
+
+java -jar "${basePath}/apache-rat-0.12.jar" --exclude-file "${ecPath}/.rat-excludes" --dir ${ecPath}
+#  > "${basePath}/rat-result.txt"
+# java -jar "${basePath}/apache-rat/apache-rat-0.12.jar" -h
\ No newline at end of file