DBF from js-harb

- merged DBF from js-harb (fixes #407 h/t @joefreire)
- updated codepage to 1.8.0
- stub for macro/dialog sheet parsing (fixes #292 h/t @GenoD)
- XLSB/XLSM write vbaraw (fixes #606 h/t @johnothetree)
- phantomjs demo (fixes #184 h/t @machinewu)

README.md

@@ -61,6 +61,7 @@ with a unified JS representation, and ES3/ES5 browser compatibility back to IE6.
   * [Excel 2003-2004 (SpreadsheetML)](#excel-2003-2004-spreadsheetml)
   * [Excel 2007+ Binary (XLSB, BIFF12)](#excel-2007-binary-xlsb-biff12)
   * [OpenDocument Spreadsheet (ODS/FODS) and Uniform Office Spreadsheet (UOS1/2)](#opendocument-spreadsheet-odsfods-and-uniform-office-spreadsheet-uos12)
+  * [dBASE and Visual FoxPro (DBF)](#dbase-and-visual-foxpro-dbf)
   * [Comma-Separated Values](#comma-separated-values)
   * [HTML](#html)
 - [Testing](#testing)
@@ -103,6 +104,7 @@ The `demos` directory includes sample projects for:
 - [`angular`](demos/angular/)
 - [`browserify`](demos/browserify/)
 - [`Adobe ExtendScript`](demos/extendscript/)
+- [`phantomjs`](demos/phantomjs/)
 - [`requirejs`](demos/requirejs/)
 - [`systemjs`](demos/systemjs/)
 - [`webpack`](demos/webpack/)
@@ -751,9 +753,12 @@ file but Excel will know how to handle it.  This library applies similar logic:
 |:-------|:--------------|:----------------------------------------------------|
 | `0xD0` | CFB Container | BIFF 5/8 or password-protected XLSX/XLSB            |
 | `0x09` | BIFF Stream   | BIFF 2/3/4/5                                        |
-| `0x3C` | XML/HTML      | SpreadsheetML or Flat ODS or UOS1 or HTML           |
-| `0x50` | ZIP Archive   | XLSB or XLSX/M or ODS or UOS2                       |
-| `0xFE` | UTF8 Text     | SpreadsheetML or Flat ODS or UOS1                   |
+| `0x3C` | XML/HTML      | SpreadsheetML / Flat ODS / UOS1 / HTML / plaintext  |
+| `0x50` | ZIP Archive   | XLSB or XLSX/M or ODS or UOS2 or plaintext          |
+| `0xFE` | UTF8 Text     | SpreadsheetML or Flat ODS or UOS1 or plaintext      |
+
+DBF files are detected based on the first byte as well as the third and fourth
+bytes (corresponding to month and day of the file date)
 
 ## Writing Options
 
@@ -997,6 +1002,7 @@ Despite the library name `xlsx`, it supports numerous spreadsheet file formats:
 | OpenDocument Spreadsheet (ODS)                               |  :o:  |  :o:  |
 | Flat XML ODF Spreadsheet (FODS)                              |  :o:  |  :o:  |
 | Uniform Office Format Spreadsheet (标文通 UOS1/UOS2)         |  :o:  |       |
+| dBASE II/III/IV / Visual FoxPro (DBF)                        |  :o:  |       |
 | **Other Common Spreadsheet Output Formats**                  |:-----:|:-----:|
 | HTML Tables                                                  |  :o:  |       |
 
@@ -1057,6 +1063,15 @@ UOS is a very similar format, and it comes in 2 varieties corresponding to ODS
 and FODS respectively.  For the most part, the difference between the formats
 lies in the names of tags and attributes.
 
+### dBASE and Visual FoxPro (DBF)
+
+DBF is really a typed table format: each column can only hold one data type and
+each record omits type information.  The parser generates a header row and
+inserts records starting at the second row of the worksheet.
+
+Multi-file extensions like external memos and tables are currently unsupported,
+limited by the general ability to read arbitrary files in the web browser.
+
 ### Comma-Separated Values
 
 Excel CSV deviates from RFC4180 in a number of important ways.  The generated

bin/xlsx.njs

@@ -101,6 +101,7 @@ else opts.cellFormula = false;
 
 if(program.all) {
 	opts.cellFormula = true;
+	opts.bookVBA = true;
 	opts.cellNF = true;
 	opts.cellStyles = true;
 	opts.sheetStubs = true;

bits/30_ctype.js

@@ -190,7 +190,7 @@ XMLNS.CT = 'http://schemas.openxmlformats.org/package/2006/content-types';
 
 function parse_ct(data/*:?string*/, opts) {
 	var ct = ({
-		workbooks:[], sheets:[], charts:[], dialogs:[],
+		workbooks:[], sheets:[], charts:[], dialogs:[], macros:[],
 		rels:[], strs:[], comments:[],
 		coreprops:[], extprops:[], custprops:[], themes:[], styles:[],
 		calcchains:[], vba: [],
@@ -266,6 +266,7 @@ function write_ct(ct, opts)/*:string*/ {
 	f3('themes');
 	['strs', 'styles'].forEach(f1);
 	['coreprops', 'extprops', 'custprops'].forEach(f3);
+	f3('vba');
 	if(o.length>2){ o[o.length] = ('</Types>'); o[1]=o[1].replace("/>",">"); }
 	return o.join("");
 }

bits/31_rels.js

 /* 9.3 Relationships */
 var RELS = ({
 	WB: "http://schemas.openxmlformats.org/officeDocument/2006/relationships/officeDocument",
-	SHEET: "http://sheetjs.openxmlformats.org/officeDocument/2006/relationships/officeDocument"
+	SHEET: "http://sheetjs.openxmlformats.org/officeDocument/2006/relationships/officeDocument",
+	VBA: "http://schemas.microsoft.com/office/2006/relationships/vbaProject"
 }/*:any*/);
 
 /* 9.3.3 Representing Relationships */

bits/40_harb.js

+/* from js-harb (C) 2014-present  SheetJS */
+var DBF = (function() {
+var dbf_codepage_map = {
+	/* Code Pages Supported by Visual FoxPro */
+	/*::[*/0x01/*::]*/:   437,           /*::[*/0x02/*::]*/:   850,
+	/*::[*/0x03/*::]*/:  1252,           /*::[*/0x04/*::]*/: 10000,
+	/*::[*/0x64/*::]*/:   852,           /*::[*/0x65/*::]*/:   866,
+	/*::[*/0x66/*::]*/:   865,           /*::[*/0x67/*::]*/:   861,
+	/*::[*/0x68/*::]*/:   895,           /*::[*/0x69/*::]*/:   620,
+	/*::[*/0x6A/*::]*/:   737,           /*::[*/0x6B/*::]*/:   857,
+	/*::[*/0x78/*::]*/:   950,           /*::[*/0x79/*::]*/:   949,
+	/*::[*/0x7A/*::]*/:   936,           /*::[*/0x7B/*::]*/:   932,
+	/*::[*/0x7C/*::]*/:   874,           /*::[*/0x7D/*::]*/:  1255,
+	/*::[*/0x7E/*::]*/:  1256,           /*::[*/0x96/*::]*/: 10007,
+	/*::[*/0x97/*::]*/: 10029,           /*::[*/0x98/*::]*/: 10006,
+	/*::[*/0xC8/*::]*/:  1250,           /*::[*/0xC9/*::]*/:  1251,
+	/*::[*/0xCA/*::]*/:  1254,           /*::[*/0xCB/*::]*/:  1253,
+
+	/* shapefile DBF extension */
+	/*::[*/0x00/*::]*/: 20127,           /*::[*/0x08/*::]*/:   865,
+	/*::[*/0x09/*::]*/:   437,           /*::[*/0x0A/*::]*/:   850,
+	/*::[*/0x0B/*::]*/:   437,           /*::[*/0x0D/*::]*/:   437,
+	/*::[*/0x0E/*::]*/:   850,           /*::[*/0x0F/*::]*/:   437,
+	/*::[*/0x10/*::]*/:   850,           /*::[*/0x11/*::]*/:   437,
+	/*::[*/0x12/*::]*/:   850,           /*::[*/0x13/*::]*/:   932,
+	/*::[*/0x14/*::]*/:   850,           /*::[*/0x15/*::]*/:   437,
+	/*::[*/0x16/*::]*/:   850,           /*::[*/0x17/*::]*/:   865,
+	/*::[*/0x18/*::]*/:   437,           /*::[*/0x19/*::]*/:   437,
+	/*::[*/0x1A/*::]*/:   850,           /*::[*/0x1B/*::]*/:   437,
+	/*::[*/0x1C/*::]*/:   863,           /*::[*/0x1D/*::]*/:   850,
+	/*::[*/0x1F/*::]*/:   852,           /*::[*/0x22/*::]*/:   852,
+	/*::[*/0x23/*::]*/:   852,           /*::[*/0x24/*::]*/:   860,
+	/*::[*/0x25/*::]*/:   850,           /*::[*/0x26/*::]*/:   866,
+	/*::[*/0x37/*::]*/:   850,           /*::[*/0x40/*::]*/:   852,
+	/*::[*/0x4D/*::]*/:   936,           /*::[*/0x4E/*::]*/:   949,
+	/*::[*/0x4F/*::]*/:   950,           /*::[*/0x50/*::]*/:   874,
+	/*::[*/0x57/*::]*/:  1252,           /*::[*/0x58/*::]*/:  1252,
+	/*::[*/0x59/*::]*/:  1252,
+
+	/*::[*/0xFF/*::]*/: 16969
+};
+
+/* TODO: find an actual specification */
+function dbf_to_aoa(buf, opts)/*:AOA*/ {
+	var out/*:AOA*/ = [];
+	/* TODO: browser based */
+	var d/*:Block*/ = (new_raw_buf(1)/*:any*/);
+	switch(opts.type) {
+		case 'base64': d = s2a(Base64.decode(buf)); break;
+		case 'binary': d = s2a(buf); break;
+		case 'buffer':
+		case 'array': d = buf; break;
+	}
+	prep_blob(d, 0);
+	/* header */
+	var ft = d.read_shift(1);
+	var memo = false;
+	var vfp = false;
+	switch(ft) {
+		case 0x03: break;
+		case 0x30: vfp = true; memo = true; break;
+		case 0x31: vfp = true; break;
+		case 0x83: memo = true; break;
+		case 0x8B: memo = true; break;
+		case 0xF5: memo = true; break;
+		default: process.exit(); throw new Error("DBF Unsupported Version: " + ft.toString(16));
+	}
+	var filedate = new Date(d.read_shift(1) + 1900, d.read_shift(1) - 1, d.read_shift(1));
+	var nrow = d.read_shift(4);
+	var fpos = d.read_shift(2);
+	var rlen = d.read_shift(2);
+	d.l+=16;
+
+	var flags = d.read_shift(1);
+	//if(memo && ((flags & 0x02) === 0)) throw new Error("DBF Flags " + flags.toString(16) + " ft " + ft.toString(16));
+
+	/* codepage present in FoxPro */
+	var current_cp = 1252;
+	if(d[d.l] !== 0) current_cp = dbf_codepage_map[d[d.l]];
+	d.l+=1;
+
+	d.l+=2;
+	var fields = [], field = {};
+	var hend = fpos - 10 - (vfp ? 264 : 0);
+	while(d.l < hend) {
+		field = {};
+		field.name = cptable.utils.decode(current_cp, d.slice(d.l, d.l+10)).replace(/[\u0000\r\n].*$/g,"");
+		d.l += 11;
+		field.type = String.fromCharCode(d.read_shift(1));
+		field.offset = d.read_shift(4);
+		field.len = d.read_shift(1);
+		field.dec = d.read_shift(1);
+		if(field.name.length) fields.push(field);
+		d.l += 14;
+		switch(field.type) {
+			// case 'B': break; // Binary
+			case 'C': break; // character
+			case 'D': break; // date
+			case 'F': break; // floating point
+			// case 'G': break; // General
+			case 'I': break; // long
+			case 'L': break; // boolean
+			case 'M': break; // memo
+			case 'N': break; // number
+			// case 'O': break; // double
+			// case 'P': break; // Picture
+			case 'T': break; // datetime
+			case 'Y': break; // currency
+			case '0': break; // null ?
+			case '+': break; // autoincrement
+			case '@': break; // timestamp
+			default: throw new Error('Unknown Field Type: ' + field.type);
+		}
+	}
+	if(d[d.l] !== 0x0D) d.l = fpos-1;
+	if(d.read_shift(1) !== 0x0D) throw new Error("DBF Terminator not found " + d.l + " " + d[d.l]);
+	d.l = fpos;
+	/* data */
+	var R = 0, C = 0;
+	out[0] = [];
+	for(C = 0; C != fields.length; ++C) out[0][C] = fields[C].name;
+	while(nrow-- > 0) {
+		if(d[d.l] === 0x2A) { d.l+=rlen; continue; }
+		++d.l;
+		out[++R] = []; C = 0;
+		for(C = 0; C != fields.length; ++C) {
+			var dd = d.slice(d.l, d.l+fields[C].len); d.l+=fields[C].len;
+			prep_blob(dd, 0);
+			var s = cptable.utils.decode(current_cp, dd);
+			switch(fields[C].type) {
+				case 'C':
+					out[R][C] = cptable.utils.decode(current_cp, dd);
+					out[R][C] = out[R][C].trim();
+					break;
+				case 'D':
+					if(s.length === 8) out[R][C] = new Date(+s.substr(0,4), +s.substr(4,2)-1, +s.substr(6,2));
+					else out[R][C] = s;
+					break;
+				case 'F': out[R][C] = parseFloat(s.trim()); break;
+				case 'I': out[R][C] = dd.read_shift(4, 'i'); break;
+				case 'L': switch(s.toUpperCase()) {
+					case 'Y': case 'T': out[R][C] = true; break;
+					case 'N': case 'F': out[R][C] = false; break;
+					case ' ': case '?': out[R][C] = false; break; /* NOTE: technically unitialized */
+					default: throw new Error("DBF Unrecognized L:|" + s + "|");
+					} break;
+				case 'M': /* TODO: handle memo files */
+					if(!memo) throw new Error("DBF Unexpected MEMO for type " + ft.toString(16));
+					out[R][C] = "##MEMO##" + dd.read_shift(4);
+					break;
+				case 'N': out[R][C] = +s.replace(/\u0000/g,"").trim(); break;
+				case 'T':
+					var day = dd.read_shift(4), ms = dd.read_shift(4);
+					throw new Error(day + " | " + ms);
+					//out[R][C] = new Date(); // FIXME!!!
+					//break;
+				case 'Y': out[R][C] = dd.read(4,'i')/1e4; break;
+				case '0':
+					if(fields[C].name === '_NullFlags') break;
+					/* falls through */
+				default: throw new Error("DBF Unsupported data type " + fields[C].type);
+			}
+		}
+	}
+	if(d.l < d.length && d[d.l++] != 0x1A) throw new Error("DBF EOF Marker missing " + (d.l-1) + " of " + d.length + " " + d[d.l-1].toString(16));
+	return out;
+}
+
+function dbf_to_sheet(buf, opts)/*:Worksheet*/ {
+	var o = opts || {};
+	if(!o.dateNF) o.dateNF = "yyyymmdd";
+	return aoa_to_sheet(dbf_to_aoa(buf, o), o);
+}
+
+function dbf_to_workbook(buf, opts)/*:Workbook*/ {
+	try { return sheet_to_workbook(dbf_to_sheet(buf, opts), opts); }
+	catch(e) { if(opts && opts.WTF) throw e; }
+	return ({SheetNames:[],Sheets:{}});
+}
+	return {
+		to_workbook: dbf_to_workbook,
+		to_sheet: dbf_to_sheet
+	};
+})();

bits/60_macrovba.js

+RELS.DS = "http://schemas.openxmlformats.org/officeDocument/2006/relationships/dialogsheet";
+RELS.MS = "http://schemas.microsoft.com/office/2006/relationships/xlMacrosheet";
+
+/* macro and dialog sheet stubs */
+function parse_ds_bin() { return {'!type':'dialog'}; }
+function parse_ds_xml() { return {'!type':'dialog'}; }
+function parse_ms_bin() { return {'!type':'macro'}; }
+function parse_ms_xml() { return {'!type':'macro'}; }

bits/74_xmlbin.js

@@ -13,6 +13,16 @@ function parse_cs(data, name/*:string*/, opts, rels, wb, themes, styles)/*:Works
 	return parse_cs_xml((data/*:any*/), opts, rels, wb, themes, styles);
 }
 
+function parse_ms(data, name/*:string*/, opts, rels, wb, themes, styles)/*:Worksheet*/ {
+	if(name.slice(-4)===".bin") return parse_ms_bin((data/*:any*/), opts, rels, wb, themes, styles);
+	return parse_ms_xml((data/*:any*/), opts, rels, wb, themes, styles);
+}
+
+function parse_ds(data, name/*:string*/, opts, rels, wb, themes, styles)/*:Worksheet*/ {
+	if(name.slice(-4)===".bin") return parse_ds_bin((data/*:any*/), opts, rels, wb, themes, styles);
+	return parse_ds_xml((data/*:any*/), opts, rels, wb, themes, styles);
+}
+
 function parse_sty(data, name/*:string*/, themes, opts) {
 	if(name.slice(-4)===".bin") return parse_sty_bin((data/*:any*/), themes, opts);
 	return parse_sty_xml((data/*:any*/), themes, opts);

bits/85_parsezip.js

-function safe_parse_wbrels(wbrels, sheets) {
-	if(!wbrels) return 0;
-	function get_type(n) {
+function get_sheet_type(n) {
 	if(RELS.WS.indexOf(n) > -1) return "sheet";
 	if(RELS.CS && n == RELS.CS) return "chart";
 	if(RELS.DS && n == RELS.DS) return "dialog";
 	if(RELS.MS && n == RELS.MS) return "macro";
 	if(!n || !n.length) return "sheet";
 	return n;
-	}
+}
+function safe_parse_wbrels(wbrels, sheets) {
+	if(!wbrels) return 0;
 	try {
-		wbrels = sheets.map(function pwbr(w) { if(!w.id) w.id = w.strRelID; return [w.name, wbrels['!id'][w.id].Target, get_type(wbrels['!id'][w.id].Type)]; });
+		wbrels = sheets.map(function pwbr(w) { if(!w.id) w.id = w.strRelID; return [w.name, wbrels['!id'][w.id].Target, get_sheet_type(wbrels['!id'][w.id].Type)]; });
 	} catch(e) { return null; }
 	return !wbrels || wbrels.length === 0 ? null : wbrels;
 }
@@ -31,6 +31,8 @@ function safe_parse_sheet(zip, path/*:string*/, relsPath/*:string*/, sheet, shee
 				var crelsp = get_rels_path(chartp);
 				cs = parse_chart(getzipstr(zip, chartp, true), chartp, opts, parse_rels(getzipstr(zip, crelsp,true), chartp), wb, cs);
 				break;
+			case 'macro': sheets[sheet]=parse_ms(data, path, opts,sheetRels[sheet], wb, themes, styles); break;
+			case 'dialog': sheets[sheet]=parse_ds(data, path, opts,sheetRels[sheet], wb, themes, styles); break;
 		}
 	} catch(e) { if(opts.WTF) throw e; }
 }

bits/86_writezip.js

@@ -23,6 +23,7 @@ function write_zip(wb/*:Workbook*/, opts/*:WriteOpts*/)/*:ZIP*/ {
 	opts.rels = {}; opts.wbrels = {};
 	opts.Strings = /*::((*/[]/*:: :any):SST)*/; opts.Strings.Count = 0; opts.Strings.Unique = 0;
 	var wbext = opts.bookType == "xlsb" ? "bin" : "xml";
+	var vbafmt = opts.bookType == "xlsb" || opts.bookType == "xlsm";
 	var ct = { workbooks: [], sheets: [], calcchains: [], themes: [], styles: [],
 		coreprops: [], extprops: [], custprops: [], strs:[], comments: [], vba: [],
 		TODO:[], rels:[], xmlns: "" };
@@ -89,6 +90,13 @@ function write_zip(wb/*:Workbook*/, opts/*:WriteOpts*/)/*:ZIP*/ {
 	ct.styles.push(f);
 	add_rels(opts.wbrels, ++rId, "styles." + wbext, RELS.STY);
 
+	if(wb.vbaraw && vbafmt) {
+		f = "xl/vbaProject.bin";
+		zip.file(f, wb.vbaraw);
+		ct.vba.push(f);
+		add_rels(opts.wbrels, ++rId, "vbaProject.bin", RELS.VBA);
+	}
+
 	zip.file("[Content_Types].xml", write_ct(ct, opts));
 	zip.file('_rels/.rels', write_rels(opts.rels));
 	zip.file('xl/_rels/workbook.' + wbext + '.rels', write_rels(opts.wbrels));

bits/87_read.js

@@ -35,6 +35,8 @@ function readSync(data/*:RawData*/, opts/*:?ParseOpts*/)/*:Workbook*/ {
 		case 0x3C: return parse_xlml(d, o);
 		case 0x50: if(n[1] == 0x4B && n[2] < 0x20 && n[3] < 0x20) return read_zip(d, o); break;
 		case 0xEF: return parse_xlml(d, o);
+		case 0x03: case 0x83: case 0x8B: return DBF.to_workbook(d, o);
+		case 0x30: case 0x31: if(n[2] <= 12 && n[3] <= 31) return DBF.to_workbook(d, o); break;
 		default: throw new Error("Unsupported file " + n.join("|"));
 	}
 	throw new Error("Unsupported file format " + n.join("|"));

demos/extendscript/.gitignore

+jszip.js
+shim.js
+xlsx.flow.js

demos/phantomjs/README.md

+# PhantomJS Demo
+
+This was tested in phantomjs 2.1.1, installed using the node module:
+
+```bash
+$ npm install -g phantomjs
+$ phantomjs phantomjs.js
+```

demos/phantomjs/phantomjs.js

+var fs = require('fs');
+var xlsx = require('../../xlsx');
+var page = require('webpage').create();
+
+page.open('http://www.google.com', function(status) {
+
+  var data = fs.read('sheetjs.xlsx', {mode: 'rb', charset: 'utf8'});
+  var workbook = xlsx.read(data, {type: 'binary'});
+  data = xlsx.utils.sheet_to_csv(workbook.Sheets['SheetJS']);
+  console.log("Data: " + data);
+
+  phantom.exit();
+});
+

demos/phantomjs/sheetjs.xlsx

+../extendscript/sheetjs.xlsx
\ No newline at end of file

docbits/10_install.md

@@ -28,6 +28,7 @@ The `demos` directory includes sample projects for:
 - [`angular`](demos/angular/)
 - [`browserify`](demos/browserify/)
 - [`Adobe ExtendScript`](demos/extendscript/)
+- [`phantomjs`](demos/phantomjs/)
 - [`requirejs`](demos/requirejs/)
 - [`systemjs`](demos/systemjs/)
 - [`webpack`](demos/webpack/)

docbits/80_parseopts.md

@@ -63,7 +63,10 @@ file but Excel will know how to handle it.  This library applies similar logic:
 |:-------|:--------------|:----------------------------------------------------|
 | `0xD0` | CFB Container | BIFF 5/8 or password-protected XLSX/XLSB            |
 | `0x09` | BIFF Stream   | BIFF 2/3/4/5                                        |
-| `0x3C` | XML/HTML      | SpreadsheetML or Flat ODS or UOS1 or HTML           |
-| `0x50` | ZIP Archive   | XLSB or XLSX/M or ODS or UOS2                       |
-| `0xFE` | UTF8 Text     | SpreadsheetML or Flat ODS or UOS1                   |
+| `0x3C` | XML/HTML      | SpreadsheetML / Flat ODS / UOS1 / HTML / plaintext  |
+| `0x50` | ZIP Archive   | XLSB or XLSX/M or ODS or UOS2 or plaintext          |
+| `0xFE` | UTF8 Text     | SpreadsheetML or Flat ODS or UOS1 or plaintext      |
+
+DBF files are detected based on the first byte as well as the third and fourth
+bytes (corresponding to month and day of the file date)
 

docbits/85_filetype.md

@@ -19,6 +19,7 @@ Despite the library name `xlsx`, it supports numerous spreadsheet file formats:
 | OpenDocument Spreadsheet (ODS)                               |  :o:  |  :o:  |
 | Flat XML ODF Spreadsheet (FODS)                              |  :o:  |  :o:  |
 | Uniform Office Format Spreadsheet (标文通 UOS1/UOS2)         |  :o:  |       |
+| dBASE II/III/IV / Visual FoxPro (DBF)                        |  :o:  |       |
 | **Other Common Spreadsheet Output Formats**                  |:-----:|:-----:|
 | HTML Tables                                                  |  :o:  |       |
 
@@ -79,6 +80,15 @@ UOS is a very similar format, and it comes in 2 varieties corresponding to ODS
 and FODS respectively.  For the most part, the difference between the formats
 lies in the names of tags and attributes.
 
+### dBASE and Visual FoxPro (DBF)
+
+DBF is really a typed table format: each column can only hold one data type and
+each record omits type information.  The parser generates a header row and
+inserts records starting at the second row of the worksheet.
+
+Multi-file extensions like external memos and tables are currently unsupported,
+limited by the general ability to read arbitrary files in the web browser.
+
 ### Comma-Separated Values
 
 Excel CSV deviates from RFC4180 in a number of important ways.  The generated

formats.dot

@@ -24,6 +24,7 @@ digraph G {
 		node  [style=filled,color=cyan];
 		html  [label="HTML\nTable"];
 		csv   [label="CSV"];
+		dbf   [label="DBF"];
 	}
 
 	subgraph JSXLSX {
@@ -46,6 +47,7 @@ digraph G {
 		fods -> csf
 		csf -> fods
 		uos -> csf
+		dbf -> csf
 		html -> csf
 	}
 }

package.json

@@ -16,7 +16,7 @@
 	"dependencies": {
 		"exit-on-epipe":"~1.0.0",
 		"ssf":"~0.9.0",
-		"codepage":"~1.7.0",
+		"codepage":"~1.8.0",
 		"cfb":"~0.11.0",
 		"crc-32":"~1.0.0",
 		"adler-32":"~1.0.0",

tests.lst

 AutoFilter.xlsb
-BlankSheetTypes.xlsb.pending # macrosheets
+BlankSheetTypes.xlsb
 ErrorTypes.xlsb
 NumberFormatCondition.xlsb
 RkNumber.xlsb
@@ -496,7 +496,7 @@ xlrd_test_comments_gdocs.xlsx
 xlrd_text_bar.xlsx
 xlsx-stream-d-date-cell.xlsx
 חישוב_נקודות_זיכוי.xlsx
-BlankSheetTypes.xlsm.pending
+BlankSheetTypes.xlsm
 NumberFormatCondition.xlsm
 apachepoi_45431.xlsm
 apachepoi_47026.xlsm

xlsx.flow.js

@@ -2744,7 +2744,7 @@ XMLNS.CT = 'http://schemas.openxmlformats.org/package/2006/content-types';
 
 function parse_ct(data/*:?string*/, opts) {
 	var ct = ({
-		workbooks:[], sheets:[], charts:[], dialogs:[],
+		workbooks:[], sheets:[], charts:[], dialogs:[], macros:[],
 		rels:[], strs:[], comments:[],
 		coreprops:[], extprops:[], custprops:[], themes:[], styles:[],
 		calcchains:[], vba: [],
@@ -2820,13 +2820,15 @@ function write_ct(ct, opts)/*:string*/ {
 	f3('themes');
 	['strs', 'styles'].forEach(f1);
 	['coreprops', 'extprops', 'custprops'].forEach(f3);
+	f3('vba');
 	if(o.length>2){ o[o.length] = ('</Types>'); o[1]=o[1].replace("/>",">"); }
 	return o.join("");
 }
 /* 9.3 Relationships */
 var RELS = ({
 	WB: "http://schemas.openxmlformats.org/officeDocument/2006/relationships/officeDocument",
-	SHEET: "http://sheetjs.openxmlformats.org/officeDocument/2006/relationships/officeDocument"
+	SHEET: "http://sheetjs.openxmlformats.org/officeDocument/2006/relationships/officeDocument",
+	VBA: "http://schemas.microsoft.com/office/2006/relationships/vbaProject"
 }/*:any*/);
 
 /* 9.3.3 Representing Relationships */
@@ -4717,6 +4719,190 @@ function parse_RString(blob, length, opts) {
 	cell.val = str;
 	return cell;
 }
+/* from js-harb (C) 2014-present  SheetJS */
+var DBF = (function() {
+var dbf_codepage_map = {
+	/* Code Pages Supported by Visual FoxPro */
+	/*::[*/0x01/*::]*/:   437,           /*::[*/0x02/*::]*/:   850,
+	/*::[*/0x03/*::]*/:  1252,           /*::[*/0x04/*::]*/: 10000,
+	/*::[*/0x64/*::]*/:   852,           /*::[*/0x65/*::]*/:   866,
+	/*::[*/0x66/*::]*/:   865,           /*::[*/0x67/*::]*/:   861,
+	/*::[*/0x68/*::]*/:   895,           /*::[*/0x69/*::]*/:   620,
+	/*::[*/0x6A/*::]*/:   737,           /*::[*/0x6B/*::]*/:   857,
+	/*::[*/0x78/*::]*/:   950,           /*::[*/0x79/*::]*/:   949,
+	/*::[*/0x7A/*::]*/:   936,           /*::[*/0x7B/*::]*/:   932,
+	/*::[*/0x7C/*::]*/:   874,           /*::[*/0x7D/*::]*/:  1255,
+	/*::[*/0x7E/*::]*/:  1256,           /*::[*/0x96/*::]*/: 10007,
+	/*::[*/0x97/*::]*/: 10029,           /*::[*/0x98/*::]*/: 10006,
+	/*::[*/0xC8/*::]*/:  1250,           /*::[*/0xC9/*::]*/:  1251,
+	/*::[*/0xCA/*::]*/:  1254,           /*::[*/0xCB/*::]*/:  1253,
+
+	/* shapefile DBF extension */
+	/*::[*/0x00/*::]*/: 20127,           /*::[*/0x08/*::]*/:   865,
+	/*::[*/0x09/*::]*/:   437,           /*::[*/0x0A/*::]*/:   850,
+	/*::[*/0x0B/*::]*/:   437,           /*::[*/0x0D/*::]*/:   437,
+	/*::[*/0x0E/*::]*/:   850,           /*::[*/0x0F/*::]*/:   437,
+	/*::[*/0x10/*::]*/:   850,           /*::[*/0x11/*::]*/:   437,
+	/*::[*/0x12/*::]*/:   850,           /*::[*/0x13/*::]*/:   932,
+	/*::[*/0x14/*::]*/:   850,           /*::[*/0x15/*::]*/:   437,
+	/*::[*/0x16/*::]*/:   850,           /*::[*/0x17/*::]*/:   865,
+	/*::[*/0x18/*::]*/:   437,           /*::[*/0x19/*::]*/:   437,
+	/*::[*/0x1A/*::]*/:   850,           /*::[*/0x1B/*::]*/:   437,
+	/*::[*/0x1C/*::]*/:   863,           /*::[*/0x1D/*::]*/:   850,
+	/*::[*/0x1F/*::]*/:   852,           /*::[*/0x22/*::]*/:   852,
+	/*::[*/0x23/*::]*/:   852,           /*::[*/0x24/*::]*/:   860,
+	/*::[*/0x25/*::]*/:   850,           /*::[*/0x26/*::]*/:   866,
+	/*::[*/0x37/*::]*/:   850,           /*::[*/0x40/*::]*/:   852,
+	/*::[*/0x4D/*::]*/:   936,           /*::[*/0x4E/*::]*/:   949,
+	/*::[*/0x4F/*::]*/:   950,           /*::[*/0x50/*::]*/:   874,
+	/*::[*/0x57/*::]*/:  1252,           /*::[*/0x58/*::]*/:  1252,
+	/*::[*/0x59/*::]*/:  1252,
+
+	/*::[*/0xFF/*::]*/: 16969
+};
+
+/* TODO: find an actual specification */
+function dbf_to_aoa(buf, opts)/*:AOA*/ {
+	var out/*:AOA*/ = [];
+	/* TODO: browser based */
+	var d/*:Block*/ = (new_raw_buf(1)/*:any*/);
+	switch(opts.type) {
+		case 'base64': d = s2a(Base64.decode(buf)); break;
+		case 'binary': d = s2a(buf); break;
+		case 'buffer':
+		case 'array': d = buf; break;
+	}
+	prep_blob(d, 0);
+	/* header */
+	var ft = d.read_shift(1);
+	var memo = false;
+	var vfp = false;
+	switch(ft) {
+		case 0x03: break;
+		case 0x30: vfp = true; memo = true; break;
+		case 0x31: vfp = true; break;
+		case 0x83: memo = true; break;
+		case 0x8B: memo = true; break;
+		case 0xF5: memo = true; break;
+		default: process.exit(); throw new Error("DBF Unsupported Version: " + ft.toString(16));
+	}
+	var filedate = new Date(d.read_shift(1) + 1900, d.read_shift(1) - 1, d.read_shift(1));
+	var nrow = d.read_shift(4);
+	var fpos = d.read_shift(2);
+	var rlen = d.read_shift(2);
+	d.l+=16;
+
+	var flags = d.read_shift(1);
+	//if(memo && ((flags & 0x02) === 0)) throw new Error("DBF Flags " + flags.toString(16) + " ft " + ft.toString(16));
+
+	/* codepage present in FoxPro */
+	var current_cp = 1252;
+	if(d[d.l] !== 0) current_cp = dbf_codepage_map[d[d.l]];
+	d.l+=1;
+
+	d.l+=2;
+	var fields = [], field = {};
+	var hend = fpos - 10 - (vfp ? 264 : 0);
+	while(d.l < hend) {
+		field = {};
+		field.name = cptable.utils.decode(current_cp, d.slice(d.l, d.l+10)).replace(/[\u0000\r\n].*$/g,"");
+		d.l += 11;
+		field.type = String.fromCharCode(d.read_shift(1));
+		field.offset = d.read_shift(4);
+		field.len = d.read_shift(1);
+		field.dec = d.read_shift(1);
+		if(field.name.length) fields.push(field);
+		d.l += 14;
+		switch(field.type) {
+			// case 'B': break; // Binary
+			case 'C': break; // character
+			case 'D': break; // date
+			case 'F': break; // floating point
+			// case 'G': break; // General
+			case 'I': break; // long
+			case 'L': break; // boolean
+			case 'M': break; // memo
+			case 'N': break; // number
+			// case 'O': break; // double
+			// case 'P': break; // Picture
+			case 'T': break; // datetime
+			case 'Y': break; // currency
+			case '0': break; // null ?
+			case '+': break; // autoincrement
+			case '@': break; // timestamp
+			default: throw new Error('Unknown Field Type: ' + field.type);
+		}
+	}
+	if(d[d.l] !== 0x0D) d.l = fpos-1;
+	if(d.read_shift(1) !== 0x0D) throw new Error("DBF Terminator not found " + d.l + " " + d[d.l]);
+	d.l = fpos;
+	/* data */
+	var R = 0, C = 0;
+	out[0] = [];
+	for(C = 0; C != fields.length; ++C) out[0][C] = fields[C].name;
+	while(nrow-- > 0) {
+		if(d[d.l] === 0x2A) { d.l+=rlen; continue; }
+		++d.l;
+		out[++R] = []; C = 0;
+		for(C = 0; C != fields.length; ++C) {
+			var dd = d.slice(d.l, d.l+fields[C].len); d.l+=fields[C].len;
+			prep_blob(dd, 0);
+			var s = cptable.utils.decode(current_cp, dd);
+			switch(fields[C].type) {
+				case 'C':
+					out[R][C] = cptable.utils.decode(current_cp, dd);
+					out[R][C] = out[R][C].trim();
+					break;
+				case 'D':
+					if(s.length === 8) out[R][C] = new Date(+s.substr(0,4), +s.substr(4,2)-1, +s.substr(6,2));
+					else out[R][C] = s;
+					break;
+				case 'F': out[R][C] = parseFloat(s.trim()); break;
+				case 'I': out[R][C] = dd.read_shift(4, 'i'); break;
+				case 'L': switch(s.toUpperCase()) {
+					case 'Y': case 'T': out[R][C] = true; break;
+					case 'N': case 'F': out[R][C] = false; break;
+					case ' ': case '?': out[R][C] = false; break; /* NOTE: technically unitialized */
+					default: throw new Error("DBF Unrecognized L:|" + s + "|");
+					} break;
+				case 'M': /* TODO: handle memo files */
+					if(!memo) throw new Error("DBF Unexpected MEMO for type " + ft.toString(16));
+					out[R][C] = "##MEMO##" + dd.read_shift(4);
+					break;
+				case 'N': out[R][C] = +s.replace(/\u0000/g,"").trim(); break;
+				case 'T':
+					var day = dd.read_shift(4), ms = dd.read_shift(4);
+					throw new Error(day + " | " + ms);
+					//out[R][C] = new Date(); // FIXME!!!
+					//break;
+				case 'Y': out[R][C] = dd.read(4,'i')/1e4; break;
+				case '0':
+					if(fields[C].name === '_NullFlags') break;
+					/* falls through */
+				default: throw new Error("DBF Unsupported data type " + fields[C].type);
+			}
+		}
+	}
+	if(d.l < d.length && d[d.l++] != 0x1A) throw new Error("DBF EOF Marker missing " + (d.l-1) + " of " + d.length + " " + d[d.l-1].toString(16));
+	return out;
+}
+
+function dbf_to_sheet(buf, opts)/*:Worksheet*/ {
+	var o = opts || {};
+	if(!o.dateNF) o.dateNF = "yyyymmdd";
+	return aoa_to_sheet(dbf_to_aoa(buf, o), o);
+}
+
+function dbf_to_workbook(buf, opts)/*:Workbook*/ {
+	try { return sheet_to_workbook(dbf_to_sheet(buf, opts), opts); }
+	catch(e) { if(opts && opts.WTF) throw e; }
+	return ({SheetNames:[],Sheets:{}});
+}
+	return {
+		to_workbook: dbf_to_workbook,
+		to_sheet: dbf_to_sheet
+	};
+})();
 /* 18.4.1 charset to codepage mapping */
 var CS2CP = ({
 	/*::[*/0/*::]*/:    1252, /* ANSI */
@@ -6296,6 +6482,14 @@ function parse_comments_bin(data, opts) {
 }
 
 function write_comments_bin(data, opts) { }
+RELS.DS = "http://schemas.openxmlformats.org/officeDocument/2006/relationships/dialogsheet";
+RELS.MS = "http://schemas.microsoft.com/office/2006/relationships/xlMacrosheet";
+
+/* macro and dialog sheet stubs */
+function parse_ds_bin() { return {'!type':'dialog'}; }
+function parse_ds_xml() { return {'!type':'dialog'}; }
+function parse_ms_bin() { return {'!type':'macro'}; }
+function parse_ms_xml() { return {'!type':'macro'}; }
 /* TODO: it will be useful to parse the function str */
 var rc_to_a1 = (function(){
 	var rcregex = /(^|[^A-Za-z])R(\[?)(-?\d+|)\]?C(\[?)(-?\d+|)\]?/g;
@@ -10221,6 +10415,16 @@ function parse_cs(data, name/*:string*/, opts, rels, wb, themes, styles)/*:Works
 	return parse_cs_xml((data/*:any*/), opts, rels, wb, themes, styles);
 }
 
+function parse_ms(data, name/*:string*/, opts, rels, wb, themes, styles)/*:Worksheet*/ {
+	if(name.slice(-4)===".bin") return parse_ms_bin((data/*:any*/), opts, rels, wb, themes, styles);
+	return parse_ms_xml((data/*:any*/), opts, rels, wb, themes, styles);
+}
+
+function parse_ds(data, name/*:string*/, opts, rels, wb, themes, styles)/*:Worksheet*/ {
+	if(name.slice(-4)===".bin") return parse_ds_bin((data/*:any*/), opts, rels, wb, themes, styles);
+	return parse_ds_xml((data/*:any*/), opts, rels, wb, themes, styles);
+}
+
 function parse_sty(data, name/*:string*/, themes, opts) {
 	if(name.slice(-4)===".bin") return parse_sty_bin((data/*:any*/), themes, opts);
 	return parse_sty_xml((data/*:any*/), themes, opts);
@@ -13850,18 +14054,18 @@ var fix_write_opts = fix_opts_func([
 
 	['WTF', false] /* WTF mode (throws errors) */
 ]);
-function safe_parse_wbrels(wbrels, sheets) {
-	if(!wbrels) return 0;
-	function get_type(n) {
+function get_sheet_type(n) {
 	if(RELS.WS.indexOf(n) > -1) return "sheet";
 	if(RELS.CS && n == RELS.CS) return "chart";
 	if(RELS.DS && n == RELS.DS) return "dialog";
 	if(RELS.MS && n == RELS.MS) return "macro";
 	if(!n || !n.length) return "sheet";
 	return n;
-	}
+}
+function safe_parse_wbrels(wbrels, sheets) {
+	if(!wbrels) return 0;
 	try {
-		wbrels = sheets.map(function pwbr(w) { if(!w.id) w.id = w.strRelID; return [w.name, wbrels['!id'][w.id].Target, get_type(wbrels['!id'][w.id].Type)]; });
+		wbrels = sheets.map(function pwbr(w) { if(!w.id) w.id = w.strRelID; return [w.name, wbrels['!id'][w.id].Target, get_sheet_type(wbrels['!id'][w.id].Type)]; });
 	} catch(e) { return null; }
 	return !wbrels || wbrels.length === 0 ? null : wbrels;
 }
@@ -13883,6 +14087,8 @@ function safe_parse_sheet(zip, path/*:string*/, relsPath/*:string*/, sheet, shee
 				var crelsp = get_rels_path(chartp);
 				cs = parse_chart(getzipstr(zip, chartp, true), chartp, opts, parse_rels(getzipstr(zip, crelsp,true), chartp), wb, cs);
 				break;
+			case 'macro': sheets[sheet]=parse_ms(data, path, opts,sheetRels[sheet], wb, themes, styles); break;
+			case 'dialog': sheets[sheet]=parse_ds(data, path, opts,sheetRels[sheet], wb, themes, styles); break;
 		}
 	} catch(e) { if(opts.WTF) throw e; }
 }
@@ -14042,6 +14248,7 @@ function write_zip(wb/*:Workbook*/, opts/*:WriteOpts*/)/*:ZIP*/ {
 	opts.rels = {}; opts.wbrels = {};
 	opts.Strings = /*::((*/[]/*:: :any):SST)*/; opts.Strings.Count = 0; opts.Strings.Unique = 0;
 	var wbext = opts.bookType == "xlsb" ? "bin" : "xml";
+	var vbafmt = opts.bookType == "xlsb" || opts.bookType == "xlsm";
 	var ct = { workbooks: [], sheets: [], calcchains: [], themes: [], styles: [],
 		coreprops: [], extprops: [], custprops: [], strs:[], comments: [], vba: [],
 		TODO:[], rels:[], xmlns: "" };
@@ -14108,6 +14315,13 @@ function write_zip(wb/*:Workbook*/, opts/*:WriteOpts*/)/*:ZIP*/ {
 	ct.styles.push(f);
 	add_rels(opts.wbrels, ++rId, "styles." + wbext, RELS.STY);
 
+	if(wb.vbaraw && vbafmt) {
+		f = "xl/vbaProject.bin";
+		zip.file(f, wb.vbaraw);
+		ct.vba.push(f);
+		add_rels(opts.wbrels, ++rId, "vbaProject.bin", RELS.VBA);
+	}
+
 	zip.file("[Content_Types].xml", write_ct(ct, opts));
 	zip.file('_rels/.rels', write_rels(opts.rels));
 	zip.file('xl/_rels/workbook.' + wbext + '.rels', write_rels(opts.wbrels));
@@ -14150,6 +14364,8 @@ function readSync(data/*:RawData*/, opts/*:?ParseOpts*/)/*:Workbook*/ {
 		case 0x3C: return parse_xlml(d, o);
 		case 0x50: if(n[1] == 0x4B && n[2] < 0x20 && n[3] < 0x20) return read_zip(d, o); break;
 		case 0xEF: return parse_xlml(d, o);
+		case 0x03: case 0x83: case 0x8B: return DBF.to_workbook(d, o);
+		case 0x30: case 0x31: if(n[2] <= 12 && n[3] <= 31) return DBF.to_workbook(d, o); break;
 		default: throw new Error("Unsupported file " + n.join("|"));
 	}
 	throw new Error("Unsupported file format " + n.join("|"));