changeset 5247d0b71e4d in sao:default
details: https://hg.tryton.org/sao?cmd=changeset;node=5247d0b71e4d
description:
Always encode CSV export in UTF-8
Browsers do not support the charset in Blob, so it is already always
encoded in
UTF-8. But to help users on Windows, we add the UTF-8 BOM because
Windows
programs use it as magic number to detect encoding.
issue9527
review290761002
diffstat:
CHANGELOG | 1 +
src/sao.js | 2 +
src/tab.js | 8 +-
src/window.js | 169 ++++++++++++++++++++++++++++-----------------------------
4 files changed, 92 insertions(+), 88 deletions(-)
diffs (270 lines):
diff -r db82cdef5541 -r 5247d0b71e4d CHANGELOG
--- a/CHANGELOG Sat Sep 12 18:28:22 2020 +0200
+++ b/CHANGELOG Sat Sep 12 18:30:11 2020 +0200
@@ -1,3 +1,4 @@
+* Always encode CSV export in UTF-8
* Use tempusdominus and Popper for date time picker
* Support PYSON comparison of date and datetime
* Customize bootstrap default style
diff -r db82cdef5541 -r 5247d0b71e4d src/sao.js
--- a/src/sao.js Sat Sep 12 18:28:22 2020 +0200
+++ b/src/sao.js Sat Sep 12 18:30:11 2020 +0200
@@ -336,6 +336,8 @@
};
Sao.i18n.locale = {};
+ Sao.BOM_UTF8 = '\uFEFF';
+
Sao.get_preferences = function() {
var session = Sao.Session.current_session;
return session.reload_context().then(function() {
diff -r db82cdef5541 -r 5247d0b71e4d src/tab.js
--- a/src/tab.js Sat Sep 12 18:28:22 2020 +0200
+++ b/src/tab.js Sat Sep 12 18:30:11 2020 +0200
@@ -1304,19 +1304,21 @@
return Sao.Window.Export.format_row(row);
});
var delimiter = ',';
- var encoding = 'utf-8';
if (navigator.platform &&
navigator.platform.slice(0, 3) == 'Win') {
delimiter = ';';
- encoding = 'cp1252';
}
var csv = Papa.unparse(unparse_obj, {
quoteChar: '"',
delimiter: delimiter,
});
+ if (navigator.platform &&
+ navigator.platform.slice(0, 3) == 'Win') {
+ csv = Sao.BOM_UTF8 + csv;
+ }
Sao.common.download_file(
csv, export_.name + '.csv',
- {'type': 'text/csv;charset=' + encoding});
+ {'type': 'text/csv;charset=utf-8'});
});
}.bind(this));
},
diff -r db82cdef5541 -r 5247d0b71e4d src/window.js
--- a/src/window.js Sat Sep 12 18:28:22 2020 +0200
+++ b/src/window.js Sat Sep 12 18:30:11 2020 +0200
@@ -3,6 +3,53 @@
(function() {
'use strict';
+ var ENCODINGS = ["866", "ansi_x3.4-1968", "arabic", "ascii",
+ "asmo-708", "big5", "big5-hkscs", "chinese", "cn-big5", "cp1250",
+ "cp1251", "cp1252", "cp1253", "cp1254", "cp1255", "cp1256",
+ "cp1257", "cp1258", "cp819", "cp866", "csbig5", "cseuckr",
+ "cseucpkdfmtjapanese", "csgb2312", "csibm866", "csiso2022jp",
+ "csiso2022kr", "csiso58gb231280", "csiso88596e", "csiso88596i",
+ "csiso88598e", "csiso88598i", "csisolatin1", "csisolatin2",
+ "csisolatin3", "csisolatin4", "csisolatin5", "csisolatin6",
+ "csisolatin9", "csisolatinarabic", "csisolatincyrillic",
+ "csisolatingreek", "csisolatinhebrew", "cskoi8r", "csksc56011987",
+ "csmacintosh", "csshiftjis", "cyrillic", "dos-874", "ecma-114",
+ "ecma-118", "elot_928", "euc-jp", "euc-kr", "gb18030", "gb2312",
+ "gb_2312", "gb_2312-80", "gbk", "greek", "greek8", "hebrew",
+ "hz-gb-2312", "ibm819", "ibm866", "iso-2022-cn", "iso-2022-cn-ext",
+ "iso-2022-jp", "iso-2022-kr", "iso-8859-1", "iso-8859-10",
+ "iso-8859-11", "iso-8859-13", "iso-8859-14", "iso-8859-15",
+ "iso-8859-16", "iso-8859-2", "iso-8859-3", "iso-8859-4",
+ "iso-8859-5", "iso-8859-6", "iso-8859-6-e", "iso-8859-6-i",
+ "iso-8859-7", "iso-8859-8", "iso-8859-8-e", "iso-8859-8-i",
+ "iso-8859-9", "iso-ir-100", "iso-ir-101", "iso-ir-109",
+ "iso-ir-110", "iso-ir-126", "iso-ir-127", "iso-ir-138",
+ "iso-ir-144", "iso-ir-148", "iso-ir-149", "iso-ir-157", "iso-ir-58",
+ "iso8859-1", "iso8859-10", "iso8859-11", "iso8859-13", "iso8859-14",
+ "iso8859-15", "iso8859-2", "iso8859-3", "iso8859-4", "iso8859-5",
+ "iso8859-6", "iso8859-7", "iso8859-8", "iso8859-9", "iso88591",
+ "iso885910", "iso885911", "iso885913", "iso885914", "iso885915",
+ "iso88592", "iso88593", "iso88594", "iso88595", "iso88596",
+ "iso88597", "iso88598", "iso88599", "iso_8859-1", "iso_8859-15",
+ "iso_8859-1:1987", "iso_8859-2", "iso_8859-2:1987", "iso_8859-3",
+ "iso_8859-3:1988", "iso_8859-4", "iso_8859-4:1988", "iso_8859-5",
+ "iso_8859-5:1988", "iso_8859-6", "iso_8859-6:1987", "iso_8859-7",
+ "iso_8859-7:1987", "iso_8859-8", "iso_8859-8:1988", "iso_8859-9",
+ "iso_8859-9:1989", "koi", "koi8", "koi8-r", "koi8-ru", "koi8-u",
+ "koi8_r", "korean", "ks_c_5601-1987", "ks_c_5601-1989", "ksc5601",
+ "ksc_5601", "l1", "l2", "l3", "l4", "l5", "l6", "l9", "latin1",
+ "latin2", "latin3", "latin4", "latin5", "latin6", "logical", "mac",
+ "macintosh", "ms932", "ms_kanji", "shift-jis", "shift_jis", "sjis",
+ "sun_eu_greek", "tis-620", "unicode-1-1-utf-8", "us-ascii",
+ "utf-16", "utf-16be", "utf-16le", "utf-8", "utf8", "visual",
+ "windows-1250", "windows-1251", "windows-1252", "windows-1253",
+ "windows-1254", "windows-1255", "windows-1256", "windows-1257",
+ "windows-1258", "windows-31j", "windows-874", "windows-949",
+ "x-cp1250", "x-cp1251", "x-cp1252", "x-cp1253", "x-cp1254",
+ "x-cp1255", "x-cp1256", "x-cp1257", "x-cp1258", "x-euc-jp", "x-gbk",
+ "x-mac-cyrillic", "x-mac-roman", "x-mac-ukrainian", "x-sjis",
+ "x-user-defined", "x-x-big5"];
+
Sao.Window = {};
Sao.Window.InfoBar = Sao.class_(Object, {
@@ -860,52 +907,6 @@
Sao.Window.CSV = Sao.class_(Object, {
init: function(title) {
- this.encodings = ["866", "ansi_x3.4-1968", "arabic", "ascii",
- "asmo-708", "big5", "big5-hkscs", "chinese", "cn-big5", "cp1250",
- "cp1251", "cp1252", "cp1253", "cp1254", "cp1255", "cp1256",
- "cp1257", "cp1258", "cp819", "cp866", "csbig5", "cseuckr",
- "cseucpkdfmtjapanese", "csgb2312", "csibm866", "csiso2022jp",
- "csiso2022kr", "csiso58gb231280", "csiso88596e", "csiso88596i",
- "csiso88598e", "csiso88598i", "csisolatin1", "csisolatin2",
- "csisolatin3", "csisolatin4", "csisolatin5", "csisolatin6",
- "csisolatin9", "csisolatinarabic", "csisolatincyrillic",
- "csisolatingreek", "csisolatinhebrew", "cskoi8r", "csksc56011987",
- "csmacintosh", "csshiftjis", "cyrillic", "dos-874", "ecma-114",
- "ecma-118", "elot_928", "euc-jp", "euc-kr", "gb18030", "gb2312",
- "gb_2312", "gb_2312-80", "gbk", "greek", "greek8", "hebrew",
- "hz-gb-2312", "ibm819", "ibm866", "iso-2022-cn", "iso-2022-cn-ext",
- "iso-2022-jp", "iso-2022-kr", "iso-8859-1", "iso-8859-10",
- "iso-8859-11", "iso-8859-13", "iso-8859-14", "iso-8859-15",
- "iso-8859-16", "iso-8859-2", "iso-8859-3", "iso-8859-4",
- "iso-8859-5", "iso-8859-6", "iso-8859-6-e", "iso-8859-6-i",
- "iso-8859-7", "iso-8859-8", "iso-8859-8-e", "iso-8859-8-i",
- "iso-8859-9", "iso-ir-100", "iso-ir-101", "iso-ir-109",
- "iso-ir-110", "iso-ir-126", "iso-ir-127", "iso-ir-138",
- "iso-ir-144", "iso-ir-148", "iso-ir-149", "iso-ir-157",
"iso-ir-58",
- "iso8859-1", "iso8859-10", "iso8859-11", "iso8859-13",
"iso8859-14",
- "iso8859-15", "iso8859-2", "iso8859-3", "iso8859-4", "iso8859-5",
- "iso8859-6", "iso8859-7", "iso8859-8", "iso8859-9", "iso88591",
- "iso885910", "iso885911", "iso885913", "iso885914", "iso885915",
- "iso88592", "iso88593", "iso88594", "iso88595", "iso88596",
- "iso88597", "iso88598", "iso88599", "iso_8859-1", "iso_8859-15",
- "iso_8859-1:1987", "iso_8859-2", "iso_8859-2:1987", "iso_8859-3",
- "iso_8859-3:1988", "iso_8859-4", "iso_8859-4:1988", "iso_8859-5",
- "iso_8859-5:1988", "iso_8859-6", "iso_8859-6:1987", "iso_8859-7",
- "iso_8859-7:1987", "iso_8859-8", "iso_8859-8:1988", "iso_8859-9",
- "iso_8859-9:1989", "koi", "koi8", "koi8-r", "koi8-ru", "koi8-u",
- "koi8_r", "korean", "ks_c_5601-1987", "ks_c_5601-1989", "ksc5601",
- "ksc_5601", "l1", "l2", "l3", "l4", "l5", "l6", "l9", "latin1",
- "latin2", "latin3", "latin4", "latin5", "latin6", "logical", "mac",
- "macintosh", "ms932", "ms_kanji", "shift-jis", "shift_jis", "sjis",
- "sun_eu_greek", "tis-620", "unicode-1-1-utf-8", "us-ascii",
- "utf-16", "utf-16be", "utf-16le", "utf-8", "utf8", "visual",
- "windows-1250", "windows-1251", "windows-1252", "windows-1253",
- "windows-1254", "windows-1255", "windows-1256", "windows-1257",
- "windows-1258", "windows-31j", "windows-874", "windows-949",
- "x-cp1250", "x-cp1251", "x-cp1252", "x-cp1253", "x-cp1254",
- "x-cp1255", "x-cp1256", "x-cp1257", "x-cp1258", "x-euc-jp",
"x-gbk",
- "x-mac-cyrillic", "x-mac-roman", "x-mac-ukrainian", "x-sjis",
- "x-user-defined", "x-x-big5"];
this.dialog = new Sao.Dialog(title, 'csv', 'lg');
this.el = this.dialog.modal;
@@ -1078,38 +1079,6 @@
.appendTo(this.expander_csv);
this.expander_csv.append(' ');
- var encoding_label = jQuery('<label/>', {
- 'text': Sao.i18n.gettext('Encoding:'),
- 'class': 'control-label',
- 'for': 'input-encoding'
- });
-
- this.el_csv_encoding = jQuery('<select/>', {
- 'class': 'form-control',
- 'id': 'input-encoding'
- });
-
- for(var i=0; i<this.encodings.length; i++) {
- jQuery('<option/>', {
- 'val': this.encodings[i]
- }).append(this.encodings[i]).appendTo(this.el_csv_encoding);
- }
-
- var enc = 'utf-8';
- if (navigator.platform &&
- navigator.platform.slice(0, 3) == 'Win') {
- enc = 'cp1252';
- }
- this.el_csv_encoding.children('option[value="' + enc + '"]')
- .attr('selected', 'selected');
-
- jQuery('<div/>', {
- 'class': 'form-group'
- }).append(encoding_label)
- .append(this.el_csv_encoding)
- .appendTo(this.expander_csv);
- this.expander_csv.append(' ');
-
this.el.modal('show');
this.el.on('hidden.bs.modal', function() {
jQuery(this).remove();
@@ -1169,6 +1138,38 @@
}).append(this.file_input))
.appendTo(this.chooser_form);
+ var encoding_label = jQuery('<label/>', {
+ 'text': Sao.i18n.gettext('Encoding:'),
+ 'class': 'control-label',
+ 'for': 'input-encoding'
+ });
+
+ this.el_csv_encoding = jQuery('<select/>', {
+ 'class': 'form-control',
+ 'id': 'input-encoding'
+ });
+
+ for(var i=0; i < ENCODINGS.length; i++) {
+ jQuery('<option/>', {
+ 'val': ENCODINGS[i]
+ }).append(ENCODINGS[i]).appendTo(this.el_csv_encoding);
+ }
+
+ var enc = 'utf-8';
+ if (navigator.platform &&
+ navigator.platform.slice(0, 3) == 'Win') {
+ enc = 'cp1252';
+ }
+ this.el_csv_encoding.children('option[value="' + enc + '"]')
+ .attr('selected', 'selected');
+
+ jQuery('<div/>', {
+ 'class': 'form-group'
+ }).append(encoding_label)
+ .append(this.el_csv_encoding)
+ .appendTo(this.expander_csv);
+ this.expander_csv.append(' ');
+
var skip_label = jQuery('<label/>', {
'text': Sao.i18n.gettext('Lines to Skip:'),
'class': 'control-label',
@@ -1870,7 +1871,6 @@
}
},
export_csv: function(fields, data) {
- var encoding = this.el_csv_encoding.val();
var locale_format = this.el_csv_locale.prop('checked');
var unparse_obj = {};
unparse_obj.data = data.map(function(row) {
@@ -1883,8 +1883,12 @@
quoteChar: this.el_csv_quotechar.val(),
delimiter: this.el_csv_delimiter.val()
});
+ if (navigator.platform &&
+ navigator.platform.slice(0, 3) == 'Win') {
+ csv = Sao.BOM_UTF8 + csv;
+ }
Sao.common.download_file(
- csv, this.name + '.csv', {type: 'text/csv;charset=' +
encoding});
+ csv, this.name + '.csv', {type: 'text/csv;charset=utf-8'});
return Sao.common.message.run(
Sao.i18n.ngettext('%1 record saved', '%1 records saved',
data.length));
@@ -1926,11 +1930,6 @@
query_string.push(['f', field.getAttribute('path')]);
});
- var encoding = this.el_csv_encoding.val();
- if (encoding) {
- query_string.push(['enc', encoding]);
- }
-
query_string.push(['dl', this.el_csv_delimiter.val()]);
query_string.push(['qc', this.el_csv_quotechar.val()]);
