Hi,
PFA patch to fix the issue where user was not able to open the file with
non utf-8 encoding.
--
Regards,
Murtuza Zabuawala
EnterpriseDB: http://www.enterprisedb.com
The Enterprise PostgreSQL Company
diff --git a/web/pgadmin/tools/sqleditor/__init__.py
b/web/pgadmin/tools/sqleditor/__init__.py
index 8c17c97..0f3c909 100644
--- a/web/pgadmin/tools/sqleditor/__init__.py
+++ b/web/pgadmin/tools/sqleditor/__init__.py
@@ -38,6 +38,8 @@ from pgadmin.utils.exception import ConnectionLost
from pgadmin.utils.sqlautocomplete.autocomplete import SQLAutoComplete
from pgadmin.tools.sqleditor.utils.query_tool_preferences import \
RegisterQueryToolPreferences
+from pgadmin.tools.sqleditor.utils.query_tool_fs_utils import \
+ read_file_generator
MODULE_NAME = 'sqleditor'
@@ -1360,16 +1362,7 @@ def load_file():
errormsg=gettext("File type not supported")
)
- def gen():
- with codecs.open(file_path, 'r', encoding=enc) as fileObj:
- while True:
- # 4MB chunk (4 * 1024 * 1024 Bytes)
- data = fileObj.read(4194304)
- if not data:
- break
- yield data
-
- return Response(gen(), mimetype='text/plain')
+ return Response(read_file_generator(file_path, enc), mimetype='text/plain')
@blueprint.route('/save_file/', methods=["PUT", "POST"], endpoint='save_file')
diff --git a/web/pgadmin/tools/sqleditor/utils/query_tool_fs_utils.py
b/web/pgadmin/tools/sqleditor/utils/query_tool_fs_utils.py
new file mode 100644
index 0000000..7ed96e0
--- /dev/null
+++ b/web/pgadmin/tools/sqleditor/utils/query_tool_fs_utils.py
@@ -0,0 +1,53 @@
+##########################################################################
+#
+# pgAdmin 4 - PostgreSQL Tools
+#
+# Copyright (C) 2013 - 2018, The pgAdmin Development Team
+# This software is released under the PostgreSQL Licence
+#
+##########################################################################
+
+import codecs
+
+def read_file_generator(file, enc):
+ """
+ This will read the content of the file selected by user
+
+ Returns:
+ Content of file
+ """
+ try:
+ with codecs.open(file, 'r', encoding=enc) as fileObj:
+ while True:
+ # 4MB chunk (4 * 1024 * 1024 Bytes)
+ data = fileObj.read(4194304)
+ if not data:
+ break
+ yield data
+ except UnicodeDecodeError:
+ # This is the closest equivalent Python 3 offers to the permissive
+ # Python 2 text handling model. The latin-1 encoding in Python
+ # implements ISO_8859-1:1987 which maps all possible byte values
+ # to the first 256 Unicode code points, and thus ensures decoding
+ # errors will never occur regardless of the configured error and
+ # handles most of the Windows encodings
+ # handler.
+ # Ref: https://goo.gl/vDhggS
+ with codecs.open(file, 'r', encoding='latin-1') as fileObj:
+ while True:
+ # 4MB chunk (4 * 1024 * 1024 Bytes)
+ data = fileObj.read(4194304)
+ if not data:
+ break
+ yield data
+ except Exception:
+ # As a last resort we will use the provided encoding and then
+ # ignore the decoding errors
+ with codecs.open(file, 'r', encoding=enc, errors='ignore') as fileObj:
+ while True:
+ # 4MB chunk (4 * 1024 * 1024 Bytes)
+ data = fileObj.read(4194304)
+ if not data:
+ break
+ yield data
+
diff --git
a/web/pgadmin/tools/sqleditor/utils/tests/test_file_other_encoding.sql
b/web/pgadmin/tools/sqleditor/utils/tests/test_file_other_encoding.sql
new file mode 100644
index 0000000..6e3bf3f
--- /dev/null
+++ b/web/pgadmin/tools/sqleditor/utils/tests/test_file_other_encoding.sql
@@ -0,0 +1,2 @@
+/*Copyright � 2017/*
+SELECT 1;
\ No newline at end of file
diff --git
a/web/pgadmin/tools/sqleditor/utils/tests/test_file_utf8_encoding.sql
b/web/pgadmin/tools/sqleditor/utils/tests/test_file_utf8_encoding.sql
new file mode 100644
index 0000000..7358741
--- /dev/null
+++ b/web/pgadmin/tools/sqleditor/utils/tests/test_file_utf8_encoding.sql
@@ -0,0 +1,2 @@
+/*Copyright © 2017/*
+SELECT 1;
\ No newline at end of file
diff --git
a/web/pgadmin/tools/sqleditor/utils/tests/test_query_tool_fs_utils.py
b/web/pgadmin/tools/sqleditor/utils/tests/test_query_tool_fs_utils.py
new file mode 100644
index 0000000..d4e8dc4
--- /dev/null
+++ b/web/pgadmin/tools/sqleditor/utils/tests/test_query_tool_fs_utils.py
@@ -0,0 +1,39 @@
+##########################################################################
+#
+# pgAdmin 4 - PostgreSQL Tools
+#
+# Copyright (C) 2013 - 2018, The pgAdmin Development Team
+# This software is released under the PostgreSQL Licence
+#
+##########################################################################
+import os
+from pgadmin.utils.route import BaseTestGenerator
+from pgadmin.tools.sqleditor.utils.query_tool_fs_utils import \
+ read_file_generator
+
+
+class TestReadFileGeneratorForEncoding(BaseTestGenerator):
+ """
+ Check that the start_running_query method works as intended
+ """
+
+ scenarios = [
+ ('When user is trying to load the file with utf-8 encoding', dict(
+ file='test_file_utf8_encoding.sql',
+ encoding='utf-8'
+ )),
+ ('When user is trying to load the file with other encoding and'
+ ' trying to use utf-8 encoding to read it', dict(
+ file='test_file_other_encoding.sql',
+ encoding='utf-8'
+ )),
+ ]
+
+ def setUp(self):
+ self.dir_path = os.path.dirname(os.path.realpath(__file__))
+ self.complate_path = os.path.join(self.dir_path, self.file)
+
+ def runTest(self):
+ result = read_file_generator(self.complate_path, self.encoding)
+ # Check if file is read properly by the generator
+ self.assertIn('SELECT 1', next(result))
