Hi,
Please find patch to fix the issue handling BOM characters in user files
when they load their sql files in query tool.
RM#2369
FYI,
I have attached sample BOM file in redmine for testing.
--
Regards,
Murtuza Zabuawala
EnterpriseDB: http://www.enterprisedb.com
The Enterprise PostgreSQL Company
diff --git a/web/pgadmin/misc/file_manager/__init__.py
b/web/pgadmin/misc/file_manager/__init__.py
index 6e081b1..85d7fb4 100644
--- a/web/pgadmin/misc/file_manager/__init__.py
+++ b/web/pgadmin/misc/file_manager/__init__.py
@@ -17,6 +17,7 @@ import sys
import time
from sys import platform as _platform
import config
+import codecs
import simplejson as json
from flask import render_template, Response, session, request as req, url_for
@@ -932,6 +933,68 @@ class Filemanager(object):
else:
return newPath, newName
+ @staticmethod
+ def check_file_for_bom_and_binary(filename, enc="utf-8"):
+ """
+ This utility function will check if file is Binary file
+ and/or if it startswith BOM character
+
+ Args:
+ filename: File
+ enc: Encoding for the file
+
+ Returns:
+ Status(Error?), Error message, Binary file flag,
+ BOM character flag and Encoding to open file
+ """
+ status = True
+ err_msg = None
+ is_startswith_bom = False
+
+ # check if file type is text or binary
+ text_chars = bytearray([7, 8, 9, 10, 12, 13, 27]) \
+ + bytearray(range(0x20, 0x7f)) \
+ + bytearray(range(0x80, 0x100))
+
+ def is_binary_string(bytes_data):
+ """Checks if string data is binary"""
+ return bool(
+ bytes_data.translate(None, text_chars)
+ )
+
+ # read the file
+ try:
+
+ with open(filename, 'rb') as f:
+ file_data = f.read(1024)
+
+ # Check for BOM in file data
+ for encoding, boms in \
+ ('utf-8-sig', (codecs.BOM_UTF8,)), \
+ ('utf-16', (codecs.BOM_UTF16_LE, codecs.BOM_UTF16_BE)), \
+ ('utf-32', (codecs.BOM_UTF32_LE, codecs.BOM_UTF32_BE)):
+ if any(file_data.startswith(bom) for bom in boms):
+ is_startswith_bom = True
+ enc = encoding
+
+ # Check if string is binary
+ is_binary = is_binary_string(file_data)
+
+ except IOError as ex:
+ status = False
+ # we don't want to expose real path of file
+ # so only show error message.
+ if ex.strerror == 'Permission denied':
+ err_msg = u"Error: {0}".format(ex.strerror)
+ else:
+ err_msg = u"Error: {0}".format(str(ex))
+
+ except Exception as ex:
+ status = False
+ err_msg = u"Error: {0}".format(str(ex))
+
+ return status, err_msg, is_binary, is_startswith_bom, enc
+
def addfolder(self, path, name):
"""
Functionality to create new folder
diff --git a/web/pgadmin/tools/sqleditor/__init__.py
b/web/pgadmin/tools/sqleditor/__init__.py
index 2d1d4e5..6431b3a 100644
--- a/web/pgadmin/tools/sqleditor/__init__.py
+++ b/web/pgadmin/tools/sqleditor/__init__.py
@@ -12,6 +12,7 @@ import simplejson as json
import os
import pickle
import random
+import codecs
from flask import Response, url_for, render_template, session, request
from flask_babel import gettext
@@ -1220,7 +1221,10 @@ def load_file():
file_path = unquote(file_data['file_name'])
if hasattr(str, 'decode'):
- file_path =
unquote(file_data['file_name']).encode('utf-8').decode('utf-8')
+ file_path = unquote(
+ file_data['file_name']
+ ).encode('utf-8').decode('utf-8')
+
# retrieve storage directory path
storage_manager_path = get_storage_directory()
if storage_manager_path:
@@ -1230,45 +1234,27 @@ def load_file():
file_path.lstrip('/').lstrip('\\')
)
- file_data = None
+ status, err_msg, is_binary, \
+ is_startswith_bom, enc = Filemanager.check_file_for_bom_and_binary(
+ file_path
+ )
- # check if file type is text or binary
- textchars = bytearray(
- [7, 8, 9, 10, 12, 13, 27]) + bytearray(
- range(0x20, 0x7f)) + bytearray(range(0x80, 0x100))
+ if not status:
+ return internal_server_error(
+ errormsg=gettext(err_msg)
+ )
- is_binary_string = lambda bytes: bool(
- bytes.translate(None, textchars)
- )
+ if is_binary:
+ return internal_server_error(
+ errormsg=gettext("File type not supported")
+ )
+
+ with codecs.open(file_path, 'r', encoding=enc) as fileObj:
+ data = fileObj.read()
- # read file
- try:
- with open(file_path, 'rb') as fileObj:
- is_binary = is_binary_string(fileObj.read(1024))
- if not is_binary:
- fileObj.seek(0)
- if hasattr(str, 'decode'):
- file_data = fileObj.read().decode('utf-8')
- else:
- file_data = fileObj.read()
- else:
- return internal_server_error(
- errormsg=gettext("File type not supported")
- )
- except IOError as e:
- # we don't want to expose real path of file
- # so only show error message.
- if e.strerror == 'Permission denied':
- err_msg = "Error: {0}".format(e.strerror)
- else:
- err_msg = "Error: {0}".format(e.strerror)
- return internal_server_error(errormsg=err_msg)
- except Exception as e:
- err_msg = "Error: {0}".format(e.strerror)
- return internal_server_error(errormsg=err_msg)
return make_json_response(
data={
- 'status': True, 'result': file_data,
+ 'status': True, 'result': data,
}
)
--
Sent via pgadmin-hackers mailing list ([email protected])
To make changes to your subscription:
http://www.postgresql.org/mailpref/pgadmin-hackers
