The attached patch against current svn addresses the following issues,
including
some bugs I spotted while browsing the relevant parts of the code which our
earlier test cases and alertness didn't pick up, but which other cases and
checks
confirmed:
- Bug: When reading from stdin fenc is not set and ff is always unix, not
representative of what was actually used to interpret the input. Solution: Get
readfile() to set options except eol when flags contain READ_BUFFER.
- Bug: When converting a file, if an incomplete byte sequence appears at the
end
of the file, no retry with next fencs entry is done. Solution: If we can retry
in
such a situation, do so.
- Bug: When not converting a file, but validating utf8, incomplete invalid
sequences are not detected as errors, as the partial sequence is not checked
and
may not trigger a backtrack to the beginning of the sequence after the next
read(). Solution: Check partial sequences for validity.
Ben.
--~--~---------~--~----~------------~-------~--~----~
You received this message from the "vim_dev" maillist.
For more information, visit http://www.vim.org/maillist.php
-~----------~----~----~----~------~----~------~--~---
Index: fileio.c
===================================================================
--- fileio.c (revision 607)
+++ fileio.c (working copy)
@@ -221,11 +221,12 @@
{
int fd = 0;
int newfile = (flags & READ_NEW);
- int set_options = newfile || (eap != NULL &&
eap->read_edit);
int check_readonly;
int filtering = (flags & READ_FILTER);
int read_stdin = (flags & READ_STDIN);
int read_buffer = (flags & READ_BUFFER);
+ int set_options = newfile || read_buffer ||
+ (eap != NULL && eap->read_edit);
linenr_T read_buf_lnum = 1; /* next line to read from curbuf */
colnr_T read_buf_col = 0; /* next char to read from this line */
char_u c;
@@ -650,8 +651,15 @@
if (set_options)
{
- curbuf->b_p_eol = TRUE;
- curbuf->b_start_eol = TRUE;
+ /*
+ * Don't change eol if reading from buffer as it will be correctly
+ * set from reading stdin
+ */
+ if (!read_buffer)
+ {
+ curbuf->b_p_eol = TRUE;
+ curbuf->b_start_eol = TRUE;
+ }
#ifdef FEAT_MBYTE
curbuf->b_p_bomb = FALSE;
curbuf->b_start_bomb = FALSE;
@@ -1280,10 +1288,14 @@
if (size < 0) /* read error */
error = TRUE;
#ifdef FEAT_MBYTE
- else if (conv_restlen > 0)
+ else if (conv_restlen > 0 || (fio_flags == 0 &&
+ enc_utf8 && conv_error == 0 && !curbuf->b_p_bin &&
+ utf_head_off(buffer, ptr) > 0))
{
/* Reached end-of-file but some trailing bytes could
* not be converted. Truncated file? */
+ if (can_retry)
+ goto rewind_retry;
if (conv_error == 0)
conv_error = linecnt;
if (bad_char_behavior != BAD_DROP)
Index: mbyte.c
===================================================================
--- mbyte.c (revision 607)
+++ mbyte.c (working copy)
@@ -1657,7 +1657,13 @@
return 1;
len = utf8len_tab[*p];
if (len > size)
- return len; /* incomplete byte sequence. */
+ {
+ /* incomplete byte sequence; check if amount read so far is valid. */
+ for (i = 1; i < size; ++i)
+ if ((p[i] & 0xc0) != 0x80)
+ return 1;
+ return len;
+ }
for (i = 1; i < len; ++i)
if ((p[i] & 0xc0) != 0x80)
return 1;
