Hi,

I have a set of working patches for "fossil export" and "fossil import"
as attached.

Please note these are clean reverse engineered dedes independent of the
code in the git using the code analysis result included.

These codes are BSD-2-Clause licensed and I agree to copyright
assignment to Hipp, Wyrick & Company, Inc. to any patches submitted to
this ML.

Please review and consider.

Osamu

PS:  The previous "fossil import" code could not handle some
non-standard filenames.
From: Osamu Aoki <os...@debian.org>
Date: Tue, 11 Oct 2016 23:45:30 +0900
Subject: Fix export

Newline and UTF-8 characters in filename is now allowed

The git-fast-export code was analyzed as below.  This analysis was used to
create an independent code as this patch.

== Git fast-export filename encoding spec ==

Filenames exported by "git fast-export" are encoded and quoted if they contain
some non-plain ASCII alphanumeric characters.

Filenames imported by "git fast-import" are unencoded if it is recorded within
double quotes.

This memo documents the details of encoding and quotation to enable creating a
clean reverse engineered GPL unencumbered code in BSD-2-Clause license.

In Git 2.9.3, the focal point defining this feature:

    * quote.c                 quote_c_style
        Escape odd characters and quote the escaped string
        Return TRUE if used

    * builtin/fast-export.c   print_path_1
        Use quote_c_style if if finds needs to quote and escape
        Otherwise, quote entire string if ' ' is found in it
        If neither, use the original filename to export

    * quote.c                 unquote_c_style
        Unescape escaped characters

The following table summarizes the end result for each character found in the
filename for the combined effects of the above 2 functions:

             01234567 89abcdef
        0x00 OOOOOOOa btnvfrOO
        0x10 OOOOOOOO OOOOOOOO
        0x20 QAEAAAAA AAAAAAAA  (quote space and escape double quote)
        0x30 AAAAAAAA AAAAAAAA
        0x40 AAAAAAAA AAAAAAAA
        0x50 AAAAAAAA AAAAEAAA  (escape backslash)
        0x60 AAAAAAAA AAAAAAAA
        0x70 AAAAAAAA AAAAAAAO  (DEL is octal!)
        0x80 OOOOOOOO OOOOOOOO
        0x90 OOOOOOOO OOOOOOOO
        0xa0 OOOOOOOO OOOOOOOO
        0xb0 OOOOOOOO OOOOOOOO
        0xc0 OOOOOOOO OOOOOOOO
        0xd0 OOOOOOOO OOOOOOOO
        0xe0 OOOOOOOO OOOOOOOO
        0xf0 OOOOOOOO OOOOOOOO

Here each character is represented as HEX number adding row and column index.

The default of the quoting flag is FALSE.  The meaning of the conversion rule
indicators are the following:
---
 src/export.c | 60 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++----
 1 file changed, 56 insertions(+), 4 deletions(-)

diff --git a/src/export.c b/src/export.c
index f524cdc..e677351 100644
--- a/src/export.c
+++ b/src/export.c
@@ -305,6 +305,49 @@ void export_marks(FILE* f, Bag *blobs, Bag *vers){
 }
 
 /*
+** Quote and escape a filename to be exported if it contains some special
+** characters.  This implements not only the minimum requirements
+** '\\', '"' and '\n' but also 3 digits octal escapes for all high bits
+** characters and other standard single character escapes such as
+** '\a'..'\r' on par with what the git-fast-export does.
+*/
+static void quote_git_filename(const char *zName, char *name){
+  int i, j;
+  int needQuote;
+  static char escs[] = "abtnvfr";
+  needQuote = 0;
+  for(i=0; zName[i]!=0; i++){
+    if( zName[i]<=' ' || zName[i]>'~' || zName[i]=='\\' || zName[i]=='"' ){
+      needQuote = 1;
+    }
+  }
+  j = 0;
+  if( needQuote==1 ){
+    name[j++] = '"';
+  }
+  for(i=0; zName[i]!=0; i++){
+    if( zName[i]=='\\' || zName[i]=='"' ){
+      name[j++] = '\\';
+      name[j++] = zName[i];
+    }else if ( zName[i]>=' ' && zName[i]<='~' ){
+      name[j++] = zName[i];
+    }else if( zName[i]>='\a' && zName[i]<='\r' ){
+      name[j++] = '\\';
+      name[j++] = escs[( zName[i] - '\a')];
+    }else{
+      name[j++] = '\\';
+      name[j++] = ((zName[i] >> 6) & '\03') + '0';
+      name[j++] = ((zName[i] >> 3) & '\07') + '0';
+      name[j++] = ((zName[i] >> 0) & '\07') + '0';
+    }
+  }
+  if( needQuote==1 ){
+    name[j++] = '"';
+  }
+  name[j] = '\0';
+}
+
+/*
 ** COMMAND: export
 **
 ** Usage: %fossil export --git ?OPTIONS? ?REPOSITORY?
@@ -516,19 +559,28 @@ void export_cmd(void){
     );
     while( db_step(&q4)==SQLITE_ROW ){
       const char *zName = db_column_text(&q4,0);
+      char *name;
       int zNew = db_column_int(&q4,1);
       int mPerm = db_column_int(&q4,2);
-      if( zNew==0)
-        printf("D %s\n", zName);
-      else if( bag_find(&blobs, zNew) ) {
+      if( zName==0 || zName=="" ){
+        name = fossil_malloc( 1 );
+        name[0] = '\0';
+      }else{
+        name = fossil_malloc( 4*strlen(zName) + 2 + 1 );
+      }
+      quote_git_filename(zName, name);
+      if( zNew==0 ){
+        printf("D %s\n", name);
+      }else if( bag_find(&blobs, zNew) ){
         const char *zPerm;
         switch( mPerm ){
           case PERM_LNK:  zPerm = "120000";   break;
           case PERM_EXE:  zPerm = "100755";   break;
           default:        zPerm = "100644";   break;
         }
-        printf("M %s :%d %s\n", zPerm, BLOBMARK(zNew), zName);
+        printf("M %s :%d %s\n", zPerm, BLOBMARK(zNew), name);
       }
+      fossil_free(name);
     }
     db_finalize(&q4);
     db_finalize(&q3);
From: Osamu Aoki <os...@debian.org>
Date: Thu, 13 Oct 2016 21:55:12 +0900
Subject: Fix import

Single chapacter escape sequences are properly handled

The git-fast-export code was analyzed.  This analysis was used to create 
an independent code as this patch.

The code is written to be robust to accept non-conforming inputs.

The octal sequence is always with 3 digits as git-fast-import.
---
 src/import.c | 67 ++++++++++++++++++++++++++++++++++++++++++------------------
 1 file changed, 47 insertions(+), 20 deletions(-)

diff --git a/src/import.c b/src/import.c
index b74186c..57e65f2 100644
--- a/src/import.c
+++ b/src/import.c
@@ -477,36 +477,63 @@ static ImportFile *import_find_file(const char *zName, int *pI, int mx){
 }
 
 /*
-** Dequote a fast-export filename.  Filenames are normally unquoted.  But
-** if the contain some obscure special characters, quotes might be added.
+** Dequote and unescape a git-fast-export filename.  Filenames are normally
+** unquoted by the git-fast-export command.  But if they contain some special
+*: characters, special characters are escaped and quotes are added.  This
+** function reverses this conversion to get the original filename.
 */
 static void dequote_git_filename(char *zName){
-  int n, i, j;
   if( zName==0 || zName[0]!='"' ) return;
-  n = (int)strlen(zName);
+  int n = (int)strlen(zName);
   if( zName[n-1]!='"' ) return;
-  for(i=0, j=1; j<n-1; j++){
-    char c = zName[j];
-    int x;
-    if( c=='\\' ){
-      if( j+3 <= n-1
-       && zName[j+1]>='0' && zName[j+1]<='3'
-       && zName[j+2]>='0' && zName[j+2]<='7'
-       && zName[j+3]>='0' && zName[j+3]<='7'
-       && (x = 64*(zName[j+1]-'0') + 8*(zName[j+2]-'0') + zName[j+3]-'0')!=0
-      ){
-        c = (unsigned char)x;
-        j += 3;
+  /* zName is quoted */
+  int i=0; /* destination index, always i<j */
+  int j=1; /* source index: 0..n-1 */
+  while( j<n-1 ){
+    if( zName[j]=='\\' ){
+      if( j+1<=n-1 && !( zName[j+1]<'0' || zName[j+1]>'3' ) ){
+        switch( zName[j+1] ){
+          case 'a': zName[i++] = '\a'; break;
+          case 'b': zName[i++] = '\b'; break;
+          case 't': zName[i++] = '\t'; break;
+          case 'n': zName[i++] = '\n'; break;
+          case 'v': zName[i++] = '\v'; break;
+          case 'f': zName[i++] = '\f'; break;
+          case 'r': zName[i++] = '\r'; break;
+          case '\\': zName[i++] = '\\'; break;
+          case '"': zName[i++] = '"'; break;
+          default: /* illegal sequence */
+                    zName[i++] = '\\';
+                    zName[i++] = zName[j+1]; break;
+        }
+        j += 2;
+      }else if( j+3<=n-1
+                && zName[j+1]>='0' && zName[j+1]<='3'
+                && zName[j+2]>='0' && zName[j+2]<='7'
+                && zName[j+3]>='0' && zName[j+3]<='7' ){
+        /* Only 3 digits are used by the git-fast-export octal escape */
+        zName[i++] = (zName[j+1]-'0')<<6;
+        zName[i++] |= (zName[j+2]-'0')<<3;
+        zName[i++] |= (zName[j+3]-'0')<<0;
+        j += 4;
+      }else if(j+1<=n-1){
+        /* illegal sequence */
+        zName[i++] = zName[j];
+        zName[i++] = zName[j+1];
+        j += 2;
       }else{
-        c = zName[++j];
+        /* illegal sequence */
+        zName[i++] = zName[j];
+        j++;
       }
+    }else{
+      zName[i++] = zName[j];
+      j++;
     }
-    zName[i++] = c;
   }
-  zName[i] = 0;
+  zName[i] = '\0';
 }
 
-
 /*
 ** Read the git-fast-import format from pIn and insert the corresponding
 ** content into the database.

Attachment: signature.asc
Description: PGP signature

_______________________________________________
fossil-users mailing list
fossil-users@lists.fossil-scm.org
http://lists.fossil-scm.org:8080/cgi-bin/mailman/listinfo/fossil-users

Reply via email to