Fossil

Check-in [22b570f4]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Give the user the option to convert file into UTF8 if they are in some other encoding when committed. The commit aborts regardless, to give the operator an opportunity to retest the changes before committing again.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: 22b570f4f33e1d9ba32d0eddc47c5fed7fabc0bd
User & Date: drh 2012-11-29 01:45:37
Original Comment: Give the user the option to convert file into UTF8 if they are in some other encoding when committed. The commit aborts regardless, to give the operator an opporunity to retest the changes before committing again.
Context
2012-11-29
07:56
Fix a couple line length issues and modify an error message. check-in: 3527aa44 user: mistachkin tags: trunk
03:14
Untested changes designed to fix broken filename encodings on MacOS. Move all character-set converting routines into a new file called utf8.c. Add the fossil_filename_to_utf8() routine, which converts Unicode to utf8 on windows and decomposed utf8 into precomposed utf8 on Mac. Ticket [e399bc1edfe45b] check-in: c24e568f user: drh tags: ticket-e399bc
01:45
Give the user the option to convert file into UTF8 if they are in some other encoding when committed. The commit aborts regardless, to give the operator an opportunity to retest the changes before committing again. check-in: 22b570f4 user: drh tags: trunk
2012-11-28
23:05
Serious bug fix: Avoid deleting the respository if the repository is in the checkout and you do a "fossil close". check-in: fcdeaa29 user: drh tags: trunk
08:44
merge trunk Closed-Leaf check-in: 9d67e2e2 user: jan.nijtmans tags: convert_before_commit_v2
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to src/checkin.c.

884
885
886
887
888
889
890




891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909

910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927

928
929
930
931
932



933
934
935
936
937
938
939
940
941
942
943
















944
945
946
947
948
949
950

951
952
953
954
955
956
957
....
1063
1064
1065
1066
1067
1068
1069

1070
1071
1072
1073
1074
1075
1076
....
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
....
1333
1334
1335
1336
1337
1338
1339


1340
1341
1342
1343
1344
1345
1346
  if( pnFBcard ) *pnFBcard = nFBcard;
}

/*
** Issue a warning and give the user an opportunity to abandon out
** if a Unicode (UTF-16) byte-order-mark (BOM) or a \r\n line ending
** is seen in a text file.




*/
static void commit_warning(
  const Blob *p,        /* The content of the file being committed. */
  int crnlOk,           /* Non-zero if CR/NL warnings should be disabled. */
  int binOk,            /* Non-zero if binary warnings should be disabled. */
  int unicodeOk,        /* Non-zero if unicode warnings should be disabled. */
  const char *zFilename /* The full name of the file being committed. */
){
  int eType;              /* return value of looks_like_utf8/utf16() */
  int fUnicode;           /* return value of starts_with_utf16_bom() */
  char *zMsg;             /* Warning message */
  Blob fname;             /* Relative pathname of the file */
  static int allOk = 0;   /* Set to true to disable this routine */

  if( allOk ) return;
  fUnicode = starts_with_utf16_bom(p, 0);
  eType = fUnicode ? looks_like_utf16(p) : looks_like_utf8(p);
  if( eType==0 || eType==-1 || fUnicode ){
    const char *zWarning;

    Blob ans;
    char cReply;

    if( eType==-1 && fUnicode ){
      if ( crnlOk && unicodeOk ){
        return; /* We don't want Unicode/CR/NL warnings for this file. */
      }
      zWarning = "Unicode and CR/NL line endings";
    }else if( eType==-1 ){
      if( crnlOk ){
        return; /* We don't want CR/NL warnings for this file. */
      }
      zWarning = "CR/NL line endings";
    }else if( eType==0 ){
      if( binOk ){
        return; /* We don't want binary warnings for this file. */
      }
      zWarning = "binary data";

    }else{
      if ( unicodeOk ){
        return; /* We don't want unicode warnings for this file. */
      }
      zWarning = "Unicode";



    }
    file_relative_name(zFilename, &fname, 0);
    blob_zero(&ans);
    zMsg = mprintf(
         "%s contains %s.  commit anyhow (a=all/y/N)? ",
         blob_str(&fname), zWarning);
    prompt_user(zMsg, &ans);
    fossil_free(zMsg);
    cReply = blob_str(&ans)[0];
    if( cReply=='a' || cReply=='A' ){
      allOk = 1;
















    }else if( cReply!='y' && cReply!='Y' ){
      fossil_fatal("Abandoning commit due to %s in %s",
                   zWarning, blob_str(&fname));
    }
    blob_reset(&ans);
    blob_reset(&fname);
  }

}

/*
** qsort() comparison routine for an array of pointers to strings.
*/
static int tagCmp(const void *a, const void *b){
  char **pA = (char**)a;
................................................................................
  Blob manifest;         /* Manifest in baseline form */
  Blob muuid;            /* Manifest uuid */
  Blob cksum1, cksum2;   /* Before and after commit checksums */
  Blob cksum1b;          /* Checksum recorded in the manifest */
  int szD;               /* Size of the delta manifest */
  int szB;               /* Size of the baseline manifest */
  int nConflict = 0;     /* Number of unresolved merge conflicts */

  Blob ans;
  char cReply;

  url_proxy_options();
  noSign = find_option("nosign",0,0)!=0;
  forceDelta = find_option("delta",0,0)!=0;
  forceBaseline = find_option("baseline",0,0)!=0;
................................................................................
      /* Instead of file content, put link destination path */
      blob_read_link(&content, zFullname);
    }else{
      blob_read_from_file(&content, zFullname);
    }
    /* Do not emit any warnings when they are disabled. */
    if( !noWarningFlag ){
      commit_warning(&content, crnlOk, binOk, unicodeOk, zFullname);
    }
    if( chnged==1 && contains_merge_marker(&content) ){
      Blob fname; /* Relative pathname of the file */

      nConflict++;
      file_relative_name(zFullname, &fname, 0);
      fossil_print("possible unresolved merge conflict in %s\n",
................................................................................
    db_multi_exec("UPDATE vfile SET mrid=%d, rid=%d WHERE id=%d", nrid,nrid,id);
    db_multi_exec("INSERT OR IGNORE INTO unsent VALUES(%d)", nrid);
  }
  db_finalize(&q);
  if( nConflict && !allowConflict ){
    fossil_fatal("abort due to unresolved merge conflicts; "
                 "use --allow-conflict to override");


  }

  /* Create the new manifest */
  if( blob_size(&comment)==0 ){
    blob_append(&comment, "(no comment)", -1);
  }
  if( forceDelta ){







>
>
>
>

|
|











|




>





|




|




|


>


|


>
>
>




|
|





>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







>







 







>







 







|







 







>
>







884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
....
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
....
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
....
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
  if( pnFBcard ) *pnFBcard = nFBcard;
}

/*
** Issue a warning and give the user an opportunity to abandon out
** if a Unicode (UTF-16) byte-order-mark (BOM) or a \r\n line ending
** is seen in a text file.
**
** Return 1 if the user pressed 'c'. In that case, the file will have
** been converted to UTF-8 (if it was UTF-16) with NL line-endings,
** and the original file will have been renamed to "<filename>-original".
*/
static int commit_warning(
  Blob *p,              /* The content of the file being committed. */
  int crnlOk,           /* Non-zero if CR/NL warnings should be disabled. */
  int binOk,            /* Non-zero if binary warnings should be disabled. */
  int unicodeOk,        /* Non-zero if unicode warnings should be disabled. */
  const char *zFilename /* The full name of the file being committed. */
){
  int eType;              /* return value of looks_like_utf8/utf16() */
  int fUnicode;           /* return value of starts_with_utf16_bom() */
  char *zMsg;             /* Warning message */
  Blob fname;             /* Relative pathname of the file */
  static int allOk = 0;   /* Set to true to disable this routine */

  if( allOk ) return 0;
  fUnicode = starts_with_utf16_bom(p, 0);
  eType = fUnicode ? looks_like_utf16(p) : looks_like_utf8(p);
  if( eType==0 || eType==-1 || fUnicode ){
    const char *zWarning;
    const char *zConvert = "c=convert/";
    Blob ans;
    char cReply;

    if( eType==-1 && fUnicode ){
      if ( crnlOk && unicodeOk ){
        return 0; /* We don't want Unicode/CR/NL warnings for this file. */
      }
      zWarning = "Unicode and CR/NL line endings";
    }else if( eType==-1 ){
      if( crnlOk ){
        return 0; /* We don't want CR/NL warnings for this file. */
      }
      zWarning = "CR/NL line endings";
    }else if( eType==0 ){
      if( binOk ){
        return 0; /* We don't want binary warnings for this file. */
      }
      zWarning = "binary data";
      zConvert = ""; /* We cannot convert binary files. */
    }else{
      if ( unicodeOk ){
        return 0; /* We don't want unicode warnings for this file. */
      }
      zWarning = "Unicode";
#ifndef _WIN32
      zConvert = ""; /* On Unix, we cannot easily convert Unicode files. */
#endif
    }
    file_relative_name(zFilename, &fname, 0);
    blob_zero(&ans);
    zMsg = mprintf(
         "%s contains %s.  commit anyhow (a=all/%sy/N)? ",
         blob_str(&fname), zWarning, zConvert);
    prompt_user(zMsg, &ans);
    fossil_free(zMsg);
    cReply = blob_str(&ans)[0];
    if( cReply=='a' || cReply=='A' ){
      allOk = 1;
    }else if( *zConvert && (cReply=='c' || cReply=='C') ){
      char *zOrig = file_newname(zFilename, "original", 1);
      FILE *f;
      blob_write_to_file(p, zOrig);
      fossil_free(zOrig);
      f = fossil_fopen(zFilename, "wb");
      if( fUnicode ) {
        int bomSize;
        const unsigned char *bom = get_utf8_bom(&bomSize);
        fwrite(bom, 1, bomSize, f);
        blob_to_utf8_no_bom(p, 0);
      }
      blob_remove_cr(p);
      fwrite(blob_buffer(p), 1, blob_size(p), f);
      fclose(f);
      return 1;
    }else if( cReply!='y' && cReply!='Y' ){
      fossil_fatal("Abandoning commit due to %s in %s",
                   zWarning, blob_str(&fname));
    }
    blob_reset(&ans);
    blob_reset(&fname);
  }
  return 0;
}

/*
** qsort() comparison routine for an array of pointers to strings.
*/
static int tagCmp(const void *a, const void *b){
  char **pA = (char**)a;
................................................................................
  Blob manifest;         /* Manifest in baseline form */
  Blob muuid;            /* Manifest uuid */
  Blob cksum1, cksum2;   /* Before and after commit checksums */
  Blob cksum1b;          /* Checksum recorded in the manifest */
  int szD;               /* Size of the delta manifest */
  int szB;               /* Size of the baseline manifest */
  int nConflict = 0;     /* Number of unresolved merge conflicts */
  int abortCommit = 0;
  Blob ans;
  char cReply;

  url_proxy_options();
  noSign = find_option("nosign",0,0)!=0;
  forceDelta = find_option("delta",0,0)!=0;
  forceBaseline = find_option("baseline",0,0)!=0;
................................................................................
      /* Instead of file content, put link destination path */
      blob_read_link(&content, zFullname);
    }else{
      blob_read_from_file(&content, zFullname);
    }
    /* Do not emit any warnings when they are disabled. */
    if( !noWarningFlag ){
      abortCommit |= commit_warning(&content, crnlOk, binOk, unicodeOk, zFullname);
    }
    if( chnged==1 && contains_merge_marker(&content) ){
      Blob fname; /* Relative pathname of the file */

      nConflict++;
      file_relative_name(zFullname, &fname, 0);
      fossil_print("possible unresolved merge conflict in %s\n",
................................................................................
    db_multi_exec("UPDATE vfile SET mrid=%d, rid=%d WHERE id=%d", nrid,nrid,id);
    db_multi_exec("INSERT OR IGNORE INTO unsent VALUES(%d)", nrid);
  }
  db_finalize(&q);
  if( nConflict && !allowConflict ){
    fossil_fatal("abort due to unresolved merge conflicts; "
                 "use --allow-conflict to override");
  } else if( abortCommit ){
    fossil_fatal("files are converted on your request. Please re-test before committing");
  }

  /* Create the new manifest */
  if( blob_size(&comment)==0 ){
    blob_append(&comment, "(no comment)", -1);
  }
  if( forceDelta ){