Fossil

Check-in [62a00bc7]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Enhance the 'reconstruct' command to set the correct hash policy (SHA1 or SHA3-256) for artifacts read from disk, inferred from the length of the path name. Also enhance the 'deconstruct' and 'reconstruct' commands with an option to ensure the artifact with RID=1 is a valid manifest. See the wiki page linked to this branch for more information and tests.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | reconstruct-sha3
Files: files | file ages | folders
SHA3-256:62a00bc7284a82762e40d75df247c1039e318ff13372aec926bc561ad6024e08
User & Date: florian 2019-01-28 10:12:00
Wiki:reconstruct-sha3
Context
2019-01-28
10:18
Add the new option --keep-rid1 for the 'deconstruct' and 'reconstruct' commands. check-in: 79b51366 user: florian tags: reconstruct-sha3
10:12
Enhance the 'reconstruct' command to set the correct hash policy (SHA1 or SHA3-256) for artifacts read from disk, inferred from the length of the path name. Also enhance the 'deconstruct' and 'reconstruct' commands with an option to ensure the artifact with RID=1 is a valid manifest. See the wiki page linked to this branch for more information and tests. check-in: 62a00bc7 user: florian tags: reconstruct-sha3
2019-01-27
19:32
Update the built-in SQLite to the latest 3.27.0 alpha. Updates to the change log. check-in: 5280c1ab user: drh tags: trunk
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to src/rebuild.c.

180
181
182
183
184
185
186


187

188
189
190
191
192
193
194
...
272
273
274
275
276
277
278











279
280
281
282
283
284
285
...
929
930
931
932
933
934
935



936




































937
938
939
940
941
942
943
...
951
952
953
954
955
956
957
958
959
960
961
962
963
964

965

966
967
968
969
970
971
972
973
974
975
976
977
978







































979
980
981
982
983
984
985
....
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
....
1090
1091
1092
1093
1094
1095
1096

1097
1098
1099
1100
1101
1102
1103
*/
static int totalSize;       /* Total number of artifacts to process */
static int processCnt;      /* Number processed so far */
static int ttyOutput;       /* Do progress output */
static Bag bagDone;         /* Bag of records rebuilt */

static char *zFNameFormat;  /* Format string for filenames on deconstruct */


static int prefixLength;    /* Length of directory prefix for deconstruct */



/*
** Draw the percent-complete message.
** The input is actually the permill complete.
*/
static void percent_complete(int permill){
................................................................................
      manifest_crosslink(rid, pUse, MC_NONE);
    }else{
      /* We are doing "fossil deconstruct" */
      char *zUuid = db_text(0, "SELECT uuid FROM blob WHERE rid=%d", rid);
      char *zFile = mprintf(zFNameFormat /*works-like:"%s:%s"*/,
                            zUuid, zUuid+prefixLength);
      blob_write_to_file(pUse,zFile);











      free(zFile);
      free(zUuid);
      blob_reset(pUse);
    }
    assert( blob_is_reset(pUse) );
    rebuild_step_done(rid);

................................................................................
void recon_read_dir(char *zPath){
  DIR *d;
  struct dirent *pEntry;
  Blob aContent; /* content of the just read artifact */
  static int nFileRead = 0;
  void *zUnicodePath;
  char *zUtf8Name;








































  zUnicodePath = fossil_utf8_to_path(zPath, 1);
  d = opendir(zUnicodePath);
  if( d ){
    while( (pEntry=readdir(d))!=0 ){
      Blob path;
      char *zSubpath;

................................................................................
      if( (pEntry->d_type==DT_UNKNOWN || pEntry->d_type==DT_LNK)
          ? (file_isdir(zSubpath, ExtFILE)==1) : (pEntry->d_type==DT_DIR) )
#else
      if( file_isdir(zSubpath, ExtFILE)==1 )
#endif
      {
        recon_read_dir(zSubpath);
      }else{
        blob_init(&path, 0, 0);
        blob_appendf(&path, "%s", zSubpath);
        if( blob_read_from_file(&aContent, blob_str(&path), ExtFILE)==-1 ){
          fossil_fatal("some unknown error occurred while reading \"%s\"",
                       blob_str(&path));
        }

        content_put(&aContent);

        blob_reset(&path);
        blob_reset(&aContent);
        fossil_print("\r%d", ++nFileRead);
        fflush(stdout);
      }
      free(zSubpath);
    }
    closedir(d);
  }else {
    fossil_fatal("encountered error %d while trying to open \"%s\".",
                  errno, g.argv[3]);
  }
  fossil_path_free(zUnicodePath);







































}

/*
** COMMAND: reconstruct*
**
** Usage: %fossil reconstruct FILENAME DIRECTORY
**
................................................................................
**   -L|--prefixlength N         set the length of the names of the DESTINATION
**                               subdirectories to N
**   --private                   Include private artifacts.
**
** See also: rebuild, reconstruct
*/
void deconstruct_cmd(void){
  const char *zDestDir;
  const char *zPrefixOpt;
  Stmt        s;
  int privateFlag;

  /* get and check prefix length argument and build format string */
  zPrefixOpt=find_option("prefixlength","L",1);
  if( !zPrefixOpt ){
................................................................................
  */
#endif
  if( prefixLength ){
    zFNameFormat = mprintf("%s/%%.%ds/%%s",zDestDir,prefixLength);
  }else{
    zFNameFormat = mprintf("%s/%%s",zDestDir);
  }


  bag_init(&bagDone);
  ttyOutput = 1;
  processCnt = 0;
  if (!g.fQuiet) {
    fossil_print("0 (0%%)...\r");
    fflush(stdout);







>
>

>







 







>
>
>
>
>
>
>
>
>
>
>







 







>
>
>

>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







 







|






>

>













>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







 







<







 







>







180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
...
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
...
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
....
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
....
1141
1142
1143
1144
1145
1146
1147

1148
1149
1150
1151
1152
1153
1154
....
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
*/
static int totalSize;       /* Total number of artifacts to process */
static int processCnt;      /* Number processed so far */
static int ttyOutput;       /* Do progress output */
static Bag bagDone;         /* Bag of records rebuilt */

static char *zFNameFormat;  /* Format string for filenames on deconstruct */
static int cchFNamePrefix;  /* Length of directory prefix in zFNameFormat */
static char *zDestDir;      /* Destination directory on deconstruct */
static int prefixLength;    /* Length of directory prefix for deconstruct */
static int fKeepRid1;       /* Flag to preserve RID=1 on de- and reconstruct */


/*
** Draw the percent-complete message.
** The input is actually the permill complete.
*/
static void percent_complete(int permill){
................................................................................
      manifest_crosslink(rid, pUse, MC_NONE);
    }else{
      /* We are doing "fossil deconstruct" */
      char *zUuid = db_text(0, "SELECT uuid FROM blob WHERE rid=%d", rid);
      char *zFile = mprintf(zFNameFormat /*works-like:"%s:%s"*/,
                            zUuid, zUuid+prefixLength);
      blob_write_to_file(pUse,zFile);
      if( rid==1 && fKeepRid1!=0 ){
        char *zFnDotRid1 = mprintf("%s/.rid1", zDestDir);
        char *zFnRid1 = zFile + cchFNamePrefix + 1; /* Skip directory slash */
        Blob bFileContents = empty_blob;
        blob_appendf(&bFileContents,
          "# The file holding the artifact with RID=1\n"
          "%s\n", zFnRid1);
        blob_write_to_file(&bFileContents, zFnDotRid1);
        blob_reset(&bFileContents);
        free(zFnDotRid1);
      }
      free(zFile);
      free(zUuid);
      blob_reset(pUse);
    }
    assert( blob_is_reset(pUse) );
    rebuild_step_done(rid);

................................................................................
void recon_read_dir(char *zPath){
  DIR *d;
  struct dirent *pEntry;
  Blob aContent; /* content of the just read artifact */
  static int nFileRead = 0;
  void *zUnicodePath;
  char *zUtf8Name;
  static int recursionLevel = 0;  /* Bookkeeping about the recursion level */
  static char *zFnRid1 = 0;       /* The file holding the artifact with RID=1 */
  static int cchPathInitial = 0;  /* The length of zPath on first recursion */

  recursionLevel++;
  if( recursionLevel==1 ){
    cchPathInitial = strlen(zPath);
    if( fKeepRid1!=0 ){
      char *zFnDotRid1 = mprintf("%s/.rid1", zPath);
      Blob bFileContents;
      if( blob_read_from_file(&bFileContents, zFnDotRid1, ExtFILE)!=-1 ){
        Blob line, value;
        while( blob_line(&bFileContents, &line)>0 ){
          if( blob_token(&line, &value)==0 ) continue;  /* Empty line */
          if( blob_buffer(&value)[0]=='#' ) continue;   /* Comment */
          blob_trim(&value);
          zFnRid1 = mprintf("%s/%s", zPath, blob_str(&value));
          break;
        }
        blob_reset(&bFileContents);
        if( zFnRid1 ){
          if( blob_read_from_file(&aContent, zFnRid1, ExtFILE)==-1 ){
            fossil_fatal("some unknown error occurred while reading \"%s\"",
                         zFnRid1);
          }else{
            recon_set_hash_policy(0, zFnRid1);
            content_put(&aContent);
            recon_restore_hash_policy();
            blob_reset(&aContent);
            fossil_print("\r%d", ++nFileRead);
            fflush(stdout);
          }
        }else{
          fossil_fatal("an error occurred while reading or parsing \"%s\"",
                       zFnDotRid1);
        }
      }
      free(zFnDotRid1);
    }
  }
  zUnicodePath = fossil_utf8_to_path(zPath, 1);
  d = opendir(zUnicodePath);
  if( d ){
    while( (pEntry=readdir(d))!=0 ){
      Blob path;
      char *zSubpath;

................................................................................
      if( (pEntry->d_type==DT_UNKNOWN || pEntry->d_type==DT_LNK)
          ? (file_isdir(zSubpath, ExtFILE)==1) : (pEntry->d_type==DT_DIR) )
#else
      if( file_isdir(zSubpath, ExtFILE)==1 )
#endif
      {
        recon_read_dir(zSubpath);
      }else if( fossil_strcmp(zSubpath, zFnRid1)!=0 ){
        blob_init(&path, 0, 0);
        blob_appendf(&path, "%s", zSubpath);
        if( blob_read_from_file(&aContent, blob_str(&path), ExtFILE)==-1 ){
          fossil_fatal("some unknown error occurred while reading \"%s\"",
                       blob_str(&path));
        }
        recon_set_hash_policy(cchPathInitial, blob_str(&path));
        content_put(&aContent);
        recon_restore_hash_policy();
        blob_reset(&path);
        blob_reset(&aContent);
        fossil_print("\r%d", ++nFileRead);
        fflush(stdout);
      }
      free(zSubpath);
    }
    closedir(d);
  }else {
    fossil_fatal("encountered error %d while trying to open \"%s\".",
                  errno, g.argv[3]);
  }
  fossil_path_free(zUnicodePath);
  if( recursionLevel==1 && zFnRid1!=0 ) free(zFnRid1);
  recursionLevel--;
}

/*
** Helper functions called from recon_read_dir() to set and restore the correct
** hash policy for an artifact read from disk, inferred from the length of the
** path name.
*/
#define HPOLICY_NOTDEFINED -1
static int saved_eHashPolicy = HPOLICY_NOTDEFINED;

void recon_set_hash_policy(
  const int cchPathPrefix,    /* Directory prefix length for zUuidAsFilePath */
  const char *zUuidAsFilePath /* Relative, well-formed, from recon_read_dir() */
){
  int cchTotal, cchHashPart;
  int new_eHashPolicy = HPOLICY_NOTDEFINED;
  assert( HNAME_COUNT==2 ); /* Review function if new hashes are implemented. */
  if( zUuidAsFilePath==0 ) return;
  cchTotal = strlen(zUuidAsFilePath);
  if( cchTotal==0 ) return;
  if( cchPathPrefix>=cchTotal) return;
  cchHashPart = cchTotal - cchPathPrefix;
  if( cchHashPart>=HNAME_LEN_K256 ){
    new_eHashPolicy = HPOLICY_SHA3;
  }else if( cchHashPart>=HNAME_LEN_SHA1 ){
    new_eHashPolicy = HPOLICY_SHA1;
  }
  if( new_eHashPolicy!=HPOLICY_NOTDEFINED ){
    saved_eHashPolicy = g.eHashPolicy;
    g.eHashPolicy = new_eHashPolicy;
  }
}

void recon_restore_hash_policy(){
  if( saved_eHashPolicy!=HPOLICY_NOTDEFINED ){
    g.eHashPolicy = saved_eHashPolicy;
  }
}

/*
** COMMAND: reconstruct*
**
** Usage: %fossil reconstruct FILENAME DIRECTORY
**
................................................................................
**   -L|--prefixlength N         set the length of the names of the DESTINATION
**                               subdirectories to N
**   --private                   Include private artifacts.
**
** See also: rebuild, reconstruct
*/
void deconstruct_cmd(void){

  const char *zPrefixOpt;
  Stmt        s;
  int privateFlag;

  /* get and check prefix length argument and build format string */
  zPrefixOpt=find_option("prefixlength","L",1);
  if( !zPrefixOpt ){
................................................................................
  */
#endif
  if( prefixLength ){
    zFNameFormat = mprintf("%s/%%.%ds/%%s",zDestDir,prefixLength);
  }else{
    zFNameFormat = mprintf("%s/%%s",zDestDir);
  }
  cchFNamePrefix = strlen(zDestDir);

  bag_init(&bagDone);
  ttyOutput = 1;
  processCnt = 0;
  if (!g.fQuiet) {
    fossil_print("0 (0%%)...\r");
    fflush(stdout);