Fossil

Artifact [78cf66b7]
Login

Artifact 78cf66b757b72740af39f2ae0313db500daa80ed8af817a59fb2222d131a7e49:


     1  /*
     2  ** Copyright (c) 2010 D. Richard Hipp
     3  **
     4  ** This program is free software; you can redistribute it and/or
     5  ** modify it under the terms of the Simplified BSD License (also
     6  ** known as the "2-Clause License" or "FreeBSD License".)
     7  
     8  ** This program is distributed in the hope that it will be useful,
     9  ** but without any warranty; without even the implied warranty of
    10  ** merchantability or fitness for a particular purpose.
    11  **
    12  ** Author contact information:
    13  **   drh@sqlite.org
    14  **
    15  *******************************************************************************
    16  **
    17  ** This file contains code used to export the content of a Fossil
    18  ** repository in the git-fast-import format.
    19  */
    20  #include "config.h"
    21  #include "export.h"
    22  #include <assert.h>
    23  
    24  /*
    25  ** State information common to all export types.
    26  */
    27  static struct {
    28    const char *zTrunkName;     /* Name of trunk branch */
    29  } gexport;
    30  
    31  #if INTERFACE
    32  /*
    33  ** Each line in a git-fast-export "marK" file is an instance of
    34  ** this object.
    35  */
    36  struct mark_t {
    37    char *name;       /* Name of the mark.  Also starts with ":" */
    38    int rid;          /* Corresponding object in the BLOB table */
    39    char uuid[65];    /* The GIT hash name for this object */
    40  };
    41  #endif
    42  
    43  #if defined(_WIN32) || defined(WIN32)
    44  # undef popen
    45  # define popen _popen
    46  # undef pclose
    47  # define pclose _pclose
    48  #endif
    49  
    50  /*
    51  ** Output a "committer" record for the given user.
    52  ** NOTE: the given user name may be an email itself.
    53  */
    54  static void print_person(const char *zUser){
    55    static Stmt q;
    56    const char *zContact;
    57    char *zName;
    58    char *zEmail;
    59    int i, j;
    60    int isBracketed, atEmailFirst, atEmailLast;
    61  
    62    if( zUser==0 ){
    63      printf(" <unknown>");
    64      return;
    65    }
    66    db_static_prepare(&q, "SELECT info FROM user WHERE login=:user");
    67    db_bind_text(&q, ":user", zUser);
    68    if( db_step(&q)!=SQLITE_ROW ){
    69      db_reset(&q);
    70      zName = mprintf("%s", zUser);
    71      for(i=j=0; zName[i]; i++){
    72        if( zName[i]!='<' && zName[i]!='>' && zName[i]!='"' ){
    73          zName[j++] = zName[i];
    74        }
    75      }
    76      zName[j] = 0;
    77      printf(" %s <%s>", zName, zName);
    78      free(zName);
    79      return;
    80    }
    81  
    82    /*
    83    ** We have contact information.
    84    ** It may or may not contain an email address.
    85    **
    86    ** ASSUME:
    87    ** - General case:"Name Unicoded" <email@address.com> other info
    88    ** - If contact information contains more than an email address,
    89    **   then the email address is enclosed between <>
    90    ** - When only email address is specified, then it's stored verbatim
    91    ** - When name part is absent or all-blanks, use zUser instead
    92     */
    93    zName = NULL;
    94    zEmail = NULL;
    95    zContact = db_column_text(&q, 0);
    96    atEmailFirst = -1;
    97    atEmailLast = -1;
    98    isBracketed = 0;
    99    for(i=0; zContact[i] && zContact[i]!='@'; i++){
   100       if( zContact[i]=='<' ){
   101          isBracketed = 1;
   102          atEmailFirst = i+1;
   103       }
   104       else if( zContact[i]=='>' ){
   105          isBracketed = 0;
   106          atEmailFirst = i+1;
   107       }
   108       else if( zContact[i]==' ' && !isBracketed ){
   109          atEmailFirst = i+1;
   110       }
   111    }
   112    if( zContact[i]==0 ){
   113      /* No email address found. Take as user info if not empty */
   114      zName = mprintf("%s", zContact[0] ? zContact : zUser);
   115      for(i=j=0; zName[i]; i++){
   116        if( zName[i]!='<' && zName[i]!='>' && zName[i]!='"' ){
   117          zName[j++] = zName[i];
   118        }
   119      }
   120      zName[j] = 0;
   121  
   122      printf(" %s <%s>",  zName, zName);
   123      free(zName);
   124      db_reset(&q);
   125      return;
   126    }
   127    for(j=i+1; zContact[j] && zContact[j]!=' '; j++){
   128       if( zContact[j]=='>' )
   129          atEmailLast = j-1;
   130    }
   131    if ( atEmailLast==-1 ) atEmailLast = j-1;
   132    if ( atEmailFirst==-1 ) atEmailFirst = 0; /* Found only email */
   133  
   134    /*
   135    ** Found beginning and end of email address.
   136    ** Extract the address (trimmed and sanitized).
   137    */
   138    for(j=atEmailFirst; zContact[j] && zContact[j]==' '; j++){}
   139    zEmail = mprintf("%.*s", atEmailLast-j+1, &zContact[j]);
   140  
   141    for(i=j=0; zEmail[i]; i++){
   142       if( zEmail[i]!='<' && zEmail[i]!='>' ){
   143           zEmail[j++] = zEmail[i];
   144       }
   145    }
   146    zEmail[j] = 0;
   147  
   148    /*
   149    ** When bracketed email, extract the string _before_
   150    ** email as user name (may be enquoted).
   151    ** If missing or all-blank name, use zUser.
   152    */
   153    if( isBracketed && (atEmailFirst-1) > 0){
   154       for(i=atEmailFirst-2; i>=0 && zContact[i] && zContact[i]==' '; i--){}
   155       if( i>=0 ){
   156           for(j=0; j<i && zContact[j] && zContact[j]==' '; j++){}
   157           zName = mprintf("%.*s", i-j+1, &zContact[j]);
   158       }
   159    }
   160  
   161    if( zName==NULL ) zName = mprintf("%s", zUser);
   162    for(i=j=0; zName[i]; i++){
   163       if( zName[i]!='<' && zName[i]!='>' && zName[i]!='"' ){
   164           zName[j++] = zName[i];
   165       }
   166    }
   167    zName[j] = 0;
   168  
   169    printf(" %s <%s>", zName, zEmail);
   170    free(zName);
   171    free(zEmail);
   172    db_reset(&q);
   173  }
   174  
   175  #define REFREPLACEMENT        '_'
   176  
   177  /*
   178  ** Output a sanitized git named reference.
   179  ** https://git-scm.com/docs/git-check-ref-format
   180  ** This implementation assumes we are only printing
   181  ** the branch or tag part of the reference.
   182  */
   183  static void print_ref(const char *zRef){
   184    char *zEncoded = mprintf("%s", zRef);
   185    int i, w;
   186    if (zEncoded[0]=='@' && zEncoded[1]=='\0'){
   187      putchar(REFREPLACEMENT);
   188      return;
   189    }
   190    for(i=0, w=0; zEncoded[i]; i++, w++){
   191      if( i!=0 ){ /* Two letter tests */
   192        if( (zEncoded[i-1]=='.' && zEncoded[i]=='.') ||
   193            (zEncoded[i-1]=='@' && zEncoded[i]=='{') ){
   194          zEncoded[w]=zEncoded[w-1]=REFREPLACEMENT;
   195          continue;
   196        }
   197        if( zEncoded[i-1]=='/' && zEncoded[i]=='/' ){
   198          w--; /* Normalise to a single / by rolling back w */
   199          continue;
   200        }
   201      }
   202      /* No control characters */
   203      if( (unsigned)zEncoded[i]<0x20 || zEncoded[i]==0x7f ){
   204        zEncoded[w]=REFREPLACEMENT;
   205        continue;
   206      }
   207      switch( zEncoded[i] ){
   208        case ' ':
   209        case '^':
   210        case ':':
   211        case '?':
   212        case '*':
   213        case '[':
   214        case '\\':
   215          zEncoded[w]=REFREPLACEMENT;
   216          break;
   217      }
   218    }
   219    /* Cannot begin with a . or / */
   220    if( zEncoded[0]=='.' || zEncoded[0] == '/' ) zEncoded[0]=REFREPLACEMENT;
   221    if( i>0 ){
   222      i--; w--;
   223      /* Or end with a . or / */
   224      if( zEncoded[i]=='.' || zEncoded[i] == '/' ) zEncoded[w]=REFREPLACEMENT;
   225      /* Cannot end with .lock */
   226      if ( i>4 && strcmp((zEncoded+i)-5, ".lock")==0 )
   227        memset((zEncoded+w)-5, REFREPLACEMENT, 5);
   228    }
   229    printf("%s", zEncoded);
   230    free(zEncoded);
   231  }
   232  
   233  #define BLOBMARK(rid)   ((rid) * 2)
   234  #define COMMITMARK(rid) ((rid) * 2 + 1)
   235  
   236  /*
   237  ** insert_commit_xref()
   238  **   Insert a new (mark,rid,uuid) entry into the 'xmark' table.
   239  **   zName and zUuid must be non-null and must point to NULL-terminated strings.
   240  */
   241  void insert_commit_xref(int rid, const char *zName, const char *zUuid){
   242    db_multi_exec(
   243      "INSERT OR IGNORE INTO xmark(tname, trid, tuuid)"
   244      "VALUES(%Q,%d,%Q)",
   245      zName, rid, zUuid
   246    );
   247  }
   248  
   249  /*
   250  ** create_mark()
   251  **   Create a new (mark,rid,uuid) entry for the given rid in the 'xmark' table,
   252  **   and return that information as a struct mark_t in *mark.
   253  **   *unused_mark is a value representing a mark that is free for use--that is,
   254  **   it does not appear in the marks file, and has not been used during this
   255  **   export run.  Specifically, it is the supremum of the set of used marks
   256  **   plus one.
   257  **   This function returns -1 in the case where 'rid' does not exist, otherwise
   258  **   it returns 0.
   259  **   mark->name is dynamically allocated and is owned by the caller upon return.
   260  */
   261  int create_mark(int rid, struct mark_t *mark, unsigned int *unused_mark){
   262    char sid[13];
   263    char *zUuid = rid_to_uuid(rid);
   264    if( !zUuid ){
   265      fossil_trace("Undefined rid=%d\n", rid);
   266      return -1;
   267    }
   268    mark->rid = rid;
   269    sqlite3_snprintf(sizeof(sid), sid, ":%d", *unused_mark);
   270    *unused_mark += 1;
   271    mark->name = fossil_strdup(sid);
   272    sqlite3_snprintf(sizeof(mark->uuid), mark->uuid, "%s", zUuid);
   273    free(zUuid);
   274    insert_commit_xref(mark->rid, mark->name, mark->uuid);
   275    return 0;
   276  }
   277  
   278  /*
   279  ** mark_name_from_rid()
   280  **   Find the mark associated with the given rid.  Mark names always start
   281  **   with ':', and are pulled from the 'xmark' temporary table.
   282  **   If the given rid doesn't have a mark associated with it yet, one is
   283  **   created with a value of *unused_mark.
   284  **   *unused_mark functions exactly as in create_mark().
   285  **   This function returns NULL if the rid does not have an associated UUID,
   286  **   (i.e. is not valid).  Otherwise, it returns the name of the mark, which is
   287  **   dynamically allocated and is owned by the caller of this function.
   288  */
   289  char * mark_name_from_rid(int rid, unsigned int *unused_mark){
   290    char *zMark = db_text(0, "SELECT tname FROM xmark WHERE trid=%d", rid);
   291    if( zMark==NULL ){
   292      struct mark_t mark;
   293      if( create_mark(rid, &mark, unused_mark)==0 ){
   294        zMark = mark.name;
   295      }else{
   296        return NULL;
   297      }
   298    }
   299    return zMark;
   300  }
   301  
   302  /*
   303  ** Parse a single line of the mark file.  Store the result in the mark object.
   304  **
   305  ** "line" is a single line of input.
   306  ** This function returns -1 in the case that the line is blank, malformed, or
   307  ** the rid/uuid named in 'line' does not match what is in the repository
   308  ** database.  Otherwise, 0 is returned.
   309  **
   310  ** mark->name is dynamically allocated, and owned by the caller.
   311  */
   312  int parse_mark(char *line, struct mark_t *mark){
   313    char *cur_tok;
   314    char type_;
   315    cur_tok = strtok(line, " \t");
   316    if( !cur_tok || strlen(cur_tok)<2 ){
   317      return -1;
   318    }
   319    mark->rid = atoi(&cur_tok[1]);
   320    type_ = cur_tok[0];
   321    if( type_!='c' && type_!='b' ){
   322      /* This is probably a blob mark */
   323      mark->name = NULL;
   324      return 0;
   325    }
   326  
   327    cur_tok = strtok(NULL, " \t");
   328    if( !cur_tok ){
   329      /* This mark was generated by an older version of Fossil and doesn't
   330      ** include the mark name and uuid.  create_mark() will name the new mark
   331      ** exactly as it was when exported to git, so that we should have a
   332      ** valid mapping from git hash<->mark name<->fossil hash. */
   333      unsigned int mid;
   334      if( type_=='c' ){
   335        mid = COMMITMARK(mark->rid);
   336      }
   337      else{
   338        mid = BLOBMARK(mark->rid);
   339      }
   340      return create_mark(mark->rid, mark, &mid);
   341    }else{
   342      mark->name = fossil_strdup(cur_tok);
   343    }
   344  
   345    cur_tok = strtok(NULL, "\n");
   346    if( !cur_tok || (strlen(cur_tok)!=40 && strlen(cur_tok)!=64) ){
   347      free(mark->name);
   348      fossil_trace("Invalid SHA-1/SHA-3 in marks file: %s\n", cur_tok);
   349      return -1;
   350    }else{
   351      sqlite3_snprintf(sizeof(mark->uuid), mark->uuid, "%s", cur_tok);
   352    }
   353  
   354    /* make sure that rid corresponds to UUID */
   355    if( fast_uuid_to_rid(mark->uuid)!=mark->rid ){
   356      free(mark->name);
   357      fossil_trace("Non-existent SHA-1/SHA-3 in marks file: %s\n", mark->uuid);
   358      return -1;
   359    }
   360  
   361    /* insert a cross-ref into the 'xmark' table */
   362    insert_commit_xref(mark->rid, mark->name, mark->uuid);
   363    return 0;
   364  }
   365  
   366  /*
   367  ** Import the marks specified in file 'f';
   368  ** If 'blobs' is non-null, insert all blob marks into it.
   369  ** If 'vers' is non-null, insert all commit marks into it.
   370  ** If 'unused_marks' is non-null, upon return of this function, all values
   371  ** x >= *unused_marks are free to use as marks, i.e. they do not clash with
   372  ** any marks appearing in the marks file.
   373  **
   374  ** Each line in the file must be at most 100 characters in length.  This
   375  ** seems like a reasonable maximum for a 40-character uuid, and 1-13
   376  ** character rid.
   377  **
   378  ** The function returns -1 if any of the lines in file 'f' are malformed,
   379  ** or the rid/uuid information doesn't match what is in the repository
   380  ** database.  Otherwise, 0 is returned.
   381  */
   382  int import_marks(FILE* f, Bag *blobs, Bag *vers, unsigned int *unused_mark){
   383    char line[101];
   384    while(fgets(line, sizeof(line), f)){
   385      struct mark_t mark;
   386      if( strlen(line)==100 && line[99]!='\n' ){
   387        /* line too long */
   388        return -1;
   389      }
   390      if( parse_mark(line, &mark)<0 ){
   391        return -1;
   392      }else if( line[0]=='b' ){
   393        if( blobs!=NULL ){
   394          bag_insert(blobs, mark.rid);
   395        }
   396      }else{
   397        if( vers!=NULL ){
   398          bag_insert(vers, mark.rid);
   399        }
   400      }
   401      if( unused_mark!=NULL ){
   402        unsigned int mid = atoi(mark.name + 1);
   403        if( mid>=*unused_mark ){
   404          *unused_mark = mid + 1;
   405        }
   406      }
   407      free(mark.name);
   408    }
   409    return 0;
   410  }
   411  
   412  void export_mark(FILE* f, int rid, char obj_type)
   413  {
   414    unsigned int z = 0;
   415    char *zUuid = rid_to_uuid(rid);
   416    char *zMark;
   417    if( zUuid==NULL ){
   418      fossil_trace("No uuid matching rid=%d when exporting marks\n", rid);
   419      return;
   420    }
   421    /* Since rid is already in the 'xmark' table, the value of z won't be
   422    ** used, but pass in a valid pointer just to be safe. */
   423    zMark = mark_name_from_rid(rid, &z);
   424    fprintf(f, "%c%d %s %s\n", obj_type, rid, zMark, zUuid);
   425    free(zMark);
   426    free(zUuid);
   427  }
   428  
   429  /*
   430  **  If 'blobs' is non-null, it must point to a Bag of blob rids to be
   431  **  written to disk.  Blob rids are written as 'b<rid>'.
   432  **  If 'vers' is non-null, it must point to a Bag of commit rids to be
   433  **  written to disk.  Commit rids are written as 'c<rid> :<mark> <uuid>'.
   434  **  All commit (mark,rid,uuid) tuples are stored in 'xmark' table.
   435  **  This function does not fail, but may produce errors if a uuid cannot
   436  **  be found for an rid in 'vers'.
   437  */
   438  void export_marks(FILE* f, Bag *blobs, Bag *vers){
   439    int rid;
   440  
   441    if( blobs!=NULL ){
   442      rid = bag_first(blobs);
   443      if( rid!=0 ){
   444        do{
   445          export_mark(f, rid, 'b');
   446        }while( (rid = bag_next(blobs, rid))!=0 );
   447      }
   448    }
   449    if( vers!=NULL ){
   450      rid = bag_first(vers);
   451      if( rid!=0 ){
   452        do{
   453          export_mark(f, rid, 'c');
   454        }while( (rid = bag_next(vers, rid))!=0 );
   455      }
   456    }
   457  }
   458  
   459  /* This is the original header command (and hence documentation) for
   460  ** the "fossil export" command:
   461  ** 
   462  ** Usage: %fossil export --git ?OPTIONS? ?REPOSITORY?
   463  **
   464  ** Write an export of all check-ins to standard output.  The export is
   465  ** written in the git-fast-export file format assuming the --git option is
   466  ** provided.  The git-fast-export format is currently the only VCS
   467  ** interchange format supported, though other formats may be added in
   468  ** the future.
   469  **
   470  ** Run this command within a checkout.  Or use the -R or --repository
   471  ** option to specify a Fossil repository to be exported.
   472  **
   473  ** Only check-ins are exported using --git.  Git does not support tickets
   474  ** or wiki or tech notes or attachments, so none of those are exported.
   475  **
   476  ** If the "--import-marks FILE" option is used, it contains a list of
   477  ** rids to skip.
   478  **
   479  ** If the "--export-marks FILE" option is used, the rid of all commits and
   480  ** blobs written on exit for use with "--import-marks" on the next run.
   481  **
   482  ** Options:
   483  **   --export-marks FILE          export rids of exported data to FILE
   484  **   --import-marks FILE          read rids of data to ignore from FILE
   485  **   --rename-trunk NAME          use NAME as name of exported trunk branch
   486  **   --repository|-R REPOSITORY   export the given REPOSITORY
   487  **
   488  ** See also: import
   489  */
   490  /*
   491  ** COMMAND: export*
   492  **
   493  ** This command is deprecated.  Use "fossil git export" instead.
   494  */
   495  void export_cmd(void){
   496    Stmt q, q2, q3;
   497    Bag blobs, vers;
   498    unsigned int unused_mark = 1;
   499    const char *markfile_in;
   500    const char *markfile_out;
   501  
   502    bag_init(&blobs);
   503    bag_init(&vers);
   504  
   505    find_option("git", 0, 0);   /* Ignore the --git option for now */
   506    markfile_in = find_option("import-marks", 0, 1);
   507    markfile_out = find_option("export-marks", 0, 1);
   508  
   509    if( !(gexport.zTrunkName = find_option("rename-trunk", 0, 1)) ){
   510      gexport.zTrunkName = "trunk";
   511    }
   512  
   513    db_find_and_open_repository(0, 2);
   514    verify_all_options();
   515    if( g.argc!=2 && g.argc!=3 ){ usage("--git ?REPOSITORY?"); }
   516  
   517    db_multi_exec("CREATE TEMPORARY TABLE oldblob(rid INTEGER PRIMARY KEY)");
   518    db_multi_exec("CREATE TEMPORARY TABLE oldcommit(rid INTEGER PRIMARY KEY)");
   519    db_multi_exec("CREATE TEMP TABLE xmark(tname TEXT UNIQUE, trid INT,"
   520                  " tuuid TEXT)");
   521    db_multi_exec("CREATE INDEX xmark_trid ON xmark(trid)");
   522    if( markfile_in!=0 ){
   523      Stmt qb,qc;
   524      FILE *f;
   525      int rid;
   526  
   527      f = fossil_fopen(markfile_in, "r");
   528      if( f==0 ){
   529        fossil_fatal("cannot open %s for reading", markfile_in);
   530      }
   531      if( import_marks(f, &blobs, &vers, &unused_mark)<0 ){
   532        fossil_fatal("error importing marks from file: %s", markfile_in);
   533      }
   534      db_prepare(&qb, "INSERT OR IGNORE INTO oldblob VALUES (:rid)");
   535      db_prepare(&qc, "INSERT OR IGNORE INTO oldcommit VALUES (:rid)");
   536      rid = bag_first(&blobs);
   537      if( rid!=0 ){
   538        do{
   539          db_bind_int(&qb, ":rid", rid);
   540          db_step(&qb);
   541          db_reset(&qb);
   542        }while((rid = bag_next(&blobs, rid))!=0);
   543      }
   544      rid = bag_first(&vers);
   545      if( rid!=0 ){
   546        do{
   547          db_bind_int(&qc, ":rid", rid);
   548          db_step(&qc);
   549          db_reset(&qc);
   550        }while((rid = bag_next(&vers, rid))!=0);
   551      }
   552      db_finalize(&qb);
   553      db_finalize(&qc);
   554      fclose(f);
   555    }
   556  
   557    /* Step 1:  Generate "blob" records for every artifact that is part
   558    ** of a check-in
   559    */
   560    fossil_binary_mode(stdout);
   561    db_multi_exec("CREATE TEMP TABLE newblob(rid INTEGER KEY, srcid INTEGER)");
   562    db_multi_exec("CREATE INDEX newblob_src ON newblob(srcid)");
   563    db_multi_exec(
   564      "INSERT INTO newblob"
   565      " SELECT DISTINCT fid,"
   566      "  CASE WHEN EXISTS(SELECT 1 FROM delta"
   567                         " WHERE rid=fid"
   568                         "   AND NOT EXISTS(SELECT 1 FROM oldblob"
   569                                           " WHERE srcid=fid))"
   570      "   THEN (SELECT srcid FROM delta WHERE rid=fid)"
   571      "   ELSE 0"
   572      "  END"
   573      " FROM mlink"
   574      " WHERE fid>0 AND NOT EXISTS(SELECT 1 FROM oldblob WHERE rid=fid)");
   575    db_prepare(&q,
   576      "SELECT DISTINCT fid FROM mlink"
   577      " WHERE fid>0 AND NOT EXISTS(SELECT 1 FROM oldblob WHERE rid=fid)");
   578    db_prepare(&q2, "INSERT INTO oldblob VALUES (:rid)");
   579    db_prepare(&q3, "SELECT rid FROM newblob WHERE srcid= (:srcid)");
   580    while( db_step(&q)==SQLITE_ROW ){
   581      int rid = db_column_int(&q, 0);
   582      Blob content;
   583  
   584      while( !bag_find(&blobs, rid) ){
   585        char *zMark;
   586        content_get(rid, &content);
   587        db_bind_int(&q2, ":rid", rid);
   588        db_step(&q2);
   589        db_reset(&q2);
   590        zMark = mark_name_from_rid(rid, &unused_mark);
   591        printf("blob\nmark %s\ndata %d\n", zMark, blob_size(&content));
   592        free(zMark);
   593        bag_insert(&blobs, rid);
   594        fwrite(blob_buffer(&content), 1, blob_size(&content), stdout);
   595        printf("\n");
   596        blob_reset(&content);
   597  
   598        db_bind_int(&q3, ":srcid", rid);
   599        if( db_step(&q3) != SQLITE_ROW ){
   600          db_reset(&q3);
   601          break;
   602        }
   603        rid = db_column_int(&q3, 0);
   604        db_reset(&q3);
   605      }
   606    }
   607    db_finalize(&q);
   608    db_finalize(&q2);
   609    db_finalize(&q3);
   610  
   611    /* Output the commit records.
   612    */
   613    topological_sort_checkins(0);
   614    db_prepare(&q,
   615      "SELECT strftime('%%s',mtime), objid, coalesce(ecomment,comment),"
   616      "       coalesce(euser,user),"
   617      "       (SELECT value FROM tagxref WHERE rid=objid AND tagid=%d)"
   618      "  FROM toponode, event"
   619      " WHERE toponode.tid=event.objid"
   620      "   AND event.type='ci'"
   621      "   AND NOT EXISTS (SELECT 1 FROM oldcommit WHERE toponode.tid=rid)"
   622      " ORDER BY toponode.tseq ASC",
   623      TAG_BRANCH
   624    );
   625    db_prepare(&q2, "INSERT INTO oldcommit VALUES (:rid)");
   626    while( db_step(&q)==SQLITE_ROW ){
   627      Stmt q4;
   628      const char *zSecondsSince1970 = db_column_text(&q, 0);
   629      int ckinId = db_column_int(&q, 1);
   630      const char *zComment = db_column_text(&q, 2);
   631      const char *zUser = db_column_text(&q, 3);
   632      const char *zBranch = db_column_text(&q, 4);
   633      char *zMark;
   634  
   635      bag_insert(&vers, ckinId);
   636      db_bind_int(&q2, ":rid", ckinId);
   637      db_step(&q2);
   638      db_reset(&q2);
   639      if( zBranch==0 || fossil_strcmp(zBranch, "trunk")==0 ){
   640        zBranch = gexport.zTrunkName;
   641      }
   642      zMark = mark_name_from_rid(ckinId, &unused_mark);
   643      printf("commit refs/heads/");
   644      print_ref(zBranch);
   645      printf("\nmark %s\n", zMark);
   646      free(zMark);
   647      printf("committer");
   648      print_person(zUser);
   649      printf(" %s +0000\n", zSecondsSince1970);
   650      if( zComment==0 ) zComment = "null comment";
   651      printf("data %d\n%s\n", (int)strlen(zComment), zComment);
   652      db_prepare(&q3,
   653        "SELECT pid FROM plink"
   654        " WHERE cid=%d AND isprim"
   655        "   AND pid IN (SELECT objid FROM event)",
   656        ckinId
   657      );
   658      if( db_step(&q3) == SQLITE_ROW ){
   659        int pid = db_column_int(&q3, 0);
   660        zMark = mark_name_from_rid(pid, &unused_mark);
   661        printf("from %s\n", zMark);
   662        free(zMark);
   663        db_prepare(&q4,
   664          "SELECT pid FROM plink"
   665          " WHERE cid=%d AND NOT isprim"
   666          "   AND NOT EXISTS(SELECT 1 FROM phantom WHERE rid=pid)"
   667          " ORDER BY pid",
   668          ckinId);
   669        while( db_step(&q4)==SQLITE_ROW ){
   670          zMark = mark_name_from_rid(db_column_int(&q4, 0), &unused_mark);
   671          printf("merge %s\n", zMark);
   672          free(zMark);
   673        }
   674        db_finalize(&q4);
   675      }else{
   676        printf("deleteall\n");
   677      }
   678  
   679      db_prepare(&q4,
   680        "SELECT filename.name, mlink.fid, mlink.mperm FROM mlink"
   681        " JOIN filename ON filename.fnid=mlink.fnid"
   682        " WHERE mlink.mid=%d",
   683        ckinId
   684      );
   685      while( db_step(&q4)==SQLITE_ROW ){
   686        const char *zName = db_column_text(&q4,0);
   687        int zNew = db_column_int(&q4,1);
   688        int mPerm = db_column_int(&q4,2);
   689        if( zNew==0 ){
   690          printf("D %s\n", zName);
   691        }else if( bag_find(&blobs, zNew) ){
   692          const char *zPerm;
   693          zMark = mark_name_from_rid(zNew, &unused_mark);
   694          switch( mPerm ){
   695            case PERM_LNK:  zPerm = "120000";   break;
   696            case PERM_EXE:  zPerm = "100755";   break;
   697            default:        zPerm = "100644";   break;
   698          }
   699          printf("M %s %s %s\n", zPerm, zMark, zName);
   700          free(zMark);
   701        }
   702      }
   703      db_finalize(&q4);
   704      db_finalize(&q3);
   705      printf("\n");
   706    }
   707    db_finalize(&q2);
   708    db_finalize(&q);
   709    manifest_cache_clear();
   710  
   711  
   712    /* Output tags */
   713    db_prepare(&q,
   714       "SELECT tagname, rid, strftime('%%s',mtime),"
   715       "       (SELECT coalesce(euser, user) FROM event WHERE objid=rid),"
   716       "       value"
   717       "  FROM tagxref JOIN tag USING(tagid)"
   718       " WHERE tagtype=1 AND tagname GLOB 'sym-*'"
   719    );
   720    while( db_step(&q)==SQLITE_ROW ){
   721      const char *zTagname = db_column_text(&q, 0);
   722      int rid = db_column_int(&q, 1);
   723      char *zMark = mark_name_from_rid(rid, &unused_mark);
   724      const char *zSecSince1970 = db_column_text(&q, 2);
   725      const char *zUser = db_column_text(&q, 3);
   726      const char *zValue = db_column_text(&q, 4);
   727      if( rid==0 || !bag_find(&vers, rid) ) continue;
   728      zTagname += 4;
   729      printf("tag ");
   730      print_ref(zTagname);
   731      printf("\nfrom %s\n", zMark);
   732      free(zMark);
   733      printf("tagger");
   734      print_person(zUser);
   735      printf(" %s +0000\n", zSecSince1970);
   736      printf("data %d\n", zValue==NULL?0:(int)strlen(zValue)+1);
   737      if( zValue!=NULL ) printf("%s\n",zValue);
   738    }
   739    db_finalize(&q);
   740  
   741    if( markfile_out!=0 ){
   742      FILE *f;
   743      f = fossil_fopen(markfile_out, "w");
   744      if( f == 0 ){
   745        fossil_fatal("cannot open %s for writing", markfile_out);
   746      }
   747      export_marks(f, &blobs, &vers);
   748      if( ferror(f)!=0 || fclose(f)!=0 ){
   749        fossil_fatal("error while writing %s", markfile_out);
   750      }
   751    }
   752    bag_clear(&blobs);
   753    bag_clear(&vers);
   754  }
   755  
   756  /*
   757  ** Construct the temporary table toposort as follows:
   758  **
   759  **     CREATE TEMP TABLE toponode(
   760  **        tid INTEGER PRIMARY KEY,   -- Check-in id
   761  **        tseq INT                   -- integer total order on check-ins.
   762  **     );
   763  **
   764  ** This table contains all check-ins of the repository in topological
   765  ** order.  "Topological order" means that every parent check-in comes
   766  ** before all of its children.  Topological order is *almost* the same
   767  ** thing as "ORDER BY event.mtime".  Differences only arise when there
   768  ** are timewarps.  In as much as Git hates timewarps, we have to compute
   769  ** a correct topological order when doing an export.
   770  **
   771  ** Since mtime is a usually already nearly in topological order, the
   772  ** algorithm is to start with mtime, then make adjustments as necessary
   773  ** for timewarps.  This is not a great algorithm for the general case,
   774  ** but it is very fast for the overwhelmingly common case where there
   775  ** are few timewarps.
   776  */
   777  int topological_sort_checkins(int bVerbose){
   778    int nChange = 0;
   779    Stmt q1;
   780    Stmt chng;
   781    db_multi_exec(
   782      "CREATE TEMP TABLE toponode(\n"
   783      "  tid INTEGER PRIMARY KEY,\n"
   784      "  tseq INT\n"
   785      ");\n"
   786      "INSERT INTO toponode(tid,tseq) "
   787      " SELECT objid, CAST(mtime*8640000 AS int) FROM event WHERE type='ci';\n"
   788      "CREATE TEMP TABLE topolink(\n"
   789      "  tparent INT,\n"
   790      "  tchild INT,\n"
   791      "  PRIMARY KEY(tparent,tchild)\n"
   792      ") WITHOUT ROWID;"
   793      "INSERT INTO topolink(tparent,tchild)"
   794      "  SELECT pid, cid FROM plink;\n"
   795      "CREATE INDEX topolink_child ON topolink(tchild);\n"
   796    );
   797  
   798    /* Find a timewarp instance */
   799    db_prepare(&q1,
   800      "SELECT P.tseq, C.tid, C.tseq\n"
   801      "  FROM toponode P, toponode C, topolink X\n"
   802      " WHERE X.tparent=P.tid\n"
   803      "   AND X.tchild=C.tid\n"
   804      "   AND P.tseq>=C.tseq;"
   805    );
   806  
   807    /* Update the timestamp on :tid to have value :tseq */
   808    db_prepare(&chng,
   809      "UPDATE toponode SET tseq=:tseq WHERE tid=:tid"
   810    );
   811  
   812    while( db_step(&q1)==SQLITE_ROW ){
   813      i64 iParentTime = db_column_int64(&q1, 0);
   814      int iChild = db_column_int(&q1, 1);
   815      i64 iChildTime = db_column_int64(&q1, 2);
   816      nChange++;
   817      if( nChange>10000 ){
   818        fossil_fatal("failed to fix all timewarps after 100000 attempts");
   819      }
   820      db_reset(&q1);
   821      db_bind_int64(&chng, ":tid", iChild);
   822      db_bind_int64(&chng, ":tseq", iParentTime+1);
   823      db_step(&chng);
   824      db_reset(&chng);
   825      if( bVerbose ){
   826        fossil_print("moving %d from %lld to %lld\n",
   827                     iChild, iChildTime, iParentTime+1);
   828      }
   829    }
   830  
   831    db_finalize(&q1);
   832    db_finalize(&chng);
   833    return nChange;
   834  }
   835  
   836  /*
   837  ** COMMAND: test-topological-sort
   838  **
   839  ** Invoke the topological_sort_checkins() interface for testing
   840  ** purposes.
   841  */
   842  void test_topological_sort(void){
   843    int n;
   844    db_find_and_open_repository(0, 0);
   845    n = topological_sort_checkins(1);
   846    fossil_print("%d reorderings required\n", n);
   847  }
   848  
   849  /***************************************************************************
   850  ** Implementation of the "fossil git" command follows.  We hope that the
   851  ** new code that follows will largely replace the legacy "fossil export"
   852  ** and "fossil import" code above.
   853  */
   854  
   855  /* Verbosity level.  Higher means more output.
   856  **
   857  **    0     print nothing at all
   858  **    1     Errors only
   859  **    2     Progress information (This is the default)
   860  **    3     Extra details
   861  */
   862  #define VERB_ERROR  1
   863  #define VERB_NORMAL 2
   864  #define VERB_EXTRA  3
   865  static int gitmirror_verbosity = VERB_NORMAL;
   866  
   867  /*
   868  ** Output routine that depends on verbosity
   869  */
   870  static void gitmirror_message(int iLevel, const char *zFormat, ...){
   871    va_list ap;
   872    if( iLevel>gitmirror_verbosity ) return;
   873    va_start(ap, zFormat);
   874    fossil_vprint(zFormat, ap);
   875    va_end(ap);
   876  }
   877  
   878  /*
   879  ** Convert characters of z[] that are not allowed to be in branch or
   880  ** tag names into "_".
   881  */
   882  static void gitmirror_sanitize_name(char *z){
   883    static unsigned char aSafe[] = {
   884       /* x0 x1 x2 x3 x4 x5 x6 x7 x8  x9 xA xB xC xD xE xF */
   885           0, 0, 0, 0, 0, 0, 0, 0, 0,  0, 0, 0, 0, 0, 0, 0,  /* 0x */
   886           0, 0, 0, 0, 0, 0, 0, 0, 0,  0, 0, 0, 0, 0, 0, 0,  /* 1x */
   887           0, 1, 1, 1, 1, 1, 1, 1, 1,  1, 1, 1, 1, 1, 1, 1,  /* 2x */
   888           1, 1, 1, 1, 1, 1, 1, 1, 1,  1, 0, 1, 1, 1, 1, 0,  /* 3x */
   889           0, 1, 1, 1, 1, 1, 1, 1, 1,  1, 1, 1, 1, 1, 1, 1,  /* 4x */
   890           1, 1, 1, 1, 1, 1, 1, 1, 1,  1, 1, 0, 0, 1, 0, 1,  /* 5x */
   891           1, 1, 1, 1, 1, 1, 1, 1, 1,  1, 1, 1, 1, 1, 1, 1,  /* 6x */
   892           1, 1, 1, 1, 1, 1, 1, 1, 1,  1, 1, 1, 1, 1, 0, 0,  /* 7x */
   893    };
   894    unsigned char *zu = (unsigned char*)z;
   895    int i;
   896    for(i=0; zu[i]; i++){
   897      if( zu[i]>0x7f || !aSafe[zu[i]] ){
   898        zu[i] = '_';
   899      }else if( zu[i]=='/' && (i==0 || zu[i+1]==0 || zu[i+1]=='/') ){
   900        zu[i] = '_';
   901      }else if( zu[i]=='.' && (zu[i+1]==0 || zu[i+1]=='.'
   902                               || (i>0 && zu[i-1]=='.')) ){
   903        zu[i] = '_';
   904      }
   905    }
   906  }
   907  
   908  /*
   909  ** Quote a filename as a C-style string using \\ and \" if necessary.
   910  ** If quoting is not necessary, just return a copy of the input string.
   911  **
   912  ** The return value is a held in memory obtained from fossil_malloc()
   913  ** and must be freed by the caller.
   914  */
   915  static char *gitmirror_quote_filename_if_needed(const char *zIn){
   916    int i, j;
   917    char c;
   918    int nSpecial = 0;
   919    char *zOut;
   920    for(i=0; (c = zIn[i])!=0; i++){
   921      if( c=='\\' || c=='"' || c=='\n' ){
   922        nSpecial++;
   923      }
   924    }
   925    if( nSpecial==0 ){
   926      return fossil_strdup(zIn);
   927    }
   928    zOut = fossil_malloc( i+nSpecial+3 );
   929    zOut[0] = '"';
   930    for(i=0, j=1; (c = zIn[i])!=0; i++){
   931      if( c=='\\' || c=='"' || c=='\n' ){
   932        zOut[j++] = '\\';
   933        if( c=='\n' ){
   934          zOut[j++] = 'n';
   935        }else{
   936          zOut[j++] = c;
   937        }
   938      }else{
   939        zOut[j++] = c;
   940      }
   941    }
   942    zOut[j++] = '"';
   943    zOut[j] = 0;
   944    return zOut;
   945  }
   946  
   947  /*
   948  ** Find the Git-name corresponding to the Fossil-name zUuid.
   949  **
   950  ** If the mark does not exist and if the bCreate flag is false, then
   951  ** return NULL.  If the mark does not exist and the bCreate flag is true,
   952  ** then create the mark.
   953  **
   954  ** The string returned is obtained from fossil_malloc() and should
   955  ** be freed by the caller.
   956  */
   957  static char *gitmirror_find_mark(const char *zUuid, int isFile, int bCreate){
   958    static Stmt sFind, sIns;
   959    db_static_prepare(&sFind,
   960      "SELECT coalesce(githash,printf(':%%d',id))"
   961      " FROM mirror.mmark WHERE uuid=:uuid AND isfile=:isfile"
   962    );
   963    db_bind_text(&sFind, ":uuid", zUuid);
   964    db_bind_int(&sFind, ":isfile", isFile!=0);
   965    if( db_step(&sFind)==SQLITE_ROW ){
   966      char *zMark = fossil_strdup(db_column_text(&sFind, 0));
   967      db_reset(&sFind);
   968      return zMark;
   969    }
   970    db_reset(&sFind);
   971    if( !bCreate ){
   972      return 0;
   973    }
   974    db_static_prepare(&sIns,
   975      "INSERT INTO mirror.mmark(uuid,isfile) VALUES(:uuid,:isfile)"
   976    );
   977    db_bind_text(&sIns, ":uuid", zUuid);
   978    db_bind_int(&sIns, ":isfile", isFile!=0);
   979    db_step(&sIns);
   980    db_reset(&sIns);
   981    return mprintf(":%d", db_last_insert_rowid());
   982  }
   983  
   984  /* This is the SHA3-256 hash of an empty file */
   985  static const char zEmptySha3[] = 
   986    "a7ffc6f8bf1ed76651c14756a061d662f580ff4de43b49fa82d80a4b80f8434a";
   987  
   988  /*
   989  ** Export a single file named by zUuid.
   990  **
   991  ** Return 0 on success and non-zero on any failure.
   992  **
   993  ** If zUuid is a shunned file, then treat it as if it were any empty file.
   994  ** But files that are missing from the repository but have not been officially
   995  ** shunned cause an error return.  Except, if bPhantomOk is true, then missing
   996  ** files are replaced by an empty file.
   997  */
   998  static int gitmirror_send_file(FILE *xCmd, const char *zUuid, int bPhantomOk){
   999    char *zMark;
  1000    int rid;
  1001    int rc;
  1002    Blob data;
  1003    rid = fast_uuid_to_rid(zUuid);
  1004    if( rid<0 ){
  1005      if( bPhantomOk || uuid_is_shunned(zUuid) ){
  1006        gitmirror_message(VERB_EXTRA, "missing file: %s\n", zUuid);
  1007        zUuid = zEmptySha3;
  1008      }else{
  1009        return 1;
  1010      }
  1011    }else{
  1012      rc = content_get(rid, &data);
  1013      if( rc==0 ){
  1014        if( bPhantomOk ){
  1015          blob_init(&data, 0, 0);
  1016          gitmirror_message(VERB_EXTRA, "missing file: %s\n", zUuid);
  1017          zUuid = zEmptySha3;
  1018        }else{      
  1019          return 1;
  1020        }
  1021      }
  1022    }
  1023    zMark = gitmirror_find_mark(zUuid, 1, 1);
  1024    if( zMark[0]==':' ){
  1025      fprintf(xCmd, "blob\nmark %s\ndata %d\n", zMark, blob_size(&data));
  1026      fwrite(blob_buffer(&data), 1, blob_size(&data), xCmd);
  1027      fprintf(xCmd, "\n");
  1028    }
  1029    fossil_free(zMark);
  1030    blob_reset(&data);
  1031    return 0;
  1032  }
  1033  
  1034  /*
  1035  ** Transfer a check-in over to the mirror.  "rid" is the BLOB.RID for
  1036  ** the check-in to export.
  1037  **
  1038  ** If any ancestor of the check-in has not yet been exported, then
  1039  ** invoke this routine recursively to export the ancestor first.
  1040  ** This can only happen on a timewarp, so deep nesting is unlikely.
  1041  **
  1042  ** Before sending the check-in, first make sure all associated files
  1043  ** have already been exported, and send "blob" records for any that
  1044  ** have not been.  Update the MIRROR.MMARK table so that it holds the
  1045  ** marks for the exported files.
  1046  **
  1047  ** Return zero on success and non-zero if the export should be stopped.
  1048  */
  1049  static int gitmirror_send_checkin(
  1050    FILE *xCmd,           /* Write fast-import text on this pipe */
  1051    int rid,              /* BLOB.RID for the check-in to export */
  1052    const char *zUuid,    /* BLOB.UUID for the check-in to export */
  1053    int *pnLimit,         /* Stop when the counter reaches zero */
  1054    int fManifest         /* MFESTFLG_* values */
  1055  ){
  1056    Manifest *pMan;       /* The check-in to be output */
  1057    int i;                /* Loop counter */
  1058    int iParent;          /* Which immediate ancestor is primary.  -1 for none */
  1059    Stmt q;               /* An SQL query */
  1060    char *zBranch;        /* The branch of the check-in */
  1061    char *zMark;          /* The Git-name of the check-in */
  1062    Blob sql;             /* String of SQL for part of the query */
  1063    Blob comment;         /* The comment text for the check-in */
  1064    int nErr = 0;         /* Number of errors */
  1065    int bPhantomOk;       /* True if phantom files should be ignored */
  1066    char buf[24];
  1067  
  1068    pMan = manifest_get(rid, CFTYPE_MANIFEST, 0);
  1069    if( pMan==0 ){
  1070      /* Must be a phantom.  Return without doing anything, and in particular
  1071      ** without creating a mark for this check-in. */
  1072      gitmirror_message(VERB_NORMAL, "missing check-in: %s\n", zUuid);
  1073      return 0;
  1074    }
  1075  
  1076    /* Check to see if any parent logins have not yet been processed, and
  1077    ** if so, create them */
  1078    for(i=0; i<pMan->nParent; i++){
  1079      char *zPMark = gitmirror_find_mark(pMan->azParent[i], 0, 0);
  1080      if( zPMark==0 ){
  1081        int prid = db_int(0, "SELECT rid FROM blob WHERE uuid=%Q",
  1082                          pMan->azParent[i]);
  1083        int rc = gitmirror_send_checkin(xCmd, prid, pMan->azParent[i],
  1084                                        pnLimit, fManifest);
  1085        if( rc || *pnLimit<=0 ){
  1086          manifest_destroy(pMan);
  1087          return 1;
  1088        }
  1089      }
  1090      fossil_free(zPMark);
  1091    }
  1092  
  1093    /* Ignore phantom files on check-ins that are over one year old */
  1094    bPhantomOk = db_int(0, "SELECT %.6f<julianday('now','-1 year')",
  1095                        pMan->rDate);
  1096  
  1097    /* Make sure all necessary files have been exported */
  1098    db_prepare(&q,
  1099      "SELECT uuid FROM files_of_checkin(%Q)"
  1100      " WHERE uuid NOT IN (SELECT uuid FROM mirror.mmark)",
  1101      zUuid
  1102    );
  1103    while( db_step(&q)==SQLITE_ROW ){
  1104      const char *zFUuid = db_column_text(&q, 0);
  1105      int n = gitmirror_send_file(xCmd, zFUuid, bPhantomOk);
  1106      nErr += n;
  1107      if( n ) gitmirror_message(VERB_ERROR, "missing file: %s\n", zFUuid);
  1108    }
  1109    db_finalize(&q);
  1110  
  1111    /* If some required files could not be exported, abandon the check-in
  1112    ** export */
  1113    if( nErr ){
  1114      gitmirror_message(VERB_ERROR,
  1115               "export of %s abandoned due to missing files\n", zUuid);
  1116      *pnLimit = 0;
  1117      return 1;
  1118    }
  1119  
  1120    /* Figure out which branch this check-in is a member of */
  1121    zBranch = db_text(0,
  1122      "SELECT value FROM tagxref WHERE tagid=%d AND tagtype>0 AND rid=%d",
  1123      TAG_BRANCH, rid
  1124    );
  1125    if( fossil_strcmp(zBranch,"trunk")==0 ){
  1126      fossil_free(zBranch);
  1127      zBranch = mprintf("master");
  1128    }else if( zBranch==0 ){
  1129      zBranch = mprintf("unknown");
  1130    }else{
  1131      gitmirror_sanitize_name(zBranch);
  1132    }
  1133  
  1134    /* Export the check-in */
  1135    fprintf(xCmd, "commit refs/heads/%s\n", zBranch);
  1136    fossil_free(zBranch);
  1137    zMark = gitmirror_find_mark(zUuid,0,1);
  1138    fprintf(xCmd, "mark %s\n", zMark);
  1139    fossil_free(zMark);
  1140    sqlite3_snprintf(sizeof(buf), buf, "%lld",
  1141       (sqlite3_int64)((pMan->rDate-2440587.5)*86400.0)
  1142    );
1143 fprintf(xCmd, "committer %s <%s@noemail.net> %s +0000\n",
1144 pMan->zUser, pMan->zUser, buf 1145 ); 1146 blob_init(&comment, pMan->zComment, -1); 1147 if( blob_size(&comment)==0 ){ 1148 blob_append(&comment, "(no comment)", -1); 1149 } 1150 blob_appendf(&comment, "\n\nFossilOrigin-Name: %s", zUuid); 1151 fprintf(xCmd, "data %d\n%s\n", blob_size(&comment), blob_str(&comment)); 1152 blob_reset(&comment); 1153 iParent = -1; /* Which ancestor is the primary parent */ 1154 for(i=0; i<pMan->nParent; i++){ 1155 char *zOther = gitmirror_find_mark(pMan->azParent[i],0,0); 1156 if( zOther==0 ) continue; 1157 if( iParent<0 ){ 1158 iParent = i; 1159 fprintf(xCmd, "from %s\n", zOther); 1160 }else{ 1161 fprintf(xCmd, "merge %s\n", zOther); 1162 } 1163 fossil_free(zOther); 1164 } 1165 if( iParent>=0 ){ 1166 db_prepare(&q, 1167 "SELECT filename FROM files_of_checkin(%Q)" 1168 " EXCEPT SELECT filename FROM files_of_checkin(%Q)", 1169 pMan->azParent[iParent], zUuid 1170 ); 1171 while( db_step(&q)==SQLITE_ROW ){ 1172 fprintf(xCmd, "D %s\n", db_column_text(&q,0)); 1173 } 1174 db_finalize(&q); 1175 } 1176 blob_init(&sql, 0, 0); 1177 blob_append_sql(&sql, 1178 "SELECT filename, uuid, perm FROM files_of_checkin(%Q)", 1179 zUuid 1180 ); 1181 if( pMan->nParent ){ 1182 blob_append_sql(&sql, 1183 " EXCEPT SELECT filename, uuid, perm FROM files_of_checkin(%Q)", 1184 pMan->azParent[0]); 1185 } 1186 db_prepare(&q, 1187 "SELECT x.filename, x.perm," 1188 " coalesce(mmark.githash,printf(':%%d',mmark.id))" 1189 " FROM (%s) AS x, mirror.mmark" 1190 " WHERE mmark.uuid=x.uuid AND isfile", 1191 blob_sql_text(&sql) 1192 ); 1193 blob_reset(&sql); 1194 while( db_step(&q)==SQLITE_ROW ){ 1195 const char *zFilename = db_column_text(&q,0); 1196 const char *zMode = db_column_text(&q,1); 1197 const char *zMark = db_column_text(&q,2); 1198 const char *zGitMode = "100644"; 1199 char *zFNQuoted = 0; 1200 if( zMode ){ 1201 if( strchr(zMode,'x') ) zGitMode = "100755"; 1202 if( strchr(zMode,'l') ) zGitMode = "120000"; 1203 } 1204 zFNQuoted = gitmirror_quote_filename_if_needed(zFilename); 1205 fprintf(xCmd,"M %s %s %s\n", zGitMode, zMark, zFNQuoted); 1206 fossil_free(zFNQuoted); 1207 } 1208 db_finalize(&q); 1209 1210 /* Include Fossil-generated auxiliary files in the check-in */ 1211 if( fManifest & MFESTFLG_RAW ){ 1212 Blob manifest; 1213 content_get(rid, &manifest); 1214 fprintf(xCmd,"M 100644 inline manifest\ndata %d\n%s\n", 1215 blob_size(&manifest), blob_str(&manifest)); 1216 blob_reset(&manifest); 1217 } 1218 if( fManifest & MFESTFLG_UUID ){ 1219 int n = (int)strlen(zUuid); 1220 fprintf(xCmd,"M 100644 inline manifest.uuid\ndata %d\n%s\n", n, zUuid); 1221 } 1222 if( fManifest & MFESTFLG_TAGS ){ 1223 Blob tagslist; 1224 blob_init(&tagslist, 0, 0); 1225 get_checkin_taglist(rid, &tagslist); 1226 fprintf(xCmd,"M 100644 inline manifest.tags\ndata %d\n%s\n", 1227 blob_size(&tagslist), blob_str(&tagslist)); 1228 blob_reset(&tagslist); 1229 } 1230 1231 /* The check-in is finished, so decrement the counter */ 1232 (*pnLimit)--; 1233 return 0; 1234 } 1235 1236 /* 1237 ** Implementation of the "fossil git export" command. 1238 */ 1239 void gitmirror_export_command(void){ 1240 const char *zLimit; /* Text of the --limit flag */ 1241 int nLimit = 0x7fffffff; /* Numeric value of the --limit flag */ 1242 int nTotal = 0; /* Total number of check-ins to export */ 1243 char *zMirror; /* Name of the mirror */ 1244 char *z; /* Generic string */ 1245 char *zCmd; /* git command to run as a subprocess */ 1246 const char *zDebug = 0; /* Value of the --debug flag */ 1247 const char *zAutoPush = 0; /* Value of the --autopush flag */ 1248 char *zPushUrl; /* URL to sync the mirror to */ 1249 double rEnd; /* time of most recent export */ 1250 int rc; /* Result code */ 1251 int bForce; /* Do the export and sync even if no changes*/ 1252 int bNeedRepack = 0; /* True if we should run repack at the end */ 1253 int fManifest; /* Current "manifest" setting */ 1254 FILE *xCmd; /* Pipe to the "git fast-import" command */ 1255 FILE *pMarks; /* Git mark files */ 1256 Stmt q; /* Queries */ 1257 char zLine[200]; /* One line of a mark file */ 1258 1259 zDebug = find_option("debug",0,1); 1260 db_find_and_open_repository(0, 0); 1261 zLimit = find_option("limit", 0, 1); 1262 if( zLimit ){ 1263 nLimit = (unsigned int)atoi(zLimit); 1264 if( nLimit<=0 ) fossil_fatal("--limit must be positive"); 1265 } 1266 zAutoPush = find_option("autopush",0,1); 1267 bForce = find_option("force","f",0)!=0; 1268 gitmirror_verbosity = VERB_NORMAL; 1269 while( find_option("quiet","q",0)!=0 ){ gitmirror_verbosity--; } 1270 while( find_option("verbose","v",0)!=0 ){ gitmirror_verbosity++; } 1271 verify_all_options(); 1272 if( g.argc!=4 && g.argc!=3 ){ usage("export ?MIRROR?"); } 1273 if( g.argc==4 ){ 1274 Blob mirror; 1275 file_canonical_name(g.argv[3], &mirror, 0); 1276 db_set("last-git-export-repo", blob_str(&mirror), 0); 1277 blob_reset(&mirror); 1278 } 1279 zMirror = db_get("last-git-export-repo", 0); 1280 if( zMirror==0 ){ 1281 fossil_fatal("no Git repository specified"); 1282 } 1283 1284 /* Make sure the GIT repository directory exists */ 1285 rc = file_mkdir(zMirror, ExtFILE, 0); 1286 if( rc ) fossil_fatal("cannot create directory \"%s\"", zMirror); 1287 1288 /* Make sure GIT has been initialized */ 1289 z = mprintf("%s/.git", zMirror); 1290 if( !file_isdir(z, ExtFILE) ){ 1291 zCmd = mprintf("git init \"%s\"",zMirror); 1292 gitmirror_message(VERB_NORMAL, "%s\n", zCmd); 1293 rc = fossil_system(zCmd); 1294 if( rc ){ 1295 fossil_fatal("cannot initialize the git repository using: \"%s\"", zCmd); 1296 } 1297 fossil_free(zCmd); 1298 bNeedRepack = 1; 1299 } 1300 fossil_free(z); 1301 1302 /* Make sure the .mirror_state subdirectory exists */ 1303 z = mprintf("%s/.mirror_state", zMirror); 1304 rc = file_mkdir(z, ExtFILE, 0); 1305 if( rc ) fossil_fatal("cannot create directory \"%s\"", z); 1306 fossil_free(z); 1307 1308 /* Attach the .mirror_state/db database */ 1309 db_multi_exec("ATTACH '%q/.mirror_state/db' AS mirror;", zMirror); 1310 db_begin_write(); 1311 db_multi_exec( 1312 "CREATE TABLE IF NOT EXISTS mirror.mconfig(\n" 1313 " key TEXT PRIMARY KEY,\n" 1314 " Value ANY\n" 1315 ") WITHOUT ROWID;\n" 1316 "CREATE TABLE IF NOT EXISTS mirror.mmark(\n" 1317 " id INTEGER PRIMARY KEY,\n" 1318 " uuid TEXT,\n" 1319 " isfile BOOLEAN,\n" 1320 " githash TEXT,\n" 1321 " UNIQUE(uuid,isfile)\n" 1322 ");" 1323 ); 1324 if( !db_table_has_column("mirror","mmark","isfile") ){ 1325 db_multi_exec( 1326 "ALTER TABLE mirror.mmark RENAME TO mmark_old;" 1327 "CREATE TABLE IF NOT EXISTS mirror.mmark(\n" 1328 " id INTEGER PRIMARY KEY,\n" 1329 " uuid TEXT,\n" 1330 " isfile BOOLEAN,\n" 1331 " githash TEXT,\n" 1332 " UNIQUE(uuid,isfile)\n" 1333 ");" 1334 "INSERT OR IGNORE INTO mirror.mmark(id,uuid,githash,isfile)" 1335 " SELECT id,uuid,githash," 1336 " NOT EXISTS(SELECT 1 FROM repository.event, repository.blob" 1337 " WHERE event.objid=blob.rid" 1338 " AND blob.uuid=mmark_old.uuid)" 1339 " FROM mirror.mmark_old;\n" 1340 "DROP TABLE mirror.mmark_old;\n" 1341 ); 1342 } 1343 1344 /* Change the autopush setting if the --autopush flag is present */ 1345 if( zAutoPush ){ 1346 if( is_false(zAutoPush) ){ 1347 db_multi_exec("DELETE FROM mirror.mconfig WHERE key='autopush'"); 1348 }else{ 1349 db_multi_exec( 1350 "REPLACE INTO mirror.mconfig(key,value)" 1351 "VALUES('autopush',%Q)", 1352 zAutoPush 1353 ); 1354 } 1355 } 1356 1357 /* See if there is any work to be done. Exit early if not, before starting 1358 ** the "git fast-import" command. */ 1359 if( !bForce 1360 && !db_exists("SELECT 1 FROM event WHERE type IN ('ci','t')" 1361 " AND mtime>coalesce((SELECT value FROM mconfig" 1362 " WHERE key='start'),0.0)") 1363 ){ 1364 gitmirror_message(VERB_NORMAL, "no changes\n"); 1365 db_commit_transaction(); 1366 return; 1367 } 1368 1369 /* Do we need to include manifest files in the clone? */ 1370 fManifest = db_get_manifest_setting(); 1371 1372 /* Change to the MIRROR directory so that the Git commands will work */ 1373 rc = file_chdir(zMirror, 0); 1374 if( rc ) fossil_fatal("cannot change the working directory to \"%s\"", 1375 zMirror); 1376 1377 /* Start up the git fast-import command */ 1378 if( zDebug ){ 1379 if( fossil_strcmp(zDebug,"stdout")==0 ){ 1380 xCmd = stdout; 1381 }else{ 1382 xCmd = fopen(zDebug, "wb"); 1383 if( xCmd==0 ) fossil_fatal("cannot open file \"%s\" for writing", zDebug); 1384 } 1385 }else{ 1386 zCmd = mprintf("git fast-import" 1387 " --export-marks=.mirror_state/marks.txt" 1388 " --quiet --done"); 1389 gitmirror_message(VERB_NORMAL, "%s\n", zCmd); 1390 #ifdef _WIN32 1391 xCmd = popen(zCmd, "wb"); 1392 #else 1393 xCmd = popen(zCmd, "w"); 1394 #endif 1395 if( zCmd==0 ){ 1396 fossil_fatal("cannot start the \"git fast-import\" command"); 1397 } 1398 fossil_free(zCmd); 1399 } 1400 1401 /* Run the export */ 1402 rEnd = 0.0; 1403 db_multi_exec( 1404 "CREATE TEMP TABLE tomirror(objid,mtime,uuid);\n" 1405 "INSERT INTO tomirror " 1406 "SELECT objid, mtime, blob.uuid FROM event, blob\n" 1407 " WHERE type='ci'" 1408 " AND mtime>coalesce((SELECT value FROM mconfig WHERE key='start'),0.0)" 1409 " AND blob.rid=event.objid" 1410 " AND blob.uuid NOT IN (SELECT uuid FROM mirror.mmark WHERE NOT isfile);" 1411 ); 1412 nTotal = db_int(0, "SELECT count(*) FROM tomirror"); 1413 if( nLimit<nTotal ){ 1414 nTotal = nLimit; 1415 }else if( nLimit>nTotal ){ 1416 nLimit = nTotal; 1417 } 1418 db_prepare(&q, 1419 "SELECT objid, mtime, uuid FROM tomirror ORDER BY mtime" 1420 ); 1421 while( nLimit && db_step(&q)==SQLITE_ROW ){ 1422 int rid = db_column_int(&q, 0); 1423 double rMTime = db_column_double(&q, 1); 1424 const char *zUuid = db_column_text(&q, 2); 1425 if( rMTime>rEnd ) rEnd = rMTime; 1426 rc = gitmirror_send_checkin(xCmd, rid, zUuid, &nLimit, fManifest); 1427 if( rc ) break; 1428 gitmirror_message(VERB_NORMAL,"%d/%d \r", nTotal-nLimit, nTotal); 1429 fflush(stdout); 1430 } 1431 db_finalize(&q); 1432 fprintf(xCmd, "done\n"); 1433 if( zDebug ){ 1434 if( xCmd!=stdout ) fclose(xCmd); 1435 }else{ 1436 pclose(xCmd); 1437 } 1438 gitmirror_message(VERB_NORMAL, "%d check-ins added to the %s\n", 1439 nTotal-nLimit, zMirror); 1440 1441 /* Read the export-marks file. Transfer the new marks over into 1442 ** the import-marks file. 1443 */ 1444 pMarks = fopen(".mirror_state/marks.txt", "rb"); 1445 if( pMarks ){ 1446 db_prepare(&q, "UPDATE mirror.mmark SET githash=:githash WHERE id=:id"); 1447 while( fgets(zLine, sizeof(zLine), pMarks) ){ 1448 int j, k; 1449 if( zLine[0]!=':' ) continue; 1450 db_bind_int(&q, ":id", atoi(zLine+1)); 1451 for(j=1; zLine[j] && zLine[j]!=' '; j++){} 1452 if( zLine[j]!=' ' ) continue; 1453 j++; 1454 if( zLine[j]==0 ) continue; 1455 for(k=j; fossil_isalnum(zLine[k]); k++){} 1456 zLine[k] = 0; 1457 db_bind_text(&q, ":githash", &zLine[j]); 1458 db_step(&q); 1459 db_reset(&q); 1460 } 1461 db_finalize(&q); 1462 fclose(pMarks); 1463 file_delete(".mirror_state/marks.txt"); 1464 }else{ 1465 fossil_fatal("git fast-import didn't generate a marks file!"); 1466 } 1467 db_multi_exec( 1468 "CREATE INDEX IF NOT EXISTS mirror.mmarkx1 ON mmark(githash);" 1469 ); 1470 1471 /* Do any tags that have been created since the start time */ 1472 db_prepare(&q, 1473 "SELECT substr(tagname,5), githash" 1474 " FROM (SELECT tagxref.tagid AS xtagid, tagname, rid, max(mtime) AS mtime" 1475 " FROM tagxref JOIN tag ON tag.tagid=tagxref.tagid" 1476 " WHERE tag.tagname GLOB 'sym-*'" 1477 " AND tagxref.tagtype=1" 1478 " AND tagxref.mtime > coalesce((SELECT value FROM mconfig" 1479 " WHERE key='start'),0.0)" 1480 " GROUP BY tagxref.tagid) AS tx" 1481 " JOIN blob ON tx.rid=blob.rid" 1482 " JOIN mmark ON mmark.uuid=blob.uuid;" 1483 ); 1484 while( db_step(&q)==SQLITE_ROW ){ 1485 char *zTagname = fossil_strdup(db_column_text(&q,0)); 1486 const char *zObj = db_column_text(&q,1); 1487 char *zTagCmd; 1488 gitmirror_sanitize_name(zTagname); 1489 zTagCmd = mprintf("git tag -f \"%s\" %s", zTagname, zObj); 1490 fossil_free(zTagname); 1491 gitmirror_message(VERB_NORMAL, "%s\n", zTagCmd); 1492 fossil_system(zTagCmd); 1493 fossil_free(zTagCmd); 1494 } 1495 db_finalize(&q); 1496 1497 /* Update all references that might have changed since the start time */ 1498 db_prepare(&q, 1499 "SELECT" 1500 " tagxref.value AS name," 1501 " max(event.mtime) AS mtime," 1502 " mmark.githash AS gitckin" 1503 " FROM tagxref, tag, event, blob, mmark" 1504 " WHERE tagxref.tagid=tag.tagid" 1505 " AND tagxref.tagtype>0" 1506 " AND tag.tagname='branch'" 1507 " AND event.objid=tagxref.rid" 1508 " AND event.mtime > coalesce((SELECT value FROM mconfig" 1509 " WHERE key='start'),0.0)" 1510 " AND blob.rid=tagxref.rid" 1511 " AND mmark.uuid=blob.uuid" 1512 " GROUP BY 1" 1513 ); 1514 while( db_step(&q)==SQLITE_ROW ){ 1515 char *zBrname = fossil_strdup(db_column_text(&q,0)); 1516 const char *zObj = db_column_text(&q,2); 1517 char *zRefCmd; 1518 if( fossil_strcmp(zBrname,"trunk")==0 ){ 1519 fossil_free(zBrname); 1520 zBrname = fossil_strdup("master"); 1521 }else{ 1522 gitmirror_sanitize_name(zBrname); 1523 } 1524 zRefCmd = mprintf("git update-ref \"refs/heads/%s\" %s", zBrname, zObj); 1525 fossil_free(zBrname); 1526 gitmirror_message(VERB_NORMAL, "%s\n", zRefCmd); 1527 fossil_system(zRefCmd); 1528 fossil_free(zRefCmd); 1529 } 1530 db_finalize(&q); 1531 1532 /* Update the start time */ 1533 if( rEnd>0.0 ){ 1534 db_prepare(&q, "REPLACE INTO mirror.mconfig(key,value) VALUES('start',:x)"); 1535 db_bind_double(&q, ":x", rEnd); 1536 db_step(&q); 1537 db_finalize(&q); 1538 } 1539 db_commit_transaction(); 1540 1541 /* Maybe run a git repack */ 1542 if( bNeedRepack ){ 1543 const char *zRepack = "git repack -adf"; 1544 gitmirror_message(VERB_NORMAL, "%s\n", zRepack); 1545 fossil_system(zRepack); 1546 } 1547 1548 /* Optionally do a "git push" */ 1549 zPushUrl = db_text(0, "SELECT value FROM mconfig WHERE key='autopush'"); 1550 if( zPushUrl ){ 1551 char *zPushCmd; 1552 UrlData url; 1553 if( sqlite3_strglob("http*", zPushUrl)==0 ){ 1554 url_parse_local(zPushUrl, 0, &url); 1555 zPushCmd = mprintf("git push --mirror %s", url.canonical); 1556 }else{ 1557 zPushCmd = mprintf("git push --mirror %s", zPushUrl); 1558 } 1559 gitmirror_message(VERB_NORMAL, "%s\n", zPushCmd); 1560 fossil_free(zPushCmd); 1561 zPushCmd = mprintf("git push --mirror %s", zPushUrl); 1562 fossil_system(zPushCmd); 1563 fossil_free(zPushCmd); 1564 } 1565 } 1566 1567 /* 1568 ** Implementation of the "fossil git status" command. 1569 ** 1570 ** Show the status of a "git export". 1571 */ 1572 void gitmirror_status_command(void){ 1573 char *zMirror; 1574 char *z; 1575 int n, k; 1576 db_find_and_open_repository(0, 0); 1577 verify_all_options(); 1578 zMirror = db_get("last-git-export-repo", 0); 1579 if( zMirror==0 ){ 1580 fossil_print("Git mirror: none\n"); 1581 return; 1582 } 1583 fossil_print("Git mirror: %s\n", zMirror); 1584 db_multi_exec("ATTACH '%q/.mirror_state/db' AS mirror;", zMirror); 1585 z = db_text(0, "SELECT datetime(value) FROM mconfig WHERE key='start'"); 1586 if( z ){ 1587 double rAge = db_double(0.0, "SELECT julianday('now') - value" 1588 " FROM mconfig WHERE key='start'"); 1589 if( rAge>1.0/86400.0 ){ 1590 fossil_print("Last export: %s (%z ago)\n", z, human_readable_age(rAge)); 1591 }else{ 1592 fossil_print("Last export: %s (moments ago)\n", z); 1593 } 1594 } 1595 z = db_text(0, "SELECT value FROM mconfig WHERE key='autopush'"); 1596 if( z==0 ){ 1597 fossil_print("Autopush: off\n"); 1598 }else{ 1599 UrlData url; 1600 url_parse_local(z, 0, &url); 1601 fossil_print("Autopush: %s\n", url.canonical); 1602 } 1603 n = db_int(0, 1604 "SELECT count(*) FROM event" 1605 " WHERE type='ci'" 1606 " AND mtime>coalesce((SELECT value FROM mconfig" 1607 " WHERE key='start'),0.0)" 1608 ); 1609 if( n==0 ){ 1610 fossil_print("Status: up-to-date\n"); 1611 }else{ 1612 fossil_print("Status: %d check-in%s awaiting export\n", 1613 n, n==1 ? "" : "s"); 1614 } 1615 n = db_int(0, "SELECT count(*) FROM mmark WHERE isfile"); 1616 k = db_int(0, "SELECT count(*) FROm mmark WHERE NOT isfile"); 1617 fossil_print("Exported: %d check-ins and %d file blobs\n", k, n); 1618 } 1619 1620 /* 1621 ** COMMAND: git 1622 ** 1623 ** Usage: %fossil git SUBCOMMAND 1624 ** 1625 ** Do incremental import or export operations between Fossil and Git. 1626 ** Subcommands: 1627 ** 1628 ** fossil git export [MIRROR] [OPTIONS] 1629 ** 1630 ** Write content from the Fossil repository into the Git repository 1631 ** in directory MIRROR. The Git repository is created if it does not 1632 ** already exist. If the Git repository does already exist, then 1633 ** new content added to fossil since the previous export is appended. 1634 ** 1635 ** Repeat this command whenever new checkins are added to the Fossil 1636 ** repository in order to reflect those changes into the mirror. If 1637 ** the MIRROR option is omitted, the repository from the previous 1638 ** invocation is used. 1639 ** 1640 ** The MIRROR directory will contain a subdirectory named 1641 ** ".mirror_state" that contains information that Fossil needs to 1642 ** do incremental exports. Do not attempt to manage or edit the files 1643 ** in that directory since doing so can disrupt future incremental 1644 ** exports. 1645 ** 1646 ** Options: 1647 ** --autopush URL Automatically do a 'git push' to URL. The 1648 ** URL is remembered and used on subsequent exports 1649 ** to the same repository. Or if URL is "off" the 1650 ** auto-push mechanism is disabled 1651 ** --debug FILE Write fast-export text to FILE rather than 1652 ** piping it into "git fast-import". 1653 ** --force|-f Do the export even if nothing has changed 1654 ** --limit N Add no more than N new check-ins to MIRROR. 1655 ** Useful for debugging 1656 ** --quiet|-q Reduce output. Repeat for even less output. 1657 ** --verbose|-v More output. 1658 ** 1659 ** fossil git import MIRROR 1660 ** 1661 ** TBD... 1662 ** 1663 ** fossil git status 1664 ** 1665 ** Show the status of the current Git mirror, if there is one. 1666 */ 1667 void gitmirror_command(void){ 1668 char *zCmd; 1669 int nCmd; 1670 if( g.argc<3 ){ 1671 usage("export ARGS..."); 1672 } 1673 zCmd = g.argv[2]; 1674 nCmd = (int)strlen(zCmd); 1675 if( nCmd>2 && strncmp(zCmd,"export",nCmd)==0 ){ 1676 gitmirror_export_command(); 1677 }else 1678 if( nCmd>2 && strncmp(zCmd,"import",nCmd)==0 ){ 1679 fossil_fatal("not yet implemented - check back later"); 1680 }else 1681 if( nCmd>2 && strncmp(zCmd,"status",nCmd)==0 ){ 1682 gitmirror_status_command(); 1683 }else 1684 { 1685 fossil_fatal("unknown subcommand \"%s\": should be one of " 1686 "\"export\", \"import\", \"status\"", 1687 zCmd); 1688 } 1689 }