Fossil

Artifact [78cf66b7]
Login

Artifact 78cf66b757b72740af39f2ae0313db500daa80ed8af817a59fb2222d131a7e49:


     1  /*
     2  ** Copyright (c) 2010 D. Richard Hipp
     3  **
     4  ** This program is free software; you can redistribute it and/or
     5  ** modify it under the terms of the Simplified BSD License (also
     6  ** known as the "2-Clause License" or "FreeBSD License".)
     7  
     8  ** This program is distributed in the hope that it will be useful,
     9  ** but without any warranty; without even the implied warranty of
    10  ** merchantability or fitness for a particular purpose.
    11  **
    12  ** Author contact information:
    13  **   drh@sqlite.org
    14  **
    15  *******************************************************************************
    16  **
    17  ** This file contains code used to export the content of a Fossil
    18  ** repository in the git-fast-import format.
    19  */
    20  #include "config.h"
    21  #include "export.h"
    22  #include <assert.h>
    23  
    24  /*
    25  ** State information common to all export types.
    26  */
    27  static struct {
    28    const char *zTrunkName;     /* Name of trunk branch */
    29  } gexport;
    30  
    31  #if INTERFACE
    32  /*
    33  ** Each line in a git-fast-export "marK" file is an instance of
    34  ** this object.
    35  */
    36  struct mark_t {
    37    char *name;       /* Name of the mark.  Also starts with ":" */
    38    int rid;          /* Corresponding object in the BLOB table */
    39    char uuid[65];    /* The GIT hash name for this object */
    40  };
    41  #endif
    42  
    43  #if defined(_WIN32) || defined(WIN32)
    44  # undef popen
    45  # define popen _popen
    46  # undef pclose
    47  # define pclose _pclose
    48  #endif
    49  
    50  /*
    51  ** Output a "committer" record for the given user.
    52  ** NOTE: the given user name may be an email itself.
    53  */
54 static void print_person(const char *zUser){ 55 static Stmt q; 56 const char *zContact; 57 char *zName; 58 char *zEmail; 59 int i, j; 60 int isBracketed, atEmailFirst, atEmailLast; 61 62 if( zUser==0 ){ 63 printf(" <unknown>"); 64 return; 65 } 66 db_static_prepare(&q, "SELECT info FROM user WHERE login=:user"); 67 db_bind_text(&q, ":user", zUser); 68 if( db_step(&q)!=SQLITE_ROW ){ 69 db_reset(&q); 70 zName = mprintf("%s", zUser); 71 for(i=j=0; zName[i]; i++){ 72 if( zName[i]!='<' && zName[i]!='>' && zName[i]!='"' ){ 73 zName[j++] = zName[i]; 74 } 75 } 76 zName[j] = 0; 77 printf(" %s <%s>", zName, zName); 78 free(zName); 79 return; 80 } 81 82 /* 83 ** We have contact information. 84 ** It may or may not contain an email address. 85 ** 86 ** ASSUME: 87 ** - General case:"Name Unicoded" <email@address.com> other info 88 ** - If contact information contains more than an email address, 89 ** then the email address is enclosed between <> 90 ** - When only email address is specified, then it's stored verbatim 91 ** - When name part is absent or all-blanks, use zUser instead 92 */ 93 zName = NULL; 94 zEmail = NULL; 95 zContact = db_column_text(&q, 0); 96 atEmailFirst = -1; 97 atEmailLast = -1; 98 isBracketed = 0; 99 for(i=0; zContact[i] && zContact[i]!='@'; i++){ 100 if( zContact[i]=='<' ){ 101 isBracketed = 1; 102 atEmailFirst = i+1; 103 } 104 else if( zContact[i]=='>' ){ 105 isBracketed = 0; 106 atEmailFirst = i+1; 107 } 108 else if( zContact[i]==' ' && !isBracketed ){ 109 atEmailFirst = i+1; 110 } 111 } 112 if( zContact[i]==0 ){ 113 /* No email address found. Take as user info if not empty */ 114 zName = mprintf("%s", zContact[0] ? zContact : zUser); 115 for(i=j=0; zName[i]; i++){ 116 if( zName[i]!='<' && zName[i]!='>' && zName[i]!='"' ){ 117 zName[j++] = zName[i]; 118 } 119 } 120 zName[j] = 0; 121 122 printf(" %s <%s>", zName, zName); 123 free(zName); 124 db_reset(&q); 125 return; 126 } 127 for(j=i+1; zContact[j] && zContact[j]!=' '; j++){ 128 if( zContact[j]=='>' ) 129 atEmailLast = j-1; 130 } 131 if ( atEmailLast==-1 ) atEmailLast = j-1; 132 if ( atEmailFirst==-1 ) atEmailFirst = 0; /* Found only email */ 133 134 /* 135 ** Found beginning and end of email address. 136 ** Extract the address (trimmed and sanitized). 137 */ 138 for(j=atEmailFirst; zContact[j] && zContact[j]==' '; j++){} 139 zEmail = mprintf("%.*s", atEmailLast-j+1, &zContact[j]); 140 141 for(i=j=0; zEmail[i]; i++){ 142 if( zEmail[i]!='<' && zEmail[i]!='>' ){ 143 zEmail[j++] = zEmail[i]; 144 } 145 } 146 zEmail[j] = 0; 147 148 /* 149 ** When bracketed email, extract the string _before_ 150 ** email as user name (may be enquoted). 151 ** If missing or all-blank name, use zUser. 152 */ 153 if( isBracketed && (atEmailFirst-1) > 0){ 154 for(i=atEmailFirst-2; i>=0 && zContact[i] && zContact[i]==' '; i--){} 155 if( i>=0 ){ 156 for(j=0; j<i && zContact[j] && zContact[j]==' '; j++){} 157 zName = mprintf("%.*s", i-j+1, &zContact[j]); 158 } 159 } 160 161 if( zName==NULL ) zName = mprintf("%s", zUser); 162 for(i=j=0; zName[i]; i++){ 163 if( zName[i]!='<' && zName[i]!='>' && zName[i]!='"' ){ 164 zName[j++] = zName[i]; 165 } 166 } 167 zName[j] = 0; 168 169 printf(" %s <%s>", zName, zEmail); 170 free(zName); 171 free(zEmail); 172 db_reset(&q); 173 }
174 175 #define REFREPLACEMENT '_' 176 177 /* 178 ** Output a sanitized git named reference. 179 ** https://git-scm.com/docs/git-check-ref-format 180 ** This implementation assumes we are only printing 181 ** the branch or tag part of the reference. 182 */ 183 static void print_ref(const char *zRef){ 184 char *zEncoded = mprintf("%s", zRef); 185 int i, w; 186 if (zEncoded[0]=='@' && zEncoded[1]=='\0'){ 187 putchar(REFREPLACEMENT); 188 return; 189 } 190 for(i=0, w=0; zEncoded[i]; i++, w++){ 191 if( i!=0 ){ /* Two letter tests */ 192 if( (zEncoded[i-1]=='.' && zEncoded[i]=='.') || 193 (zEncoded[i-1]=='@' && zEncoded[i]=='{') ){ 194 zEncoded[w]=zEncoded[w-1]=REFREPLACEMENT; 195 continue; 196 } 197 if( zEncoded[i-1]=='/' && zEncoded[i]=='/' ){ 198 w--; /* Normalise to a single / by rolling back w */ 199 continue; 200 } 201 } 202 /* No control characters */ 203 if( (unsigned)zEncoded[i]<0x20 || zEncoded[i]==0x7f ){ 204 zEncoded[w]=REFREPLACEMENT; 205 continue; 206 } 207 switch( zEncoded[i] ){ 208 case ' ': 209 case '^': 210 case ':': 211 case '?': 212 case '*': 213 case '[': 214 case '\\': 215 zEncoded[w]=REFREPLACEMENT; 216 break; 217 } 218 } 219 /* Cannot begin with a . or / */ 220 if( zEncoded[0]=='.' || zEncoded[0] == '/' ) zEncoded[0]=REFREPLACEMENT; 221 if( i>0 ){ 222 i--; w--; 223 /* Or end with a . or / */ 224 if( zEncoded[i]=='.' || zEncoded[i] == '/' ) zEncoded[w]=REFREPLACEMENT; 225 /* Cannot end with .lock */ 226 if ( i>4 && strcmp((zEncoded+i)-5, ".lock")==0 ) 227 memset((zEncoded+w)-5, REFREPLACEMENT, 5); 228 } 229 printf("%s", zEncoded); 230 free(zEncoded); 231 } 232 233 #define BLOBMARK(rid) ((rid) * 2) 234 #define COMMITMARK(rid) ((rid) * 2 + 1) 235 236 /* 237 ** insert_commit_xref() 238 ** Insert a new (mark,rid,uuid) entry into the 'xmark' table. 239 ** zName and zUuid must be non-null and must point to NULL-terminated strings. 240 */ 241 void insert_commit_xref(int rid, const char *zName, const char *zUuid){ 242 db_multi_exec( 243 "INSERT OR IGNORE INTO xmark(tname, trid, tuuid)" 244 "VALUES(%Q,%d,%Q)", 245 zName, rid, zUuid 246 ); 247 } 248 249 /* 250 ** create_mark() 251 ** Create a new (mark,rid,uuid) entry for the given rid in the 'xmark' table, 252 ** and return that information as a struct mark_t in *mark. 253 ** *unused_mark is a value representing a mark that is free for use--that is, 254 ** it does not appear in the marks file, and has not been used during this 255 ** export run. Specifically, it is the supremum of the set of used marks 256 ** plus one. 257 ** This function returns -1 in the case where 'rid' does not exist, otherwise 258 ** it returns 0. 259 ** mark->name is dynamically allocated and is owned by the caller upon return. 260 */ 261 int create_mark(int rid, struct mark_t *mark, unsigned int *unused_mark){ 262 char sid[13]; 263 char *zUuid = rid_to_uuid(rid); 264 if( !zUuid ){ 265 fossil_trace("Undefined rid=%d\n", rid); 266 return -1; 267 } 268 mark->rid = rid; 269 sqlite3_snprintf(sizeof(sid), sid, ":%d", *unused_mark); 270 *unused_mark += 1; 271 mark->name = fossil_strdup(sid); 272 sqlite3_snprintf(sizeof(mark->uuid), mark->uuid, "%s", zUuid); 273 free(zUuid); 274 insert_commit_xref(mark->rid, mark->name, mark->uuid); 275 return 0; 276 } 277 278 /* 279 ** mark_name_from_rid() 280 ** Find the mark associated with the given rid. Mark names always start 281 ** with ':', and are pulled from the 'xmark' temporary table. 282 ** If the given rid doesn't have a mark associated with it yet, one is 283 ** created with a value of *unused_mark. 284 ** *unused_mark functions exactly as in create_mark(). 285 ** This function returns NULL if the rid does not have an associated UUID, 286 ** (i.e. is not valid). Otherwise, it returns the name of the mark, which is 287 ** dynamically allocated and is owned by the caller of this function. 288 */ 289 char * mark_name_from_rid(int rid, unsigned int *unused_mark){ 290 char *zMark = db_text(0, "SELECT tname FROM xmark WHERE trid=%d", rid); 291 if( zMark==NULL ){ 292 struct mark_t mark; 293 if( create_mark(rid, &mark, unused_mark)==0 ){ 294 zMark = mark.name; 295 }else{ 296 return NULL; 297 } 298 } 299 return zMark; 300 } 301 302 /* 303 ** Parse a single line of the mark file. Store the result in the mark object. 304 ** 305 ** "line" is a single line of input. 306 ** This function returns -1 in the case that the line is blank, malformed, or 307 ** the rid/uuid named in 'line' does not match what is in the repository 308 ** database. Otherwise, 0 is returned. 309 ** 310 ** mark->name is dynamically allocated, and owned by the caller. 311 */ 312 int parse_mark(char *line, struct mark_t *mark){ 313 char *cur_tok; 314 char type_; 315 cur_tok = strtok(line, " \t"); 316 if( !cur_tok || strlen(cur_tok)<2 ){ 317 return -1; 318 } 319 mark->rid = atoi(&cur_tok[1]); 320 type_ = cur_tok[0]; 321 if( type_!='c' && type_!='b' ){ 322 /* This is probably a blob mark */ 323 mark->name = NULL; 324 return 0; 325 } 326 327 cur_tok = strtok(NULL, " \t"); 328 if( !cur_tok ){ 329 /* This mark was generated by an older version of Fossil and doesn't 330 ** include the mark name and uuid. create_mark() will name the new mark 331 ** exactly as it was when exported to git, so that we should have a 332 ** valid mapping from git hash<->mark name<->fossil hash. */ 333 unsigned int mid; 334 if( type_=='c' ){ 335 mid = COMMITMARK(mark->rid); 336 } 337 else{ 338 mid = BLOBMARK(mark->rid); 339 } 340 return create_mark(mark->rid, mark, &mid); 341 }else{ 342 mark->name = fossil_strdup(cur_tok); 343 } 344 345 cur_tok = strtok(NULL, "\n"); 346 if( !cur_tok || (strlen(cur_tok)!=40 && strlen(cur_tok)!=64) ){ 347 free(mark->name); 348 fossil_trace("Invalid SHA-1/SHA-3 in marks file: %s\n", cur_tok); 349 return -1; 350 }else{ 351 sqlite3_snprintf(sizeof(mark->uuid), mark->uuid, "%s", cur_tok); 352 } 353 354 /* make sure that rid corresponds to UUID */ 355 if( fast_uuid_to_rid(mark->uuid)!=mark->rid ){ 356 free(mark->name); 357 fossil_trace("Non-existent SHA-1/SHA-3 in marks file: %s\n", mark->uuid); 358 return -1; 359 } 360 361 /* insert a cross-ref into the 'xmark' table */ 362 insert_commit_xref(mark->rid, mark->name, mark->uuid); 363 return 0; 364 } 365 366 /* 367 ** Import the marks specified in file 'f'; 368 ** If 'blobs' is non-null, insert all blob marks into it. 369 ** If 'vers' is non-null, insert all commit marks into it. 370 ** If 'unused_marks' is non-null, upon return of this function, all values 371 ** x >= *unused_marks are free to use as marks, i.e. they do not clash with 372 ** any marks appearing in the marks file. 373 ** 374 ** Each line in the file must be at most 100 characters in length. This 375 ** seems like a reasonable maximum for a 40-character uuid, and 1-13 376 ** character rid. 377 ** 378 ** The function returns -1 if any of the lines in file 'f' are malformed, 379 ** or the rid/uuid information doesn't match what is in the repository 380 ** database. Otherwise, 0 is returned. 381 */ 382 int import_marks(FILE* f, Bag *blobs, Bag *vers, unsigned int *unused_mark){ 383 char line[101]; 384 while(fgets(line, sizeof(line), f)){ 385 struct mark_t mark; 386 if( strlen(line)==100 && line[99]!='\n' ){ 387 /* line too long */ 388 return -1; 389 } 390 if( parse_mark(line, &mark)<0 ){ 391 return -1; 392 }else if( line[0]=='b' ){ 393 if( blobs!=NULL ){ 394 bag_insert(blobs, mark.rid); 395 } 396 }else{ 397 if( vers!=NULL ){ 398 bag_insert(vers, mark.rid); 399 } 400 } 401 if( unused_mark!=NULL ){ 402 unsigned int mid = atoi(mark.name + 1); 403 if( mid>=*unused_mark ){ 404 *unused_mark = mid + 1; 405 } 406 } 407 free(mark.name); 408 } 409 return 0; 410 } 411 412 void export_mark(FILE* f, int rid, char obj_type) 413 { 414 unsigned int z = 0; 415 char *zUuid = rid_to_uuid(rid); 416 char *zMark; 417 if( zUuid==NULL ){ 418 fossil_trace("No uuid matching rid=%d when exporting marks\n", rid); 419 return; 420 } 421 /* Since rid is already in the 'xmark' table, the value of z won't be 422 ** used, but pass in a valid pointer just to be safe. */ 423 zMark = mark_name_from_rid(rid, &z); 424 fprintf(f, "%c%d %s %s\n", obj_type, rid, zMark, zUuid); 425 free(zMark); 426 free(zUuid); 427 } 428 429 /* 430 ** If 'blobs' is non-null, it must point to a Bag of blob rids to be 431 ** written to disk. Blob rids are written as 'b<rid>'. 432 ** If 'vers' is non-null, it must point to a Bag of commit rids to be 433 ** written to disk. Commit rids are written as 'c<rid> :<mark> <uuid>'. 434 ** All commit (mark,rid,uuid) tuples are stored in 'xmark' table. 435 ** This function does not fail, but may produce errors if a uuid cannot 436 ** be found for an rid in 'vers'. 437 */ 438 void export_marks(FILE* f, Bag *blobs, Bag *vers){ 439 int rid; 440 441 if( blobs!=NULL ){ 442 rid = bag_first(blobs); 443 if( rid!=0 ){ 444 do{ 445 export_mark(f, rid, 'b'); 446 }while( (rid = bag_next(blobs, rid))!=0 ); 447 } 448 } 449 if( vers!=NULL ){ 450 rid = bag_first(vers); 451 if( rid!=0 ){ 452 do{ 453 export_mark(f, rid, 'c'); 454 }while( (rid = bag_next(vers, rid))!=0 ); 455 } 456 } 457 } 458 459 /* This is the original header command (and hence documentation) for 460 ** the "fossil export" command: 461 ** 462 ** Usage: %fossil export --git ?OPTIONS? ?REPOSITORY? 463 ** 464 ** Write an export of all check-ins to standard output. The export is 465 ** written in the git-fast-export file format assuming the --git option is 466 ** provided. The git-fast-export format is currently the only VCS 467 ** interchange format supported, though other formats may be added in 468 ** the future. 469 ** 470 ** Run this command within a checkout. Or use the -R or --repository 471 ** option to specify a Fossil repository to be exported. 472 ** 473 ** Only check-ins are exported using --git. Git does not support tickets 474 ** or wiki or tech notes or attachments, so none of those are exported. 475 ** 476 ** If the "--import-marks FILE" option is used, it contains a list of 477 ** rids to skip. 478 ** 479 ** If the "--export-marks FILE" option is used, the rid of all commits and 480 ** blobs written on exit for use with "--import-marks" on the next run. 481 ** 482 ** Options: 483 ** --export-marks FILE export rids of exported data to FILE 484 ** --import-marks FILE read rids of data to ignore from FILE 485 ** --rename-trunk NAME use NAME as name of exported trunk branch 486 ** --repository|-R REPOSITORY export the given REPOSITORY 487 ** 488 ** See also: import 489 */ 490 /* 491 ** COMMAND: export* 492 ** 493 ** This command is deprecated. Use "fossil git export" instead. 494 */ 495 void export_cmd(void){ 496 Stmt q, q2, q3; 497 Bag blobs, vers; 498 unsigned int unused_mark = 1; 499 const char *markfile_in; 500 const char *markfile_out; 501 502 bag_init(&blobs); 503 bag_init(&vers); 504 505 find_option("git", 0, 0); /* Ignore the --git option for now */ 506 markfile_in = find_option("import-marks", 0, 1); 507 markfile_out = find_option("export-marks", 0, 1); 508 509 if( !(gexport.zTrunkName = find_option("rename-trunk", 0, 1)) ){ 510 gexport.zTrunkName = "trunk"; 511 } 512 513 db_find_and_open_repository(0, 2); 514 verify_all_options(); 515 if( g.argc!=2 && g.argc!=3 ){ usage("--git ?REPOSITORY?"); } 516 517 db_multi_exec("CREATE TEMPORARY TABLE oldblob(rid INTEGER PRIMARY KEY)"); 518 db_multi_exec("CREATE TEMPORARY TABLE oldcommit(rid INTEGER PRIMARY KEY)"); 519 db_multi_exec("CREATE TEMP TABLE xmark(tname TEXT UNIQUE, trid INT," 520 " tuuid TEXT)"); 521 db_multi_exec("CREATE INDEX xmark_trid ON xmark(trid)"); 522 if( markfile_in!=0 ){ 523 Stmt qb,qc; 524 FILE *f; 525 int rid; 526 527 f = fossil_fopen(markfile_in, "r"); 528 if( f==0 ){ 529 fossil_fatal("cannot open %s for reading", markfile_in); 530 } 531 if( import_marks(f, &blobs, &vers, &unused_mark)<0 ){ 532 fossil_fatal("error importing marks from file: %s", markfile_in); 533 } 534 db_prepare(&qb, "INSERT OR IGNORE INTO oldblob VALUES (:rid)"); 535 db_prepare(&qc, "INSERT OR IGNORE INTO oldcommit VALUES (:rid)"); 536 rid = bag_first(&blobs); 537 if( rid!=0 ){ 538 do{ 539 db_bind_int(&qb, ":rid", rid); 540 db_step(&qb); 541 db_reset(&qb); 542 }while((rid = bag_next(&blobs, rid))!=0); 543 } 544 rid = bag_first(&vers); 545 if( rid!=0 ){ 546 do{ 547 db_bind_int(&qc, ":rid", rid); 548 db_step(&qc); 549 db_reset(&qc); 550 }while((rid = bag_next(&vers, rid))!=0); 551 } 552 db_finalize(&qb); 553 db_finalize(&qc); 554 fclose(f); 555 } 556 557 /* Step 1: Generate "blob" records for every artifact that is part 558 ** of a check-in 559 */ 560 fossil_binary_mode(stdout); 561 db_multi_exec("CREATE TEMP TABLE newblob(rid INTEGER KEY, srcid INTEGER)"); 562 db_multi_exec("CREATE INDEX newblob_src ON newblob(srcid)"); 563 db_multi_exec( 564 "INSERT INTO newblob" 565 " SELECT DISTINCT fid," 566 " CASE WHEN EXISTS(SELECT 1 FROM delta" 567 " WHERE rid=fid" 568 " AND NOT EXISTS(SELECT 1 FROM oldblob" 569 " WHERE srcid=fid))" 570 " THEN (SELECT srcid FROM delta WHERE rid=fid)" 571 " ELSE 0" 572 " END" 573 " FROM mlink" 574 " WHERE fid>0 AND NOT EXISTS(SELECT 1 FROM oldblob WHERE rid=fid)"); 575 db_prepare(&q, 576 "SELECT DISTINCT fid FROM mlink" 577 " WHERE fid>0 AND NOT EXISTS(SELECT 1 FROM oldblob WHERE rid=fid)"); 578 db_prepare(&q2, "INSERT INTO oldblob VALUES (:rid)"); 579 db_prepare(&q3, "SELECT rid FROM newblob WHERE srcid= (:srcid)"); 580 while( db_step(&q)==SQLITE_ROW ){ 581 int rid = db_column_int(&q, 0); 582 Blob content; 583 584 while( !bag_find(&blobs, rid) ){ 585 char *zMark; 586 content_get(rid, &content); 587 db_bind_int(&q2, ":rid", rid); 588 db_step(&q2); 589 db_reset(&q2); 590 zMark = mark_name_from_rid(rid, &unused_mark); 591 printf("blob\nmark %s\ndata %d\n", zMark, blob_size(&content)); 592 free(zMark); 593 bag_insert(&blobs, rid); 594 fwrite(blob_buffer(&content), 1, blob_size(&content), stdout); 595 printf("\n"); 596 blob_reset(&content); 597 598 db_bind_int(&q3, ":srcid", rid); 599 if( db_step(&q3) != SQLITE_ROW ){ 600 db_reset(&q3); 601 break; 602 } 603 rid = db_column_int(&q3, 0); 604 db_reset(&q3); 605 } 606 } 607 db_finalize(&q); 608 db_finalize(&q2); 609 db_finalize(&q3); 610 611 /* Output the commit records. 612 */ 613 topological_sort_checkins(0); 614 db_prepare(&q, 615 "SELECT strftime('%%s',mtime), objid, coalesce(ecomment,comment)," 616 " coalesce(euser,user)," 617 " (SELECT value FROM tagxref WHERE rid=objid AND tagid=%d)" 618 " FROM toponode, event" 619 " WHERE toponode.tid=event.objid" 620 " AND event.type='ci'" 621 " AND NOT EXISTS (SELECT 1 FROM oldcommit WHERE toponode.tid=rid)" 622 " ORDER BY toponode.tseq ASC", 623 TAG_BRANCH 624 ); 625 db_prepare(&q2, "INSERT INTO oldcommit VALUES (:rid)"); 626 while( db_step(&q)==SQLITE_ROW ){ 627 Stmt q4; 628 const char *zSecondsSince1970 = db_column_text(&q, 0); 629 int ckinId = db_column_int(&q, 1); 630 const char *zComment = db_column_text(&q, 2); 631 const char *zUser = db_column_text(&q, 3); 632 const char *zBranch = db_column_text(&q, 4); 633 char *zMark; 634 635 bag_insert(&vers, ckinId); 636 db_bind_int(&q2, ":rid", ckinId); 637 db_step(&q2); 638 db_reset(&q2); 639 if( zBranch==0 || fossil_strcmp(zBranch, "trunk")==0 ){ 640 zBranch = gexport.zTrunkName; 641 } 642 zMark = mark_name_from_rid(ckinId, &unused_mark); 643 printf("commit refs/heads/"); 644 print_ref(zBranch); 645 printf("\nmark %s\n", zMark); 646 free(zMark); 647 printf("committer"); 648 print_person(zUser); 649 printf(" %s +0000\n", zSecondsSince1970); 650 if( zComment==0 ) zComment = "null comment"; 651 printf("data %d\n%s\n", (int)strlen(zComment), zComment); 652 db_prepare(&q3, 653 "SELECT pid FROM plink" 654 " WHERE cid=%d AND isprim" 655 " AND pid IN (SELECT objid FROM event)", 656 ckinId 657 ); 658 if( db_step(&q3) == SQLITE_ROW ){ 659 int pid = db_column_int(&q3, 0); 660 zMark = mark_name_from_rid(pid, &unused_mark); 661 printf("from %s\n", zMark); 662 free(zMark); 663 db_prepare(&q4, 664 "SELECT pid FROM plink" 665 " WHERE cid=%d AND NOT isprim" 666 " AND NOT EXISTS(SELECT 1 FROM phantom WHERE rid=pid)" 667 " ORDER BY pid", 668 ckinId); 669 while( db_step(&q4)==SQLITE_ROW ){ 670 zMark = mark_name_from_rid(db_column_int(&q4, 0), &unused_mark); 671 printf("merge %s\n", zMark); 672 free(zMark); 673 } 674 db_finalize(&q4); 675 }else{ 676 printf("deleteall\n"); 677 } 678 679 db_prepare(&q4, 680 "SELECT filename.name, mlink.fid, mlink.mperm FROM mlink" 681 " JOIN filename ON filename.fnid=mlink.fnid" 682 " WHERE mlink.mid=%d", 683 ckinId 684 ); 685 while( db_step(&q4)==SQLITE_ROW ){ 686 const char *zName = db_column_text(&q4,0); 687 int zNew = db_column_int(&q4,1); 688 int mPerm = db_column_int(&q4,2); 689 if( zNew==0 ){ 690 printf("D %s\n", zName); 691 }else if( bag_find(&blobs, zNew) ){ 692 const char *zPerm; 693 zMark = mark_name_from_rid(zNew, &unused_mark); 694 switch( mPerm ){ 695 case PERM_LNK: zPerm = "120000"; break; 696 case PERM_EXE: zPerm = "100755"; break; 697 default: zPerm = "100644"; break; 698 } 699 printf("M %s %s %s\n", zPerm, zMark, zName); 700 free(zMark); 701 } 702 } 703 db_finalize(&q4); 704 db_finalize(&q3); 705 printf("\n"); 706 } 707 db_finalize(&q2); 708 db_finalize(&q); 709 manifest_cache_clear(); 710 711 712 /* Output tags */ 713 db_prepare(&q, 714 "SELECT tagname, rid, strftime('%%s',mtime)," 715 " (SELECT coalesce(euser, user) FROM event WHERE objid=rid)," 716 " value" 717 " FROM tagxref JOIN tag USING(tagid)" 718 " WHERE tagtype=1 AND tagname GLOB 'sym-*'" 719 ); 720 while( db_step(&q)==SQLITE_ROW ){ 721 const char *zTagname = db_column_text(&q, 0); 722 int rid = db_column_int(&q, 1); 723 char *zMark = mark_name_from_rid(rid, &unused_mark); 724 const char *zSecSince1970 = db_column_text(&q, 2); 725 const char *zUser = db_column_text(&q, 3); 726 const char *zValue = db_column_text(&q, 4); 727 if( rid==0 || !bag_find(&vers, rid) ) continue; 728 zTagname += 4; 729 printf("tag "); 730 print_ref(zTagname); 731 printf("\nfrom %s\n", zMark); 732 free(zMark); 733 printf("tagger"); 734 print_person(zUser); 735 printf(" %s +0000\n", zSecSince1970); 736 printf("data %d\n", zValue==NULL?0:(int)strlen(zValue)+1); 737 if( zValue!=NULL ) printf("%s\n",zValue); 738 } 739 db_finalize(&q); 740 741 if( markfile_out!=0 ){ 742 FILE *f; 743 f = fossil_fopen(markfile_out, "w"); 744 if( f == 0 ){ 745 fossil_fatal("cannot open %s for writing", markfile_out); 746 } 747 export_marks(f, &blobs, &vers); 748 if( ferror(f)!=0 || fclose(f)!=0 ){ 749 fossil_fatal("error while writing %s", markfile_out); 750 } 751 } 752 bag_clear(&blobs); 753 bag_clear(&vers); 754 } 755 756 /* 757 ** Construct the temporary table toposort as follows: 758 ** 759 ** CREATE TEMP TABLE toponode( 760 ** tid INTEGER PRIMARY KEY, -- Check-in id 761 ** tseq INT -- integer total order on check-ins. 762 ** ); 763 ** 764 ** This table contains all check-ins of the repository in topological 765 ** order. "Topological order" means that every parent check-in comes 766 ** before all of its children. Topological order is *almost* the same 767 ** thing as "ORDER BY event.mtime". Differences only arise when there 768 ** are timewarps. In as much as Git hates timewarps, we have to compute 769 ** a correct topological order when doing an export. 770 ** 771 ** Since mtime is a usually already nearly in topological order, the 772 ** algorithm is to start with mtime, then make adjustments as necessary 773 ** for timewarps. This is not a great algorithm for the general case, 774 ** but it is very fast for the overwhelmingly common case where there 775 ** are few timewarps. 776 */ 777 int topological_sort_checkins(int bVerbose){ 778 int nChange = 0; 779 Stmt q1; 780 Stmt chng; 781 db_multi_exec( 782 "CREATE TEMP TABLE toponode(\n" 783 " tid INTEGER PRIMARY KEY,\n" 784 " tseq INT\n" 785 ");\n" 786 "INSERT INTO toponode(tid,tseq) " 787 " SELECT objid, CAST(mtime*8640000 AS int) FROM event WHERE type='ci';\n" 788 "CREATE TEMP TABLE topolink(\n" 789 " tparent INT,\n" 790 " tchild INT,\n" 791 " PRIMARY KEY(tparent,tchild)\n" 792 ") WITHOUT ROWID;" 793 "INSERT INTO topolink(tparent,tchild)" 794 " SELECT pid, cid FROM plink;\n" 795 "CREATE INDEX topolink_child ON topolink(tchild);\n" 796 ); 797 798 /* Find a timewarp instance */ 799 db_prepare(&q1, 800 "SELECT P.tseq, C.tid, C.tseq\n" 801 " FROM toponode P, toponode C, topolink X\n" 802 " WHERE X.tparent=P.tid\n" 803 " AND X.tchild=C.tid\n" 804 " AND P.tseq>=C.tseq;" 805 ); 806 807 /* Update the timestamp on :tid to have value :tseq */ 808 db_prepare(&chng, 809 "UPDATE toponode SET tseq=:tseq WHERE tid=:tid" 810 ); 811 812 while( db_step(&q1)==SQLITE_ROW ){ 813 i64 iParentTime = db_column_int64(&q1, 0); 814 int iChild = db_column_int(&q1, 1); 815 i64 iChildTime = db_column_int64(&q1, 2); 816 nChange++; 817 if( nChange>10000 ){ 818 fossil_fatal("failed to fix all timewarps after 100000 attempts"); 819 } 820 db_reset(&q1); 821 db_bind_int64(&chng, ":tid", iChild); 822 db_bind_int64(&chng, ":tseq", iParentTime+1); 823 db_step(&chng); 824 db_reset(&chng); 825 if( bVerbose ){ 826 fossil_print("moving %d from %lld to %lld\n", 827 iChild, iChildTime, iParentTime+1); 828 } 829 } 830 831 db_finalize(&q1); 832 db_finalize(&chng); 833 return nChange; 834 } 835 836 /* 837 ** COMMAND: test-topological-sort 838 ** 839 ** Invoke the topological_sort_checkins() interface for testing 840 ** purposes. 841 */ 842 void test_topological_sort(void){ 843 int n; 844 db_find_and_open_repository(0, 0); 845 n = topological_sort_checkins(1); 846 fossil_print("%d reorderings required\n", n); 847 } 848 849 /*************************************************************************** 850 ** Implementation of the "fossil git" command follows. We hope that the 851 ** new code that follows will largely replace the legacy "fossil export" 852 ** and "fossil import" code above. 853 */ 854 855 /* Verbosity level. Higher means more output. 856 ** 857 ** 0 print nothing at all 858 ** 1 Errors only 859 ** 2 Progress information (This is the default) 860 ** 3 Extra details 861 */ 862 #define VERB_ERROR 1 863 #define VERB_NORMAL 2 864 #define VERB_EXTRA 3 865 static int gitmirror_verbosity = VERB_NORMAL; 866 867 /* 868 ** Output routine that depends on verbosity 869 */ 870 static void gitmirror_message(int iLevel, const char *zFormat, ...){ 871 va_list ap; 872 if( iLevel>gitmirror_verbosity ) return; 873 va_start(ap, zFormat); 874 fossil_vprint(zFormat, ap); 875 va_end(ap); 876 } 877 878 /* 879 ** Convert characters of z[] that are not allowed to be in branch or 880 ** tag names into "_". 881 */ 882 static void gitmirror_sanitize_name(char *z){ 883 static unsigned char aSafe[] = { 884 /* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xA xB xC xD xE xF */ 885 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x */ 886 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 1x */ 887 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 2x */ 888 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 0, /* 3x */ 889 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 4x */ 890 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 1, /* 5x */ 891 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 6x */ 892 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, /* 7x */ 893 }; 894 unsigned char *zu = (unsigned char*)z; 895 int i; 896 for(i=0; zu[i]; i++){ 897 if( zu[i]>0x7f || !aSafe[zu[i]] ){ 898 zu[i] = '_'; 899 }else if( zu[i]=='/' && (i==0 || zu[i+1]==0 || zu[i+1]=='/') ){ 900 zu[i] = '_'; 901 }else if( zu[i]=='.' && (zu[i+1]==0 || zu[i+1]=='.' 902 || (i>0 && zu[i-1]=='.')) ){ 903 zu[i] = '_'; 904 } 905 } 906 } 907 908 /* 909 ** Quote a filename as a C-style string using \\ and \" if necessary. 910 ** If quoting is not necessary, just return a copy of the input string. 911 ** 912 ** The return value is a held in memory obtained from fossil_malloc() 913 ** and must be freed by the caller. 914 */ 915 static char *gitmirror_quote_filename_if_needed(const char *zIn){ 916 int i, j; 917 char c; 918 int nSpecial = 0; 919 char *zOut; 920 for(i=0; (c = zIn[i])!=0; i++){ 921 if( c=='\\' || c=='"' || c=='\n' ){ 922 nSpecial++; 923 } 924 } 925 if( nSpecial==0 ){ 926 return fossil_strdup(zIn); 927 } 928 zOut = fossil_malloc( i+nSpecial+3 ); 929 zOut[0] = '"'; 930 for(i=0, j=1; (c = zIn[i])!=0; i++){ 931 if( c=='\\' || c=='"' || c=='\n' ){ 932 zOut[j++] = '\\'; 933 if( c=='\n' ){ 934 zOut[j++] = 'n'; 935 }else{ 936 zOut[j++] = c; 937 } 938 }else{ 939 zOut[j++] = c; 940 } 941 } 942 zOut[j++] = '"'; 943 zOut[j] = 0; 944 return zOut; 945 } 946 947 /* 948 ** Find the Git-name corresponding to the Fossil-name zUuid. 949 ** 950 ** If the mark does not exist and if the bCreate flag is false, then 951 ** return NULL. If the mark does not exist and the bCreate flag is true, 952 ** then create the mark. 953 ** 954 ** The string returned is obtained from fossil_malloc() and should 955 ** be freed by the caller. 956 */ 957 static char *gitmirror_find_mark(const char *zUuid, int isFile, int bCreate){ 958 static Stmt sFind, sIns; 959 db_static_prepare(&sFind, 960 "SELECT coalesce(githash,printf(':%%d',id))" 961 " FROM mirror.mmark WHERE uuid=:uuid AND isfile=:isfile" 962 ); 963 db_bind_text(&sFind, ":uuid", zUuid); 964 db_bind_int(&sFind, ":isfile", isFile!=0); 965 if( db_step(&sFind)==SQLITE_ROW ){ 966 char *zMark = fossil_strdup(db_column_text(&sFind, 0)); 967 db_reset(&sFind); 968 return zMark; 969 } 970 db_reset(&sFind); 971 if( !bCreate ){ 972 return 0; 973 } 974 db_static_prepare(&sIns, 975 "INSERT INTO mirror.mmark(uuid,isfile) VALUES(:uuid,:isfile)" 976 ); 977 db_bind_text(&sIns, ":uuid", zUuid); 978 db_bind_int(&sIns, ":isfile", isFile!=0); 979 db_step(&sIns); 980 db_reset(&sIns); 981 return mprintf(":%d", db_last_insert_rowid()); 982 } 983 984 /* This is the SHA3-256 hash of an empty file */ 985 static const char zEmptySha3[] = 986 "a7ffc6f8bf1ed76651c14756a061d662f580ff4de43b49fa82d80a4b80f8434a"; 987 988 /* 989 ** Export a single file named by zUuid. 990 ** 991 ** Return 0 on success and non-zero on any failure. 992 ** 993 ** If zUuid is a shunned file, then treat it as if it were any empty file. 994 ** But files that are missing from the repository but have not been officially 995 ** shunned cause an error return. Except, if bPhantomOk is true, then missing 996 ** files are replaced by an empty file. 997 */ 998 static int gitmirror_send_file(FILE *xCmd, const char *zUuid, int bPhantomOk){ 999 char *zMark; 1000 int rid; 1001 int rc; 1002 Blob data; 1003 rid = fast_uuid_to_rid(zUuid); 1004 if( rid<0 ){ 1005 if( bPhantomOk || uuid_is_shunned(zUuid) ){ 1006 gitmirror_message(VERB_EXTRA, "missing file: %s\n", zUuid); 1007 zUuid = zEmptySha3; 1008 }else{ 1009 return 1; 1010 } 1011 }else{ 1012 rc = content_get(rid, &data); 1013 if( rc==0 ){ 1014 if( bPhantomOk ){ 1015 blob_init(&data, 0, 0); 1016 gitmirror_message(VERB_EXTRA, "missing file: %s\n", zUuid); 1017 zUuid = zEmptySha3; 1018 }else{ 1019 return 1; 1020 } 1021 } 1022 } 1023 zMark = gitmirror_find_mark(zUuid, 1, 1); 1024 if( zMark[0]==':' ){ 1025 fprintf(xCmd, "blob\nmark %s\ndata %d\n", zMark, blob_size(&data)); 1026 fwrite(blob_buffer(&data), 1, blob_size(&data), xCmd); 1027 fprintf(xCmd, "\n"); 1028 } 1029 fossil_free(zMark); 1030 blob_reset(&data); 1031 return 0; 1032 } 1033 1034 /* 1035 ** Transfer a check-in over to the mirror. "rid" is the BLOB.RID for 1036 ** the check-in to export. 1037 ** 1038 ** If any ancestor of the check-in has not yet been exported, then 1039 ** invoke this routine recursively to export the ancestor first. 1040 ** This can only happen on a timewarp, so deep nesting is unlikely. 1041 ** 1042 ** Before sending the check-in, first make sure all associated files 1043 ** have already been exported, and send "blob" records for any that 1044 ** have not been. Update the MIRROR.MMARK table so that it holds the 1045 ** marks for the exported files. 1046 ** 1047 ** Return zero on success and non-zero if the export should be stopped. 1048 */ 1049 static int gitmirror_send_checkin( 1050 FILE *xCmd, /* Write fast-import text on this pipe */ 1051 int rid, /* BLOB.RID for the check-in to export */ 1052 const char *zUuid, /* BLOB.UUID for the check-in to export */ 1053 int *pnLimit, /* Stop when the counter reaches zero */ 1054 int fManifest /* MFESTFLG_* values */ 1055 ){ 1056 Manifest *pMan; /* The check-in to be output */ 1057 int i; /* Loop counter */ 1058 int iParent; /* Which immediate ancestor is primary. -1 for none */ 1059 Stmt q; /* An SQL query */ 1060 char *zBranch; /* The branch of the check-in */ 1061 char *zMark; /* The Git-name of the check-in */ 1062 Blob sql; /* String of SQL for part of the query */ 1063 Blob comment; /* The comment text for the check-in */ 1064 int nErr = 0; /* Number of errors */ 1065 int bPhantomOk; /* True if phantom files should be ignored */ 1066 char buf[24]; 1067 1068 pMan = manifest_get(rid, CFTYPE_MANIFEST, 0); 1069 if( pMan==0 ){ 1070 /* Must be a phantom. Return without doing anything, and in particular 1071 ** without creating a mark for this check-in. */ 1072 gitmirror_message(VERB_NORMAL, "missing check-in: %s\n", zUuid); 1073 return 0; 1074 } 1075 1076 /* Check to see if any parent logins have not yet been processed, and 1077 ** if so, create them */ 1078 for(i=0; i<pMan->nParent; i++){ 1079 char *zPMark = gitmirror_find_mark(pMan->azParent[i], 0, 0); 1080 if( zPMark==0 ){ 1081 int prid = db_int(0, "SELECT rid FROM blob WHERE uuid=%Q", 1082 pMan->azParent[i]); 1083 int rc = gitmirror_send_checkin(xCmd, prid, pMan->azParent[i], 1084 pnLimit, fManifest); 1085 if( rc || *pnLimit<=0 ){ 1086 manifest_destroy(pMan); 1087 return 1; 1088 } 1089 } 1090 fossil_free(zPMark); 1091 } 1092 1093 /* Ignore phantom files on check-ins that are over one year old */ 1094 bPhantomOk = db_int(0, "SELECT %.6f<julianday('now','-1 year')", 1095 pMan->rDate); 1096 1097 /* Make sure all necessary files have been exported */ 1098 db_prepare(&q, 1099 "SELECT uuid FROM files_of_checkin(%Q)" 1100 " WHERE uuid NOT IN (SELECT uuid FROM mirror.mmark)", 1101 zUuid 1102 ); 1103 while( db_step(&q)==SQLITE_ROW ){ 1104 const char *zFUuid = db_column_text(&q, 0); 1105 int n = gitmirror_send_file(xCmd, zFUuid, bPhantomOk); 1106 nErr += n; 1107 if( n ) gitmirror_message(VERB_ERROR, "missing file: %s\n", zFUuid); 1108 } 1109 db_finalize(&q); 1110 1111 /* If some required files could not be exported, abandon the check-in 1112 ** export */ 1113 if( nErr ){ 1114 gitmirror_message(VERB_ERROR, 1115 "export of %s abandoned due to missing files\n", zUuid); 1116 *pnLimit = 0; 1117 return 1; 1118 } 1119 1120 /* Figure out which branch this check-in is a member of */ 1121 zBranch = db_text(0, 1122 "SELECT value FROM tagxref WHERE tagid=%d AND tagtype>0 AND rid=%d", 1123 TAG_BRANCH, rid 1124 ); 1125 if( fossil_strcmp(zBranch,"trunk")==0 ){ 1126 fossil_free(zBranch); 1127 zBranch = mprintf("master"); 1128 }else if( zBranch==0 ){ 1129 zBranch = mprintf("unknown"); 1130 }else{ 1131 gitmirror_sanitize_name(zBranch); 1132 } 1133 1134 /* Export the check-in */ 1135 fprintf(xCmd, "commit refs/heads/%s\n", zBranch); 1136 fossil_free(zBranch); 1137 zMark = gitmirror_find_mark(zUuid,0,1); 1138 fprintf(xCmd, "mark %s\n", zMark); 1139 fossil_free(zMark); 1140 sqlite3_snprintf(sizeof(buf), buf, "%lld", 1141 (sqlite3_int64)((pMan->rDate-2440587.5)*86400.0) 1142 ); 1143 fprintf(xCmd, "committer %s <%s@noemail.net> %s +0000\n", 1144 pMan->zUser, pMan->zUser, buf 1145 ); 1146 blob_init(&comment, pMan->zComment, -1); 1147 if( blob_size(&comment)==0 ){ 1148 blob_append(&comment, "(no comment)", -1); 1149 } 1150 blob_appendf(&comment, "\n\nFossilOrigin-Name: %s", zUuid); 1151 fprintf(xCmd, "data %d\n%s\n", blob_size(&comment), blob_str(&comment)); 1152 blob_reset(&comment); 1153 iParent = -1; /* Which ancestor is the primary parent */ 1154 for(i=0; i<pMan->nParent; i++){ 1155 char *zOther = gitmirror_find_mark(pMan->azParent[i],0,0); 1156 if( zOther==0 ) continue; 1157 if( iParent<0 ){ 1158 iParent = i; 1159 fprintf(xCmd, "from %s\n", zOther); 1160 }else{ 1161 fprintf(xCmd, "merge %s\n", zOther); 1162 } 1163 fossil_free(zOther); 1164 } 1165 if( iParent>=0 ){ 1166 db_prepare(&q, 1167 "SELECT filename FROM files_of_checkin(%Q)" 1168 " EXCEPT SELECT filename FROM files_of_checkin(%Q)", 1169 pMan->azParent[iParent], zUuid 1170 ); 1171 while( db_step(&q)==SQLITE_ROW ){ 1172 fprintf(xCmd, "D %s\n", db_column_text(&q,0)); 1173 } 1174 db_finalize(&q); 1175 } 1176 blob_init(&sql, 0, 0); 1177 blob_append_sql(&sql, 1178 "SELECT filename, uuid, perm FROM files_of_checkin(%Q)", 1179 zUuid 1180 ); 1181 if( pMan->nParent ){ 1182 blob_append_sql(&sql, 1183 " EXCEPT SELECT filename, uuid, perm FROM files_of_checkin(%Q)", 1184 pMan->azParent[0]); 1185 } 1186 db_prepare(&q, 1187 "SELECT x.filename, x.perm," 1188 " coalesce(mmark.githash,printf(':%%d',mmark.id))" 1189 " FROM (%s) AS x, mirror.mmark" 1190 " WHERE mmark.uuid=x.uuid AND isfile", 1191 blob_sql_text(&sql) 1192 ); 1193 blob_reset(&sql); 1194 while( db_step(&q)==SQLITE_ROW ){ 1195 const char *zFilename = db_column_text(&q,0); 1196 const char *zMode = db_column_text(&q,1); 1197 const char *zMark = db_column_text(&q,2); 1198 const char *zGitMode = "100644"; 1199 char *zFNQuoted = 0; 1200 if( zMode ){ 1201 if( strchr(zMode,'x') ) zGitMode = "100755"; 1202 if( strchr(zMode,'l') ) zGitMode = "120000"; 1203 } 1204 zFNQuoted = gitmirror_quote_filename_if_needed(zFilename); 1205 fprintf(xCmd,"M %s %s %s\n", zGitMode, zMark, zFNQuoted); 1206 fossil_free(zFNQuoted); 1207 } 1208 db_finalize(&q); 1209 1210 /* Include Fossil-generated auxiliary files in the check-in */ 1211 if( fManifest & MFESTFLG_RAW ){ 1212 Blob manifest; 1213 content_get(rid, &manifest); 1214 fprintf(xCmd,"M 100644 inline manifest\ndata %d\n%s\n", 1215 blob_size(&manifest), blob_str(&manifest)); 1216 blob_reset(&manifest); 1217 } 1218 if( fManifest & MFESTFLG_UUID ){ 1219 int n = (int)strlen(zUuid); 1220 fprintf(xCmd,"M 100644 inline manifest.uuid\ndata %d\n%s\n", n, zUuid); 1221 } 1222 if( fManifest & MFESTFLG_TAGS ){ 1223 Blob tagslist; 1224 blob_init(&tagslist, 0, 0); 1225 get_checkin_taglist(rid, &tagslist); 1226 fprintf(xCmd,"M 100644 inline manifest.tags\ndata %d\n%s\n", 1227 blob_size(&tagslist), blob_str(&tagslist)); 1228 blob_reset(&tagslist); 1229 } 1230 1231 /* The check-in is finished, so decrement the counter */ 1232 (*pnLimit)--; 1233 return 0; 1234 } 1235 1236 /* 1237 ** Implementation of the "fossil git export" command. 1238 */ 1239 void gitmirror_export_command(void){ 1240 const char *zLimit; /* Text of the --limit flag */ 1241 int nLimit = 0x7fffffff; /* Numeric value of the --limit flag */ 1242 int nTotal = 0; /* Total number of check-ins to export */ 1243 char *zMirror; /* Name of the mirror */ 1244 char *z; /* Generic string */ 1245 char *zCmd; /* git command to run as a subprocess */ 1246 const char *zDebug = 0; /* Value of the --debug flag */ 1247 const char *zAutoPush = 0; /* Value of the --autopush flag */ 1248 char *zPushUrl; /* URL to sync the mirror to */ 1249 double rEnd; /* time of most recent export */ 1250 int rc; /* Result code */ 1251 int bForce; /* Do the export and sync even if no changes*/ 1252 int bNeedRepack = 0; /* True if we should run repack at the end */ 1253 int fManifest; /* Current "manifest" setting */ 1254 FILE *xCmd; /* Pipe to the "git fast-import" command */ 1255 FILE *pMarks; /* Git mark files */ 1256 Stmt q; /* Queries */ 1257 char zLine[200]; /* One line of a mark file */ 1258 1259 zDebug = find_option("debug",0,1); 1260 db_find_and_open_repository(0, 0); 1261 zLimit = find_option("limit", 0, 1); 1262 if( zLimit ){ 1263 nLimit = (unsigned int)atoi(zLimit); 1264 if( nLimit<=0 ) fossil_fatal("--limit must be positive"); 1265 } 1266 zAutoPush = find_option("autopush",0,1); 1267 bForce = find_option("force","f",0)!=0; 1268 gitmirror_verbosity = VERB_NORMAL; 1269 while( find_option("quiet","q",0)!=0 ){ gitmirror_verbosity--; } 1270 while( find_option("verbose","v",0)!=0 ){ gitmirror_verbosity++; } 1271 verify_all_options(); 1272 if( g.argc!=4 && g.argc!=3 ){ usage("export ?MIRROR?"); } 1273 if( g.argc==4 ){ 1274 Blob mirror; 1275 file_canonical_name(g.argv[3], &mirror, 0); 1276 db_set("last-git-export-repo", blob_str(&mirror), 0); 1277 blob_reset(&mirror); 1278 } 1279 zMirror = db_get("last-git-export-repo", 0); 1280 if( zMirror==0 ){ 1281 fossil_fatal("no Git repository specified"); 1282 } 1283 1284 /* Make sure the GIT repository directory exists */ 1285 rc = file_mkdir(zMirror, ExtFILE, 0); 1286 if( rc ) fossil_fatal("cannot create directory \"%s\"", zMirror); 1287 1288 /* Make sure GIT has been initialized */ 1289 z = mprintf("%s/.git", zMirror); 1290 if( !file_isdir(z, ExtFILE) ){ 1291 zCmd = mprintf("git init \"%s\"",zMirror); 1292 gitmirror_message(VERB_NORMAL, "%s\n", zCmd); 1293 rc = fossil_system(zCmd); 1294 if( rc ){ 1295 fossil_fatal("cannot initialize the git repository using: \"%s\"", zCmd); 1296 } 1297 fossil_free(zCmd); 1298 bNeedRepack = 1; 1299 } 1300 fossil_free(z); 1301 1302 /* Make sure the .mirror_state subdirectory exists */ 1303 z = mprintf("%s/.mirror_state", zMirror); 1304 rc = file_mkdir(z, ExtFILE, 0); 1305 if( rc ) fossil_fatal("cannot create directory \"%s\"", z); 1306 fossil_free(z); 1307 1308 /* Attach the .mirror_state/db database */ 1309 db_multi_exec("ATTACH '%q/.mirror_state/db' AS mirror;", zMirror); 1310 db_begin_write(); 1311 db_multi_exec( 1312 "CREATE TABLE IF NOT EXISTS mirror.mconfig(\n" 1313 " key TEXT PRIMARY KEY,\n" 1314 " Value ANY\n" 1315 ") WITHOUT ROWID;\n" 1316 "CREATE TABLE IF NOT EXISTS mirror.mmark(\n" 1317 " id INTEGER PRIMARY KEY,\n" 1318 " uuid TEXT,\n" 1319 " isfile BOOLEAN,\n" 1320 " githash TEXT,\n" 1321 " UNIQUE(uuid,isfile)\n" 1322 ");" 1323 ); 1324 if( !db_table_has_column("mirror","mmark","isfile") ){ 1325 db_multi_exec( 1326 "ALTER TABLE mirror.mmark RENAME TO mmark_old;" 1327 "CREATE TABLE IF NOT EXISTS mirror.mmark(\n" 1328 " id INTEGER PRIMARY KEY,\n" 1329 " uuid TEXT,\n" 1330 " isfile BOOLEAN,\n" 1331 " githash TEXT,\n" 1332 " UNIQUE(uuid,isfile)\n" 1333 ");" 1334 "INSERT OR IGNORE INTO mirror.mmark(id,uuid,githash,isfile)" 1335 " SELECT id,uuid,githash," 1336 " NOT EXISTS(SELECT 1 FROM repository.event, repository.blob" 1337 " WHERE event.objid=blob.rid" 1338 " AND blob.uuid=mmark_old.uuid)" 1339 " FROM mirror.mmark_old;\n" 1340 "DROP TABLE mirror.mmark_old;\n" 1341 ); 1342 } 1343 1344 /* Change the autopush setting if the --autopush flag is present */ 1345 if( zAutoPush ){ 1346 if( is_false(zAutoPush) ){ 1347 db_multi_exec("DELETE FROM mirror.mconfig WHERE key='autopush'"); 1348 }else{ 1349 db_multi_exec( 1350 "REPLACE INTO mirror.mconfig(key,value)" 1351 "VALUES('autopush',%Q)", 1352 zAutoPush 1353 ); 1354 } 1355 } 1356 1357 /* See if there is any work to be done. Exit early if not, before starting 1358 ** the "git fast-import" command. */ 1359 if( !bForce 1360 && !db_exists("SELECT 1 FROM event WHERE type IN ('ci','t')" 1361 " AND mtime>coalesce((SELECT value FROM mconfig" 1362 " WHERE key='start'),0.0)") 1363 ){ 1364 gitmirror_message(VERB_NORMAL, "no changes\n"); 1365 db_commit_transaction(); 1366 return; 1367 } 1368 1369 /* Do we need to include manifest files in the clone? */ 1370 fManifest = db_get_manifest_setting(); 1371 1372 /* Change to the MIRROR directory so that the Git commands will work */ 1373 rc = file_chdir(zMirror, 0); 1374 if( rc ) fossil_fatal("cannot change the working directory to \"%s\"", 1375 zMirror); 1376 1377 /* Start up the git fast-import command */ 1378 if( zDebug ){ 1379 if( fossil_strcmp(zDebug,"stdout")==0 ){ 1380 xCmd = stdout; 1381 }else{ 1382 xCmd = fopen(zDebug, "wb"); 1383 if( xCmd==0 ) fossil_fatal("cannot open file \"%s\" for writing", zDebug); 1384 } 1385 }else{ 1386 zCmd = mprintf("git fast-import" 1387 " --export-marks=.mirror_state/marks.txt" 1388 " --quiet --done"); 1389 gitmirror_message(VERB_NORMAL, "%s\n", zCmd); 1390 #ifdef _WIN32 1391 xCmd = popen(zCmd, "wb"); 1392 #else 1393 xCmd = popen(zCmd, "w"); 1394 #endif 1395 if( zCmd==0 ){ 1396 fossil_fatal("cannot start the \"git fast-import\" command"); 1397 } 1398 fossil_free(zCmd); 1399 } 1400 1401 /* Run the export */ 1402 rEnd = 0.0; 1403 db_multi_exec( 1404 "CREATE TEMP TABLE tomirror(objid,mtime,uuid);\n" 1405 "INSERT INTO tomirror " 1406 "SELECT objid, mtime, blob.uuid FROM event, blob\n" 1407 " WHERE type='ci'" 1408 " AND mtime>coalesce((SELECT value FROM mconfig WHERE key='start'),0.0)" 1409 " AND blob.rid=event.objid" 1410 " AND blob.uuid NOT IN (SELECT uuid FROM mirror.mmark WHERE NOT isfile);" 1411 ); 1412 nTotal = db_int(0, "SELECT count(*) FROM tomirror"); 1413 if( nLimit<nTotal ){ 1414 nTotal = nLimit; 1415 }else if( nLimit>nTotal ){ 1416 nLimit = nTotal; 1417 } 1418 db_prepare(&q, 1419 "SELECT objid, mtime, uuid FROM tomirror ORDER BY mtime" 1420 ); 1421 while( nLimit && db_step(&q)==SQLITE_ROW ){ 1422 int rid = db_column_int(&q, 0); 1423 double rMTime = db_column_double(&q, 1); 1424 const char *zUuid = db_column_text(&q, 2); 1425 if( rMTime>rEnd ) rEnd = rMTime; 1426 rc = gitmirror_send_checkin(xCmd, rid, zUuid, &nLimit, fManifest); 1427 if( rc ) break; 1428 gitmirror_message(VERB_NORMAL,"%d/%d \r", nTotal-nLimit, nTotal); 1429 fflush(stdout); 1430 } 1431 db_finalize(&q); 1432 fprintf(xCmd, "done\n"); 1433 if( zDebug ){ 1434 if( xCmd!=stdout ) fclose(xCmd); 1435 }else{ 1436 pclose(xCmd); 1437 } 1438 gitmirror_message(VERB_NORMAL, "%d check-ins added to the %s\n", 1439 nTotal-nLimit, zMirror); 1440 1441 /* Read the export-marks file. Transfer the new marks over into 1442 ** the import-marks file. 1443 */ 1444 pMarks = fopen(".mirror_state/marks.txt", "rb"); 1445 if( pMarks ){ 1446 db_prepare(&q, "UPDATE mirror.mmark SET githash=:githash WHERE id=:id"); 1447 while( fgets(zLine, sizeof(zLine), pMarks) ){ 1448 int j, k; 1449 if( zLine[0]!=':' ) continue; 1450 db_bind_int(&q, ":id", atoi(zLine+1)); 1451 for(j=1; zLine[j] && zLine[j]!=' '; j++){} 1452 if( zLine[j]!=' ' ) continue; 1453 j++; 1454 if( zLine[j]==0 ) continue; 1455 for(k=j; fossil_isalnum(zLine[k]); k++){} 1456 zLine[k] = 0; 1457 db_bind_text(&q, ":githash", &zLine[j]); 1458 db_step(&q); 1459 db_reset(&q); 1460 } 1461 db_finalize(&q); 1462 fclose(pMarks); 1463 file_delete(".mirror_state/marks.txt"); 1464 }else{ 1465 fossil_fatal("git fast-import didn't generate a marks file!"); 1466 } 1467 db_multi_exec( 1468 "CREATE INDEX IF NOT EXISTS mirror.mmarkx1 ON mmark(githash);" 1469 ); 1470 1471 /* Do any tags that have been created since the start time */ 1472 db_prepare(&q, 1473 "SELECT substr(tagname,5), githash" 1474 " FROM (SELECT tagxref.tagid AS xtagid, tagname, rid, max(mtime) AS mtime" 1475 " FROM tagxref JOIN tag ON tag.tagid=tagxref.tagid" 1476 " WHERE tag.tagname GLOB 'sym-*'" 1477 " AND tagxref.tagtype=1" 1478 " AND tagxref.mtime > coalesce((SELECT value FROM mconfig" 1479 " WHERE key='start'),0.0)" 1480 " GROUP BY tagxref.tagid) AS tx" 1481 " JOIN blob ON tx.rid=blob.rid" 1482 " JOIN mmark ON mmark.uuid=blob.uuid;" 1483 ); 1484 while( db_step(&q)==SQLITE_ROW ){ 1485 char *zTagname = fossil_strdup(db_column_text(&q,0)); 1486 const char *zObj = db_column_text(&q,1); 1487 char *zTagCmd; 1488 gitmirror_sanitize_name(zTagname); 1489 zTagCmd = mprintf("git tag -f \"%s\" %s", zTagname, zObj); 1490 fossil_free(zTagname); 1491 gitmirror_message(VERB_NORMAL, "%s\n", zTagCmd); 1492 fossil_system(zTagCmd); 1493 fossil_free(zTagCmd); 1494 } 1495 db_finalize(&q); 1496 1497 /* Update all references that might have changed since the start time */ 1498 db_prepare(&q, 1499 "SELECT" 1500 " tagxref.value AS name," 1501 " max(event.mtime) AS mtime," 1502 " mmark.githash AS gitckin" 1503 " FROM tagxref, tag, event, blob, mmark" 1504 " WHERE tagxref.tagid=tag.tagid" 1505 " AND tagxref.tagtype>0" 1506 " AND tag.tagname='branch'" 1507 " AND event.objid=tagxref.rid" 1508 " AND event.mtime > coalesce((SELECT value FROM mconfig" 1509 " WHERE key='start'),0.0)" 1510 " AND blob.rid=tagxref.rid" 1511 " AND mmark.uuid=blob.uuid" 1512 " GROUP BY 1" 1513 ); 1514 while( db_step(&q)==SQLITE_ROW ){ 1515 char *zBrname = fossil_strdup(db_column_text(&q,0)); 1516 const char *zObj = db_column_text(&q,2); 1517 char *zRefCmd; 1518 if( fossil_strcmp(zBrname,"trunk")==0 ){ 1519 fossil_free(zBrname); 1520 zBrname = fossil_strdup("master"); 1521 }else{ 1522 gitmirror_sanitize_name(zBrname); 1523 } 1524 zRefCmd = mprintf("git update-ref \"refs/heads/%s\" %s", zBrname, zObj); 1525 fossil_free(zBrname); 1526 gitmirror_message(VERB_NORMAL, "%s\n", zRefCmd); 1527 fossil_system(zRefCmd); 1528 fossil_free(zRefCmd); 1529 } 1530 db_finalize(&q); 1531 1532 /* Update the start time */ 1533 if( rEnd>0.0 ){ 1534 db_prepare(&q, "REPLACE INTO mirror.mconfig(key,value) VALUES('start',:x)"); 1535 db_bind_double(&q, ":x", rEnd); 1536 db_step(&q); 1537 db_finalize(&q); 1538 } 1539 db_commit_transaction(); 1540 1541 /* Maybe run a git repack */ 1542 if( bNeedRepack ){ 1543 const char *zRepack = "git repack -adf"; 1544 gitmirror_message(VERB_NORMAL, "%s\n", zRepack); 1545 fossil_system(zRepack); 1546 } 1547 1548 /* Optionally do a "git push" */ 1549 zPushUrl = db_text(0, "SELECT value FROM mconfig WHERE key='autopush'"); 1550 if( zPushUrl ){ 1551 char *zPushCmd; 1552 UrlData url; 1553 if( sqlite3_strglob("http*", zPushUrl)==0 ){ 1554 url_parse_local(zPushUrl, 0, &url); 1555 zPushCmd = mprintf("git push --mirror %s", url.canonical); 1556 }else{ 1557 zPushCmd = mprintf("git push --mirror %s", zPushUrl); 1558 } 1559 gitmirror_message(VERB_NORMAL, "%s\n", zPushCmd); 1560 fossil_free(zPushCmd); 1561 zPushCmd = mprintf("git push --mirror %s", zPushUrl); 1562 fossil_system(zPushCmd); 1563 fossil_free(zPushCmd); 1564 } 1565 } 1566 1567 /* 1568 ** Implementation of the "fossil git status" command. 1569 ** 1570 ** Show the status of a "git export". 1571 */ 1572 void gitmirror_status_command(void){ 1573 char *zMirror; 1574 char *z; 1575 int n, k; 1576 db_find_and_open_repository(0, 0); 1577 verify_all_options(); 1578 zMirror = db_get("last-git-export-repo", 0); 1579 if( zMirror==0 ){ 1580 fossil_print("Git mirror: none\n"); 1581 return; 1582 } 1583 fossil_print("Git mirror: %s\n", zMirror); 1584 db_multi_exec("ATTACH '%q/.mirror_state/db' AS mirror;", zMirror); 1585 z = db_text(0, "SELECT datetime(value) FROM mconfig WHERE key='start'"); 1586 if( z ){ 1587 double rAge = db_double(0.0, "SELECT julianday('now') - value" 1588 " FROM mconfig WHERE key='start'"); 1589 if( rAge>1.0/86400.0 ){ 1590 fossil_print("Last export: %s (%z ago)\n", z, human_readable_age(rAge)); 1591 }else{ 1592 fossil_print("Last export: %s (moments ago)\n", z); 1593 } 1594 } 1595 z = db_text(0, "SELECT value FROM mconfig WHERE key='autopush'"); 1596 if( z==0 ){ 1597 fossil_print("Autopush: off\n"); 1598 }else{ 1599 UrlData url; 1600 url_parse_local(z, 0, &url); 1601 fossil_print("Autopush: %s\n", url.canonical); 1602 } 1603 n = db_int(0, 1604 "SELECT count(*) FROM event" 1605 " WHERE type='ci'" 1606 " AND mtime>coalesce((SELECT value FROM mconfig" 1607 " WHERE key='start'),0.0)" 1608 ); 1609 if( n==0 ){ 1610 fossil_print("Status: up-to-date\n"); 1611 }else{ 1612 fossil_print("Status: %d check-in%s awaiting export\n", 1613 n, n==1 ? "" : "s"); 1614 } 1615 n = db_int(0, "SELECT count(*) FROM mmark WHERE isfile"); 1616 k = db_int(0, "SELECT count(*) FROm mmark WHERE NOT isfile"); 1617 fossil_print("Exported: %d check-ins and %d file blobs\n", k, n); 1618 } 1619 1620 /* 1621 ** COMMAND: git 1622 ** 1623 ** Usage: %fossil git SUBCOMMAND 1624 ** 1625 ** Do incremental import or export operations between Fossil and Git. 1626 ** Subcommands: 1627 ** 1628 ** fossil git export [MIRROR] [OPTIONS] 1629 ** 1630 ** Write content from the Fossil repository into the Git repository 1631 ** in directory MIRROR. The Git repository is created if it does not 1632 ** already exist. If the Git repository does already exist, then 1633 ** new content added to fossil since the previous export is appended. 1634 ** 1635 ** Repeat this command whenever new checkins are added to the Fossil 1636 ** repository in order to reflect those changes into the mirror. If 1637 ** the MIRROR option is omitted, the repository from the previous 1638 ** invocation is used. 1639 ** 1640 ** The MIRROR directory will contain a subdirectory named 1641 ** ".mirror_state" that contains information that Fossil needs to 1642 ** do incremental exports. Do not attempt to manage or edit the files 1643 ** in that directory since doing so can disrupt future incremental 1644 ** exports. 1645 ** 1646 ** Options: 1647 ** --autopush URL Automatically do a 'git push' to URL. The 1648 ** URL is remembered and used on subsequent exports 1649 ** to the same repository. Or if URL is "off" the 1650 ** auto-push mechanism is disabled 1651 ** --debug FILE Write fast-export text to FILE rather than 1652 ** piping it into "git fast-import". 1653 ** --force|-f Do the export even if nothing has changed 1654 ** --limit N Add no more than N new check-ins to MIRROR. 1655 ** Useful for debugging 1656 ** --quiet|-q Reduce output. Repeat for even less output. 1657 ** --verbose|-v More output. 1658 ** 1659 ** fossil git import MIRROR 1660 ** 1661 ** TBD... 1662 ** 1663 ** fossil git status 1664 ** 1665 ** Show the status of the current Git mirror, if there is one. 1666 */ 1667 void gitmirror_command(void){ 1668 char *zCmd; 1669 int nCmd; 1670 if( g.argc<3 ){ 1671 usage("export ARGS..."); 1672 } 1673 zCmd = g.argv[2]; 1674 nCmd = (int)strlen(zCmd); 1675 if( nCmd>2 && strncmp(zCmd,"export",nCmd)==0 ){ 1676 gitmirror_export_command(); 1677 }else 1678 if( nCmd>2 && strncmp(zCmd,"import",nCmd)==0 ){ 1679 fossil_fatal("not yet implemented - check back later"); 1680 }else 1681 if( nCmd>2 && strncmp(zCmd,"status",nCmd)==0 ){ 1682 gitmirror_status_command(); 1683 }else 1684 { 1685 fossil_fatal("unknown subcommand \"%s\": should be one of " 1686 "\"export\", \"import\", \"status\"", 1687 zCmd); 1688 } 1689 }