Fossil

Check-in [0a55d162]
Login

Many hyperlinks are disabled.
Use anonymous login to enable hyperlinks.

Overview
Comment:Rebuild faster by caching the last few manifest parses.
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk
Files: files | file ages | folders
SHA1: 0a55d162275a4179ed4376014a4b95f0843d5062
User & Date: drh 2010-10-08 19:55:39
Context
2010-10-09
15:39
merge from trunk check-in: e1d15514 user: wolfgang tags: wolfgangHelpCmd
13:04
Add a compile-time option to ignore the control file checksum (for a modest performance increase while parsing control files.) Enhance the test-parse-manifest command to facilitate performance studies. check-in: 71298828 user: drh tags: trunk
2010-10-08
19:55
Rebuild faster by caching the last few manifest parses. check-in: 0a55d162 user: drh tags: trunk
10:59
Obscure the text of the remote-url password so that it is not easily visible using the sqlite3 CLI. check-in: cfbbad3d user: drh tags: trunk
Changes
Hide Diffs Unified Diffs Ignore Whitespace Patch

Changes to src/manifest.c.

81
82
83
84
85
86
87












88
89
90
91
92
93
94
95
96
97
98
99
100






















































101
102
103
104
105
106
107
...
808
809
810
811
812
813
814

815
816
817
818
819
820
821
822
823
824
825
826
827


828
829

830
831
832
833
834
835
836
...
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
....
1025
1026
1027
1028
1029
1030
1031


1032
1033
1034
1035
1036
1037
1038
1039
....
1261
1262
1263
1264
1265
1266
1267



1268

1269
1270
1271
1272
1273
1274
1275
  struct { 
    char *zName;           /* Key or field name */
    char *zValue;          /* Value of the field */
  } *aField;            /* One for each J card */
};
#endif














/*
** Clear the memory allocated in a manifest object
*/
void manifest_clear(Manifest *p){
  blob_reset(&p->content);
  free(p->aFile);
  free(p->azParent);
  free(p->azCChild);
  free(p->aTag);
  free(p->aField);
  memset(p, 0, sizeof(*p));
}























































/*
** Parse a blob into a Manifest object.  The Manifest object
** takes over the input blob and will free it when the
** Manifest object is freed.  Zeros are inserted into the blob
** as string terminators so that blob should not be used again.
**
................................................................................
** Deleted files have mlink.fid=0.
** Added files have mlink.pid=0.
** Edited files have both mlink.pid!=0 and mlink.fid!=0
*/
static void add_mlink(int pid, Manifest *pParent, int cid, Manifest *pChild){
  Manifest other;
  Blob otherContent;

  int i, j;

  if( db_exists("SELECT 1 FROM mlink WHERE mid=%d", cid) ){
    return;
  }
  assert( pParent==0 || pChild==0 );
  if( pParent==0 ){
    pParent = &other;
    content_get(pid, &otherContent);
  }else{
    pChild = &other;
    content_get(cid, &otherContent);
  }


  if( blob_size(&otherContent)==0 ) return;
  if( manifest_parse(&other, &otherContent)==0 ) return;

  content_deltify(pid, cid, 0);

  /* Use the iRename fields to find the cross-linkage between
  ** renamed files.  */
  for(j=0; j<pChild->nFile; j++){
    const char *zPrior = pChild->aFile[j].zPrior;
    if( zPrior && zPrior[0] ){
................................................................................
      add_one_mlink(cid, pParent->aFile[rn].zUuid, pChild->aFile[j].zUuid, 
                    pChild->aFile[j].zName, pParent->aFile[rn].zName);
    }else{
      add_one_mlink(cid, 0, pChild->aFile[j].zUuid, pChild->aFile[j].zName,0);
    }
    j++;
  }
  manifest_clear(&other);
}

/*
** True if manifest_crosslink_begin() has been called but
** manifest_crosslink_end() is still pending.
*/
static int manifest_crosslink_busy = 0;
................................................................................
*/
int manifest_crosslink(int rid, Blob *pContent){
  int i;
  Manifest m;
  Stmt q;
  int parentid = 0;



  if( manifest_parse(&m, pContent)==0 ){
    return 0;
  }
  if( g.xlinkClusterOnly && m.type!=CFTYPE_CLUSTER ){
    manifest_clear(&m);
    return 0;
  }
  db_begin_transaction();
................................................................................
        "VALUES('t',%.17g,%d,%Q,%Q)",
        m.rDate, rid, m.zUser, zComment
      );
      free(zComment);
    }
  }
  db_end_transaction(0);



  manifest_clear(&m);

  return 1;
}

/*
** Given a checkin name, load and parse the manifest for that checkin.
** Throw a fatal error if anything goes wrong.
*/







>
>
>
>
>
>
>
>
>
>
>
>













>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>
>







 







>








|


|

>
>
|
|
>







 







|







 







>
>
|







 







>
>
>
|
>







81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
...
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
...
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
....
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
....
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
  struct { 
    char *zName;           /* Key or field name */
    char *zValue;          /* Value of the field */
  } *aField;            /* One for each J card */
};
#endif

/*
** A cache of parsed manifests.  This reduces the number of
** calls to manifest_parse() when doing a rebuild.
*/
#define MX_MANIFEST_CACHE 4
static struct {
  int nxAge;
  int aRid[MX_MANIFEST_CACHE];
  int aAge[MX_MANIFEST_CACHE];
  Manifest aLine[MX_MANIFEST_CACHE];
} manifestCache;


/*
** Clear the memory allocated in a manifest object
*/
void manifest_clear(Manifest *p){
  blob_reset(&p->content);
  free(p->aFile);
  free(p->azParent);
  free(p->azCChild);
  free(p->aTag);
  free(p->aField);
  memset(p, 0, sizeof(*p));
}

/*
** Add an element to the manifest cache using LRU replacement.
*/
void manifest_cache_insert(int rid, Manifest *p){
  int i;
  for(i=0; i<MX_MANIFEST_CACHE; i++){
    if( manifestCache.aRid[i]==0 ) break;
  }
  if( i>=MX_MANIFEST_CACHE ){
    int oldest = 0;
    int oldestAge = manifestCache.aAge[0];
    for(i=1; i<MX_MANIFEST_CACHE; i++){
      if( manifestCache.aAge[i]<oldestAge ){
        oldest = i;
        oldestAge = manifestCache.aAge[i];
      }
    }
    manifest_clear(&manifestCache.aLine[oldest]);
    i = oldest;
  }
  manifestCache.aAge[i] = ++manifestCache.nxAge;
  manifestCache.aRid[i] = rid;
  manifestCache.aLine[i] = *p;
}

/*
** Try to extract a line from the manifest cache. Return 1 if found.
** Return 0 if not found.
*/
int manifest_cache_find(int rid, Manifest *p){
  int i;
  for(i=0; i<MX_MANIFEST_CACHE; i++){
    if( manifestCache.aRid[i]==rid ){
      *p = manifestCache.aLine[i];
      manifestCache.aRid[i] = 0;
      return 1;
    }
  }
  return 0;
}

/*
** Clear the manifest cache.
*/
void manifest_cache_clear(void){
  int i;
  for(i=0; i<MX_MANIFEST_CACHE; i++){
    if( manifestCache.aRid[i]>0 ){
      manifest_clear(&manifestCache.aLine[i]);
    }
  }
  memset(&manifestCache, 0, sizeof(manifestCache));
}

/*
** Parse a blob into a Manifest object.  The Manifest object
** takes over the input blob and will free it when the
** Manifest object is freed.  Zeros are inserted into the blob
** as string terminators so that blob should not be used again.
**
................................................................................
** Deleted files have mlink.fid=0.
** Added files have mlink.pid=0.
** Edited files have both mlink.pid!=0 and mlink.fid!=0
*/
static void add_mlink(int pid, Manifest *pParent, int cid, Manifest *pChild){
  Manifest other;
  Blob otherContent;
  int otherRid;
  int i, j;

  if( db_exists("SELECT 1 FROM mlink WHERE mid=%d", cid) ){
    return;
  }
  assert( pParent==0 || pChild==0 );
  if( pParent==0 ){
    pParent = &other;
    otherRid = pid;
  }else{
    pChild = &other;
    otherRid = cid;
  }
  if( manifest_cache_find(otherRid, &other)==0 ){
    content_get(otherRid, &otherContent);
    if( blob_size(&otherContent)==0 ) return;
    if( manifest_parse(&other, &otherContent)==0 ) return;
  }
  content_deltify(pid, cid, 0);

  /* Use the iRename fields to find the cross-linkage between
  ** renamed files.  */
  for(j=0; j<pChild->nFile; j++){
    const char *zPrior = pChild->aFile[j].zPrior;
    if( zPrior && zPrior[0] ){
................................................................................
      add_one_mlink(cid, pParent->aFile[rn].zUuid, pChild->aFile[j].zUuid, 
                    pChild->aFile[j].zName, pParent->aFile[rn].zName);
    }else{
      add_one_mlink(cid, 0, pChild->aFile[j].zUuid, pChild->aFile[j].zName,0);
    }
    j++;
  }
  manifest_cache_insert(otherRid, &other);
}

/*
** True if manifest_crosslink_begin() has been called but
** manifest_crosslink_end() is still pending.
*/
static int manifest_crosslink_busy = 0;
................................................................................
*/
int manifest_crosslink(int rid, Blob *pContent){
  int i;
  Manifest m;
  Stmt q;
  int parentid = 0;

  if( manifest_cache_find(rid, &m) ){
    blob_reset(pContent);
  }else if( manifest_parse(&m, pContent)==0 ){
    return 0;
  }
  if( g.xlinkClusterOnly && m.type!=CFTYPE_CLUSTER ){
    manifest_clear(&m);
    return 0;
  }
  db_begin_transaction();
................................................................................
        "VALUES('t',%.17g,%d,%Q,%Q)",
        m.rDate, rid, m.zUser, zComment
      );
      free(zComment);
    }
  }
  db_end_transaction(0);
  if( m.type==CFTYPE_MANIFEST ){
    manifest_cache_insert(rid, &m);
  }else{
    manifest_clear(&m);
  }
  return 1;
}

/*
** Given a checkin name, load and parse the manifest for that checkin.
** Throw a fatal error if anything goes wrong.
*/