Merge the checkpoint_fullfsync pragma and the superlock demonstration into
the checkpoint-v2 experimental branch.
FossilOrigin-Name: ebf74015f09fe241c1c6902dc8954f2b59ab41ec
diff --git a/main.mk b/main.mk
index 974bbe4..19c1b1c 100644
--- a/main.mk
+++ b/main.mk
@@ -531,8 +531,8 @@
# threadtest runs a few thread-safety tests that are implemented in C. This
# target is invoked by the releasetest.tcl script.
#
-threadtest3$(EXE): sqlite3.c $(TOP)/test/threadtest3.c
- $(TCCX) -O2 sqlite3.c $(TOP)/test/threadtest3.c \
+threadtest3$(EXE): sqlite3.o $(TOP)/test/threadtest3.c $(TOP)/test/tt3_checkpoint.c
+ $(TCCX) -O2 sqlite3.o $(TOP)/test/threadtest3.c \
-o threadtest3$(EXE) $(THREADLIB)
threadtest: threadtest3$(EXE)
diff --git a/manifest b/manifest
index c90d94d..8fc2b20 100644
--- a/manifest
+++ b/manifest
@@ -1,8 +1,8 @@
-----BEGIN PGP SIGNED MESSAGE-----
Hash: SHA1
-C Merge\sin\sthe\ssuperlock\sdemonstration\schanges.
-D 2010-11-19T18:36:45
+C Merge\sthe\scheckpoint_fullfsync\spragma\sand\sthe\ssuperlock\sdemonstration\sinto\nthe\scheckpoint-v2\sexperimental\sbranch.
+D 2010-11-19T18:51:31
F Makefile.arm-wince-mingw32ce-gcc d6df77f1f48d690bd73162294bbba7f59507c72f
F Makefile.in e7a59672eaeb04408d1fa8501618d7501a3c5e39
F Makefile.linux-gcc 91d710bdc4998cb015f39edf3cb314ec4f4d7e23
@@ -102,7 +102,7 @@
F ext/rtree/viewrtree.tcl eea6224b3553599ae665b239bd827e182b466024
F install-sh 9d4de14ab9fb0facae2f48780b874848cbf2f895 x
F ltmain.sh 3ff0879076df340d2e23ae905484d8c15d5fdea8
-F main.mk 05d0f3475dd331896bd607cfb45c5e21b94589ad
+F main.mk 731380a234515bd21ba4f930e12041ed3acfcc3a
F mkdll.sh 7d09b23c05d56532e9d44a50868eb4b12ff4f74a
F mkextu.sh 416f9b7089d80e5590a29692c9d9280a10dbad9f
F mkextw.sh 4123480947681d9b434a5e7b1ee08135abe409ac
@@ -122,8 +122,8 @@
F src/backup.c d5b0137bc20327af08c14772227cc35134839c30
F src/bitvec.c af50f1c8c0ff54d6bdb7a80e2fceca5a93670bef
F src/btmutex.c 96a12f50f7a17475155971a241d85ec5171573ff
-F src/btree.c ccafb8a86e9837daabe89ec590862907a669ecad
-F src/btree.h 10f9296bf4edf034f5adce921b7b4383a56a1c90
+F src/btree.c 3578a5e812ab9b434b1b705aad547939a055be11
+F src/btree.h e2f2cd9933bf30724f53ffa12c4c5a3a864bbd6e
F src/btreeInt.h c424f2f131cc61ddf130f9bd736b3df12c8a51f0
F src/build.c 00a327120d81ace6267e714ae8010c997d55de5d
F src/callback.c a1d1b1c9c85415dff013af033e2fed9c8382d33b
@@ -144,7 +144,7 @@
F src/legacy.c a199d7683d60cef73089e892409113e69c23a99f
F src/lempar.c 7f026423f4d71d989e719a743f98a1cbd4e6d99e
F src/loadext.c 8af9fcc75708d60b88636ccba38b4a7b3c155c3e
-F src/main.c 721e5530b14b91a1fb6aead279af8b8729fa9a20
+F src/main.c 84fdf5b2d823641f6bb394a81e467b6d7ff9b7ec
F src/malloc.c 3d7284cd9346ab6e3945535761e68c23c6cf40ef
F src/mem0.c 6a55ebe57c46ca1a7d98da93aaa07f99f1059645
F src/mem1.c 00bd8265c81abb665c48fea1e0c234eb3b922206
@@ -165,13 +165,13 @@
F src/os_os2.c 72d0b2e562952a2464308c4ce5f7913ac10bef3e
F src/os_unix.c de5be4cdbf3d07018059934eaf7e5d8d594a895c
F src/os_win.c 2f90f7bdec714fad51cd31b4ecad3cc1b4bb5aad
-F src/pager.c a8b36940ca8afcb45224e0017669782b3b2c90a3
-F src/pager.h 0ea59db2a33bc6c2c02cae34de33367e1effdf76
+F src/pager.c b3e86e5d27c04e05a73a55ce2c79bb6028196431
+F src/pager.h 3f8c783de1d4706b40b1ac15b64f5f896bcc78d1
F src/parse.y 12b7ebd61ea54f0e1b1083ff69cc2c8ce9353d58
F src/pcache.c 09d38c44ab275db581f7a2f6ff8b9bc7f8c0faaa
F src/pcache.h c683390d50f856d4cd8e24342ae62027d1bb6050
F src/pcache1.c e9578a3beac26f229ee558a4e16c863f2498185f
-F src/pragma.c 8e87e9e3e8a6734995d22f60dcc8bb838db52436
+F src/pragma.c 900f480cc266428218d1a745a3e670fbd33cf12d
F src/prepare.c c2b318037d626fed27905c9446730b560637217a
F src/printf.c 8ae5082dd38a1b5456030c3755ec3a392cd51506
F src/random.c cd4a67b3953b88019f8cd4ccd81394a8ddfaba50
@@ -179,9 +179,9 @@
F src/rowset.c 69afa95a97c524ba6faf3805e717b5b7ae85a697
F src/select.c 550d67688f5e8bc8022faf6d014838afba1415af
F src/shell.c 8517fc1f9c59ae4007e6cc8b9af91ab231ea2056
-F src/sqlite.h.in e6e87d10e6a3756b8c7e9a11703716b6a1575a40
+F src/sqlite.h.in daa5e94df834537708afb917e9b4cc4489c63c00
F src/sqlite3ext.h c90bd5507099f62043832d73f6425d8d5c5da754
-F src/sqliteInt.h f5b5041bfebd5654212992f6ebaa3f575c4b9c17
+F src/sqliteInt.h eea368b138c8c90ea1aef8f06f49ec29aacb2a8b
F src/sqliteLimit.h a17dcd3fb775d63b64a43a55c54cb282f9726f44
F src/status.c 496913d4e8441195f6f2a75b1c95993a45b9b30b
F src/table.c 2cd62736f845d82200acfa1287e33feb3c15d62e
@@ -231,7 +231,7 @@
F src/utf.c 1baeeac91707a4df97ccc6141ec0f808278af685
F src/util.c cd78524566fe45671863eee78685969a4bfd4e4c
F src/vacuum.c 924bd1bcee2dfb05376f79845bd3b4cec7b54b2f
-F src/vdbe.c 63bb1e56a035bc65b20d6f9c7d7c876f19b4605f
+F src/vdbe.c 06f868b6daff465cd805d09b43661d0e0bee62a7
F src/vdbe.h 4de0efb4b0fdaaa900cf419b35c458933ef1c6d2
F src/vdbeInt.h 7f4cf1b2b69bef3a432b1f23dfebef57275436b4
F src/vdbeapi.c fb0036185b3c56e15916a5ee96309cd4acf6818f
@@ -240,8 +240,8 @@
F src/vdbemem.c 23723a12cd3ba7ab3099193094cbb2eb78956aa9
F src/vdbetrace.c 864cef96919323482ebd9986f2132435115e9cc2
F src/vtab.c b297e8fa656ab5e66244ab15680d68db0adbec30
-F src/wal.c f26b8d297bd11cb792e609917f9d4c6718ac8e0e
-F src/wal.h c1aac6593a0b02b15dc625987e619edeab39292e
+F src/wal.c 23facfd0f148ac72729fe28bbf973fe0458757b6
+F src/wal.h 7a5fbb00114b7f2cd40c7e1003d4c41ce9d26840
F src/walker.c 3112bb3afe1d85dc52317cb1d752055e9a781f8f
F src/where.c fa22d45b2577c77146f2e894d58011d472d64103
F test/aggerror.test a867e273ef9e3d7919f03ef4f0e8c0d2767944f2
@@ -260,7 +260,7 @@
F test/async3.test 93edaa9122f498e56ea98c36c72abc407f4fb11e
F test/async4.test 1787e3952128aa10238bf39945126de7ca23685a
F test/async5.test f3592d79c84d6e83a5f50d3fd500445f7d97dfdf
-F test/attach.test ce9660e51768fab93cf129787be886c5d6c4fd81
+F test/attach.test f2b4ac6931f45695082b9f02be959c9c262e4f4d
F test/attach2.test a295d2d7061adcee5884ef4a93c7c96a82765437
F test/attach3.test bd9830bc3a0d22ed1310c9bff6896927937017dc
F test/attachmalloc.test 1d5b821a676f7bf0b00d87cc106b78966789ba57
@@ -575,7 +575,7 @@
F test/notnull.test cc7c78340328e6112a13c3e311a9ab3127114347
F test/null.test a8b09b8ed87852742343b33441a9240022108993
F test/openv2.test af02ed0a9cbc0d2a61b8f35171d4d117e588e4ec
-F test/pager1.test 07b06b89d50bc38bb118a18b99c7bee645b315de
+F test/pager1.test 1e07368795dc6205a046a7a29f4455a9d5195d19
F test/pager2.test 0fbb6b6dc40ce1fecfe758c555a748ad2e9beaa3
F test/pager3.test 3856d9c80839be0668efee1b74811b1b7f7fc95f
F test/pagerfault.test 9de4d3e0c59970b4c6cb8dac511fa242f335d8a7
@@ -673,7 +673,7 @@
F test/thread_common.tcl 2aa6f2fdcd4d6e461169c3e5ca098eebf643b863
F test/threadtest1.c 6029d9c5567db28e6dc908a0c63099c3ba6c383b
F test/threadtest2.c ace893054fa134af3fc8d6e7cfecddb8e3acefb9
-F test/threadtest3.c d6d209190c7110f9a7e6a8154bdc3260efdbf8b7
+F test/threadtest3.c 0ed13e09690f6204d7455fac3b0e8ece490f6eef
F test/tkt-02a8e81d44.test 58494de77be2cf249228ada3f313fa399821c6ab
F test/tkt-26ff0c2d1e.test 888324e751512972c6e0d1a09df740d8f5aaf660
F test/tkt-2ea2425d34.test 1cf13e6f75d149b3209a0cb32927a82d3d79fb28
@@ -799,6 +799,7 @@
F test/triggerB.test 56780c031b454abac2340dbb3b71ac5c56c3d7fe
F test/triggerC.test 2a23edcc00684d084902ba5ec93e721775c3a70a
F test/triggerD.test c6add3817351451e419f6ff9e9a259b02b6e2de7
+F test/tt3_checkpoint.c 415eccce672d681b297485fc20f44cdf0eac93af
F test/types.test bf816ce73c7dfcfe26b700c19f97ef4050d194ff
F test/types2.test 3555aacf8ed8dc883356e59efc314707e6247a84
F test/types3.test a0f66bf12f80fad89493535474f7a6d16fa58150
@@ -829,11 +830,12 @@
F test/vtab_alter.test 9e374885248f69e251bdaacf480b04a197f125e5
F test/vtab_err.test 0d4d8eb4def1d053ac7c5050df3024fd47a3fbd8
F test/vtab_shared.test 0eff9ce4f19facbe0a3e693f6c14b80711a4222d
-F test/wal.test 70227190e713b3e7eb2a7d5ec3510b66db01f327
-F test/wal2.test c794b8b257af54190bb913678ad3984cbf3311b9
-F test/wal3.test 957a5f2a8fe8a6ff01de1a15285ecf2f376fcaf8
+F test/wal.test f060cae4b2164c4375109a8f803873187234661d
+F test/wal2.test 894d55dda774340fe7bebe239bed9b6130ff23d7
+F test/wal3.test 55529a3fbf0a04670558dbf0b06f04a2f3508db4
F test/wal4.test 3404b048fa5e10605facaf70384e6d2943412e30
-F test/wal_common.tcl 895d76138043b86bdccf36494054bdabcf65837b
+F test/wal5.test 1f99651d856c8b9e1376781c981d1b903e93a478
+F test/wal_common.tcl a98f17fba96206122eff624db0ab13ec377be4fe
F test/walbak.test 4df1c7369da0301caeb9a48fa45997fd592380e4
F test/walbig.test e882bc1d014afffbfa2b6ba36e0f07d30a633ad0
F test/walcksum.test a37b36375c595e61bdb7e1ec49b5f0979b6fc7ce
@@ -892,14 +894,14 @@
F tool/speedtest8.c 2902c46588c40b55661e471d7a86e4dd71a18224
F tool/speedtest8inst1.c 293327bc76823f473684d589a8160bde1f52c14e
F tool/vdbe-compress.tcl d70ea6d8a19e3571d7ab8c9b75cba86d1173ff0f
-P a069867301de3ca2e1753bd4d2e426d27365be4c 1a3e7417a2184188fe21c3284e58720da9ca11cf
-R 612a5c6e6ceeba977718ef175cdd20b9
+P 648dd157ef3b7b790764698fd4dd7107c25212c9 570e79a8eb3bb2d2a15c46c55fbf52c9dd3e3ae8
+R 29ee6d2e4bc9e75a0c4b1469b6aa491c
U drh
-Z f6076b0dc8e47c984cebf6ec68cb33ea
+Z 62a52f8bb2681a24e7959006560a6980
-----BEGIN PGP SIGNATURE-----
Version: GnuPG v1.4.6 (GNU/Linux)
-iD8DBQFM5sO/oxKgR168RlERAjMvAJ46ffI5I7vl/OKgwTBinI+EBDsO5QCeLCGK
-6NmdlpkMD1RVx2Ohr0Mr5i0=
-=Nkm+
+iD8DBQFM5sdNoxKgR168RlERAn8IAJ4xAREF+dTwudzwR6tkh9wg+ffZegCeIyuW
+SnmYKb+i1Y56lE+LONz+l/g=
+=HICj
-----END PGP SIGNATURE-----
diff --git a/manifest.uuid b/manifest.uuid
index f40be04..0da19b3 100644
--- a/manifest.uuid
+++ b/manifest.uuid
@@ -1 +1 @@
-570e79a8eb3bb2d2a15c46c55fbf52c9dd3e3ae8
\ No newline at end of file
+ebf74015f09fe241c1c6902dc8954f2b59ab41ec
\ No newline at end of file
diff --git a/src/btree.c b/src/btree.c
index f5c9b18..bc8510b 100644
--- a/src/btree.c
+++ b/src/btree.c
@@ -7941,8 +7941,10 @@
**
** Return SQLITE_LOCKED if this or any other connection has an open
** transaction on the shared-cache the argument Btree is connected to.
+**
+** Parameter eMode is one of SQLITE_CHECKPOINT_PASSIVE, FULL or RESTART.
*/
-int sqlite3BtreeCheckpoint(Btree *p){
+int sqlite3BtreeCheckpoint(Btree *p, int eMode, int *pnLog, int *pnCkpt){
int rc = SQLITE_OK;
if( p ){
BtShared *pBt = p->pBt;
@@ -7950,7 +7952,7 @@
if( pBt->inTransaction!=TRANS_NONE ){
rc = SQLITE_LOCKED;
}else{
- rc = sqlite3PagerCheckpoint(pBt->pPager);
+ rc = sqlite3PagerCheckpoint(pBt->pPager, eMode, pnLog, pnCkpt);
}
sqlite3BtreeLeave(p);
}
diff --git a/src/btree.h b/src/btree.h
index 90fa7a2..6886dd9 100644
--- a/src/btree.h
+++ b/src/btree.h
@@ -207,7 +207,7 @@
#endif
#ifndef SQLITE_OMIT_WAL
- int sqlite3BtreeCheckpoint(Btree*);
+ int sqlite3BtreeCheckpoint(Btree*, int, int *, int *);
#endif
/*
diff --git a/src/main.c b/src/main.c
index f9e1a1c..649fcab 100644
--- a/src/main.c
+++ b/src/main.c
@@ -1340,19 +1340,29 @@
#endif
}
-
/*
-** Checkpoint database zDb. If zDb is NULL, or if the buffer zDb points
-** to contains a zero-length string, all attached databases are
-** checkpointed.
+** Checkpoint database zDb.
*/
-int sqlite3_wal_checkpoint(sqlite3 *db, const char *zDb){
+int sqlite3_wal_checkpoint_v2(
+ sqlite3 *db, /* Database handle */
+ const char *zDb, /* Name of attached database (or NULL) */
+ int eMode, /* SQLITE_CHECKPOINT_* value */
+ int *pnLog, /* OUT: Size of WAL log in frames */
+ int *pnCkpt /* OUT: Total number of frames checkpointed */
+){
#ifdef SQLITE_OMIT_WAL
return SQLITE_OK;
#else
int rc; /* Return code */
int iDb = SQLITE_MAX_ATTACHED; /* sqlite3.aDb[] index of db to checkpoint */
+ if( eMode!=SQLITE_CHECKPOINT_PASSIVE
+ && eMode!=SQLITE_CHECKPOINT_FULL
+ && eMode!=SQLITE_CHECKPOINT_RESTART
+ ){
+ return SQLITE_MISUSE;
+ }
+
sqlite3_mutex_enter(db->mutex);
if( zDb && zDb[0] ){
iDb = sqlite3FindDbName(db, zDb);
@@ -1361,7 +1371,7 @@
rc = SQLITE_ERROR;
sqlite3Error(db, SQLITE_ERROR, "unknown database: %s", zDb);
}else{
- rc = sqlite3Checkpoint(db, iDb);
+ rc = sqlite3Checkpoint(db, iDb, eMode, pnLog, pnCkpt);
sqlite3Error(db, rc, 0);
}
rc = sqlite3ApiExit(db, rc);
@@ -1370,6 +1380,16 @@
#endif
}
+
+/*
+** Checkpoint database zDb. If zDb is NULL, or if the buffer zDb points
+** to contains a zero-length string, all attached databases are
+** checkpointed.
+*/
+int sqlite3_wal_checkpoint(sqlite3 *db, const char *zDb){
+ return sqlite3_wal_checkpoint_v2(db, zDb, SQLITE_CHECKPOINT_PASSIVE, 0, 0);
+}
+
#ifndef SQLITE_OMIT_WAL
/*
** Run a checkpoint on database iDb. This is a no-op if database iDb is
@@ -1387,20 +1407,29 @@
** If iDb is passed SQLITE_MAX_ATTACHED, then all attached databases are
** checkpointed. If an error is encountered it is returned immediately -
** no attempt is made to checkpoint any remaining databases.
+**
+** Parameter eMode is one of SQLITE_CHECKPOINT_PASSIVE, FULL or RESTART.
*/
-int sqlite3Checkpoint(sqlite3 *db, int iDb){
+int sqlite3Checkpoint(sqlite3 *db, int iDb, int eMode, int *pnLog, int *pnCkpt){
int rc = SQLITE_OK; /* Return code */
int i; /* Used to iterate through attached dbs */
+ int bBusy = 0; /* True if SQLITE_BUSY has been encountered */
assert( sqlite3_mutex_held(db->mutex) );
for(i=0; i<db->nDb && rc==SQLITE_OK; i++){
if( i==iDb || iDb==SQLITE_MAX_ATTACHED ){
- rc = sqlite3BtreeCheckpoint(db->aDb[i].pBt);
+ rc = sqlite3BtreeCheckpoint(db->aDb[i].pBt, eMode, pnLog, pnCkpt);
+ pnLog = 0;
+ pnCkpt = 0;
+ if( rc==SQLITE_BUSY ){
+ bBusy = 1;
+ rc = SQLITE_OK;
+ }
}
}
- return rc;
+ return (rc==SQLITE_OK && bBusy) ? SQLITE_BUSY : rc;
}
#endif /* SQLITE_OMIT_WAL */
diff --git a/src/pager.c b/src/pager.c
index a95606e..e0230c6 100644
--- a/src/pager.c
+++ b/src/pager.c
@@ -6550,14 +6550,20 @@
#ifndef SQLITE_OMIT_WAL
/*
-** This function is called when the user invokes "PRAGMA checkpoint".
+** This function is called when the user invokes "PRAGMA wal_checkpoint",
+** "PRAGMA wal_blocking_checkpoint" or calls the sqlite3_wal_checkpoint()
+** or wal_blocking_checkpoint() API functions.
+**
+** Parameter eMode is one of SQLITE_CHECKPOINT_PASSIVE, FULL or RESTART.
*/
-int sqlite3PagerCheckpoint(Pager *pPager){
+int sqlite3PagerCheckpoint(Pager *pPager, int eMode, int *pnLog, int *pnCkpt){
int rc = SQLITE_OK;
if( pPager->pWal ){
- u8 *zBuf = (u8 *)pPager->pTmpSpace;
- rc = sqlite3WalCheckpoint(pPager->pWal, pPager->ckptSyncFlags,
- pPager->pageSize, zBuf);
+ rc = sqlite3WalCheckpoint(pPager->pWal, eMode,
+ pPager->xBusyHandler, pPager->pBusyHandlerArg,
+ pPager->ckptSyncFlags, pPager->pageSize, (u8 *)pPager->pTmpSpace,
+ pnLog, pnCkpt
+ );
}
return rc;
}
diff --git a/src/pager.h b/src/pager.h
index e775b0c..eab7dda 100644
--- a/src/pager.h
+++ b/src/pager.h
@@ -138,7 +138,7 @@
int sqlite3PagerSavepoint(Pager *pPager, int op, int iSavepoint);
int sqlite3PagerSharedLock(Pager *pPager);
-int sqlite3PagerCheckpoint(Pager *pPager);
+int sqlite3PagerCheckpoint(Pager *pPager, int, int*, int*);
int sqlite3PagerWalSupported(Pager *pPager);
int sqlite3PagerWalCallback(Pager *pPager);
int sqlite3PagerOpenWal(Pager *pPager, int *pisOpen);
diff --git a/src/pragma.c b/src/pragma.c
index 7c600cd..721a13b 100644
--- a/src/pragma.c
+++ b/src/pragma.c
@@ -1394,13 +1394,29 @@
#ifndef SQLITE_OMIT_WAL
/*
- ** PRAGMA [database.]wal_checkpoint
+ ** PRAGMA [database.]wal_checkpoint = passive|full|restart
**
** Checkpoint the database.
*/
if( sqlite3StrICmp(zLeft, "wal_checkpoint")==0 ){
+ int iBt = (pId2->z?iDb:SQLITE_MAX_ATTACHED);
+ int eMode = SQLITE_CHECKPOINT_PASSIVE;
+ if( zRight ){
+ if( sqlite3StrICmp(zRight, "full")==0 ){
+ eMode = SQLITE_CHECKPOINT_FULL;
+ }else if( sqlite3StrICmp(zRight, "restart")==0 ){
+ eMode = SQLITE_CHECKPOINT_RESTART;
+ }
+ }
if( sqlite3ReadSchema(pParse) ) goto pragma_out;
- sqlite3VdbeAddOp3(v, OP_Checkpoint, pId2->z?iDb:SQLITE_MAX_ATTACHED, 0, 0);
+ sqlite3VdbeSetNumCols(v, 3);
+ pParse->nMem = 3;
+ sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "busy", SQLITE_STATIC);
+ sqlite3VdbeSetColName(v, 1, COLNAME_NAME, "log", SQLITE_STATIC);
+ sqlite3VdbeSetColName(v, 2, COLNAME_NAME, "checkpointed", SQLITE_STATIC);
+
+ sqlite3VdbeAddOp2(v, OP_Checkpoint, iBt, eMode);
+ sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 3);
}else
/*
diff --git a/src/sqlite.h.in b/src/sqlite.h.in
index 5b67b7a..dadc589 100644
--- a/src/sqlite.h.in
+++ b/src/sqlite.h.in
@@ -6180,6 +6180,89 @@
int sqlite3_wal_checkpoint(sqlite3 *db, const char *zDb);
/*
+**
+** CAPI3REF: Checkpoint a database
+**
+** Run a checkpoint operation on WAL database zDb attached to database
+** handle db. The specific operation is determined by the value of the
+** eMode parameter:
+**
+** <dl>
+** <dt>SQLITE_CHECKPOINT_PASSIVE<dd>
+** Checkpoint as many frames as possible without waiting for any database
+** readers or writers to finish. Sync the db file if all frames in the log
+** are checkpointed. This mode is the same as calling
+** sqlite3_wal_checkpoint(). The busy-handler callback is never invoked.
+**
+** <dt>SQLITE_CHECKPOINT_FULL<dd>
+** This mode blocks (calls the busy-handler callback) until there is no
+** database writer and all readers are reading from the most recent database
+** snapshot. It then checkpoints all frames in the log file and syncs the
+** database file. This call blocks database writers while it is running,
+** but not database readers.
+**
+** <dt>SQLITE_CHECKPOINT_RESTART<dd>
+** This mode works the same way as SQLITE_CHECKPOINT_FULL, except after
+** checkpointing the log file it blocks (calls the busy-handler callback)
+** until all readers are reading from the database file only. This ensures
+** that the next client to write to the database file restarts the log file
+** from the beginning. This call blocks database writers while it is running,
+** but not database readers.
+** </dl>
+**
+** If pnLog is not NULL, then *pnLog is set to the total number of frames in
+** the log file before returning. If pnCkpt is not NULL, then *pnCkpt is set to
+** the total number of checkpointed frames (including any that were already
+** checkpointed when this function is called). *pnLog and *pnCkpt may be
+** populated even if sqlite3_wal_checkpoint_v2() returns other than SQLITE_OK.
+** If no values are available because of an error, they are both set to -1
+** before returning to communicate this to the caller.
+**
+** All calls obtain an exclusive "checkpoint" lock on the database file. If
+** any other process is running a checkpoint operation at the same time, the
+** lock cannot be obtained and SQLITE_BUSY is returned. Even if there is a
+** busy-handler configured, it will not be invoked in this case.
+**
+** The SQLITE_CHECKPOINT_FULL and RESTART modes also obtain the exclusive
+** "writer" lock on the database file. If the writer lock cannot be obtained
+** immediately, and a busy-handler is configured, it is invoked and the writer
+** lock retried until either the busy-handler returns 0 or the lock is
+** successfully obtained. The busy-handler is also invoked while waiting for
+** database readers as described above. If the busy-handler returns 0 before
+** the writer lock is obtained or while waiting for database readers, the
+** checkpoint operation proceeds from that point in the same way as
+** SQLITE_CHECKPOINT_PASSIVE - checkpointing as many frames as possible
+** without blocking any further. SQLITE_BUSY is returned in this case.
+**
+** If parameter zDb is NULL or points to a zero length string, then the
+** specified operation is attempted on all WAL databases. In this case the
+** values written to output parameters *pnLog and *pnCkpt are undefined. If
+** an SQLITE_BUSY error is encountered when processing one or more of the
+** attached WAL databases, the operation is still attempted on any remaining
+** attached databases and SQLITE_BUSY is returned to the caller. If any other
+** error occurs while processing an attached database, processing is abandoned
+** and the error code returned to the caller immediately. If no error
+** (SQLITE_BUSY or otherwise) is encountered while processing the attached
+** databases, SQLITE_OK is returned.
+**
+** If database zDb is the name of an attached database that is not in WAL
+** mode, SQLITE_OK is returned and both *pnLog and *pnCkpt set to -1. If
+** zDb is not NULL (or a zero length string) and is not the name of any
+** attached database, SQLITE_ERROR is returned to the caller.
+*/
+int sqlite3_wal_checkpoint_v2(
+ sqlite3 *db, /* Database handle */
+ const char *zDb, /* Name of attached database (or NULL) */
+ int eMode, /* SQLITE_CHECKPOINT_* value */
+ int *pnLog, /* OUT: Size of WAL log in frames */
+ int *pnCkpt /* OUT: Total number of frames checkpointed */
+);
+#define SQLITE_CHECKPOINT_PASSIVE 0
+#define SQLITE_CHECKPOINT_FULL 1
+#define SQLITE_CHECKPOINT_RESTART 2
+
+
+/*
** Undo the hack that converts floating point types to integer for
** builds on processors without floating point support.
*/
diff --git a/src/sqliteInt.h b/src/sqliteInt.h
index 9f26dc5..0d65cd6 100644
--- a/src/sqliteInt.h
+++ b/src/sqliteInt.h
@@ -3037,7 +3037,7 @@
int sqlite3TempInMemory(const sqlite3*);
VTable *sqlite3GetVTable(sqlite3*, Table*);
const char *sqlite3JournalModename(int);
-int sqlite3Checkpoint(sqlite3*, int);
+int sqlite3Checkpoint(sqlite3*, int, int, int*, int*);
int sqlite3WalDefaultHook(void*,sqlite3*,const char*,int);
/* Declarations for functions in fkey.c. All of these are replaced by
diff --git a/src/vdbe.c b/src/vdbe.c
index cfcb15b..b51e1a6 100644
--- a/src/vdbe.c
+++ b/src/vdbe.c
@@ -5216,13 +5216,33 @@
}
#ifndef SQLITE_OMIT_WAL
-/* Opcode: Checkpoint P1 * * * *
+/* Opcode: Checkpoint P1 P2 P3 * *
**
** Checkpoint database P1. This is a no-op if P1 is not currently in
-** WAL mode.
+** WAL mode. Parameter P2 is one of SQLITE_CHECKPOINT_PASSIVE, FULL
+** or RESTART.
*/
case OP_Checkpoint: {
- rc = sqlite3Checkpoint(db, pOp->p1);
+ int nLog = -1; /* Number of pages in WAL log */
+ int nCkpt = -1; /* Number of checkpointed pages */
+ int bBusy = 0;
+ assert( pOp->p2==SQLITE_CHECKPOINT_PASSIVE
+ || pOp->p2==SQLITE_CHECKPOINT_FULL
+ || pOp->p2==SQLITE_CHECKPOINT_RESTART
+ );
+ rc = sqlite3Checkpoint(db, pOp->p1, pOp->p2, &nLog, &nCkpt);
+ if( rc==SQLITE_BUSY ){
+ rc = SQLITE_OK;
+ bBusy = 1;
+ }
+
+ aMem[1].u.i = bBusy;
+ aMem[2].u.i = nLog;
+ aMem[3].u.i = nCkpt;
+ MemSetTypeFlag(&aMem[1], MEM_Int);
+ MemSetTypeFlag(&aMem[2], MEM_Int);
+ MemSetTypeFlag(&aMem[3], MEM_Int);
+
break;
};
#endif
diff --git a/src/wal.c b/src/wal.c
index 3b21790..2cb0e41 100644
--- a/src/wal.c
+++ b/src/wal.c
@@ -1524,6 +1524,34 @@
}
/*
+** Attempt to obtain the exclusive WAL lock defined by parameters lockIdx and
+** n. If the attempt fails and parameter xBusy is not NULL, then it is a
+** busy-handler function. Invoke it and retry the lock until either the
+** lock is successfully obtained or the busy-handler returns 0.
+*/
+static int walBusyLock(
+ Wal *pWal, /* WAL connection */
+ int (*xBusy)(void*), /* Function to call when busy */
+ void *pBusyArg, /* Context argument for xBusyHandler */
+ int lockIdx, /* Offset of first byte to lock */
+ int n /* Number of bytes to lock */
+){
+ int rc;
+ do {
+ rc = walLockExclusive(pWal, lockIdx, n);
+ }while( xBusy && rc==SQLITE_BUSY && xBusy(pBusyArg) );
+ return rc;
+}
+
+/*
+** The cache of the wal-index header must be valid to call this function.
+** Return the page-size in bytes used by the database.
+*/
+static int walPagesize(Wal *pWal){
+ return (pWal->hdr.szPage&0xfe00) + ((pWal->hdr.szPage&0x0001)<<16);
+}
+
+/*
** Copy as much content as we can from the WAL back into the database file
** in response to an sqlite3_wal_checkpoint() request or the equivalent.
**
@@ -1556,9 +1584,12 @@
*/
static int walCheckpoint(
Wal *pWal, /* Wal connection */
+ int eMode, /* One of PASSIVE, FULL or RESTART */
+ int (*xBusyCall)(void*), /* Function to call when busy */
+ void *pBusyArg, /* Context argument for xBusyHandler */
int sync_flags, /* Flags for OsSync() (or 0) */
- int nBuf, /* Size of zBuf in bytes */
- u8 *zBuf /* Temporary buffer to use */
+ u8 *zBuf, /* Temporary buffer to use */
+ int *pnCkpt /* Total frames checkpointed */
){
int rc; /* Return code */
int szPage; /* Database page-size */
@@ -1569,11 +1600,11 @@
u32 mxPage; /* Max database page to write */
int i; /* Loop counter */
volatile WalCkptInfo *pInfo; /* The checkpoint status information */
+ int (*xBusy)(void*) = 0; /* Function to call when waiting for locks */
- szPage = (pWal->hdr.szPage&0xfe00) + ((pWal->hdr.szPage&0x0001)<<16);
+ szPage = walPagesize(pWal);
testcase( szPage<=32768 );
testcase( szPage>=65536 );
- if( pWal->hdr.mxFrame==0 ) return SQLITE_OK;
/* Allocate the iterator */
rc = walIteratorInit(pWal, &pIter);
@@ -1582,11 +1613,10 @@
}
assert( pIter );
- /*** TODO: Move this test out to the caller. Make it an assert() here ***/
- if( szPage!=nBuf ){
- rc = SQLITE_CORRUPT_BKPT;
- goto walcheckpoint_out;
- }
+ pInfo = walCkptInfo(pWal);
+ mxPage = pWal->hdr.nPage;
+ if( pnCkpt ) *pnCkpt = pInfo->nBackfill;
+ if( eMode!=SQLITE_CHECKPOINT_PASSIVE ) xBusy = xBusyCall;
/* Compute in mxSafeFrame the index of the last frame of the WAL that is
** safe to write into the database. Frames beyond mxSafeFrame might
@@ -1594,18 +1624,17 @@
** cannot be backfilled from the WAL.
*/
mxSafeFrame = pWal->hdr.mxFrame;
- mxPage = pWal->hdr.nPage;
- pInfo = walCkptInfo(pWal);
for(i=1; i<WAL_NREADER; i++){
u32 y = pInfo->aReadMark[i];
- if( mxSafeFrame>=y ){
+ if( mxSafeFrame>y ){
assert( y<=pWal->hdr.mxFrame );
- rc = walLockExclusive(pWal, WAL_READ_LOCK(i), 1);
+ rc = walBusyLock(pWal, xBusy, pBusyArg, WAL_READ_LOCK(i), 1);
if( rc==SQLITE_OK ){
pInfo->aReadMark[i] = READMARK_NOT_USED;
walUnlockExclusive(pWal, WAL_READ_LOCK(i), 1);
}else if( rc==SQLITE_BUSY ){
mxSafeFrame = y;
+ xBusy = 0;
}else{
goto walcheckpoint_out;
}
@@ -1613,7 +1642,7 @@
}
if( pInfo->nBackfill<mxSafeFrame
- && (rc = walLockExclusive(pWal, WAL_READ_LOCK(0), 1))==SQLITE_OK
+ && (rc = walBusyLock(pWal, xBusy, pBusyArg, WAL_READ_LOCK(0), 1))==SQLITE_OK
){
i64 nSize; /* Current size of database file */
u32 nBackfill = pInfo->nBackfill;
@@ -1661,18 +1690,38 @@
}
if( rc==SQLITE_OK ){
pInfo->nBackfill = mxSafeFrame;
+ if( pnCkpt ) *pnCkpt = mxSafeFrame;
}
}
/* Release the reader lock held while backfilling */
walUnlockExclusive(pWal, WAL_READ_LOCK(0), 1);
- }else if( rc==SQLITE_BUSY ){
+ }
+
+ if( rc==SQLITE_BUSY ){
/* Reset the return code so as not to report a checkpoint failure
- ** just because active readers prevent any backfill.
- */
+ ** just because there are active readers. */
rc = SQLITE_OK;
}
+ /* If this is an SQLITE_CHECKPOINT_RESTART operation, and the entire wal
+ ** file has been copied into the database file, then block until all
+ ** readers have finished using the wal file. This ensures that the next
+ ** process to write to the database restarts the wal file.
+ */
+ if( rc==SQLITE_OK && eMode!=SQLITE_CHECKPOINT_PASSIVE ){
+ assert( pWal->writeLock );
+ if( pInfo->nBackfill<pWal->hdr.mxFrame ){
+ rc = SQLITE_BUSY;
+ }else if( eMode==SQLITE_CHECKPOINT_RESTART ){
+ assert( mxSafeFrame==pWal->hdr.mxFrame );
+ rc = walBusyLock(pWal, xBusy, pBusyArg, WAL_READ_LOCK(1), WAL_NREADER-1);
+ if( rc==SQLITE_OK ){
+ walUnlockExclusive(pWal, WAL_READ_LOCK(1), WAL_NREADER-1);
+ }
+ }
+ }
+
walcheckpoint_out:
walIteratorFree(pIter);
return rc;
@@ -1704,7 +1753,9 @@
if( pWal->exclusiveMode==WAL_NORMAL_MODE ){
pWal->exclusiveMode = WAL_EXCLUSIVE_MODE;
}
- rc = sqlite3WalCheckpoint(pWal, sync_flags, nBuf, zBuf);
+ rc = sqlite3WalCheckpoint(
+ pWal, SQLITE_CHECKPOINT_PASSIVE, 0, 0, sync_flags, nBuf, zBuf, 0, 0
+ );
if( rc==SQLITE_OK ){
isDelete = 1;
}
@@ -2619,17 +2670,27 @@
**
** Obtain a CHECKPOINT lock and then backfill as much information as
** we can from WAL into the database.
+**
+** If parameter xBusy is not NULL, it is a pointer to a busy-handler
+** callback. In this case this function runs a blocking checkpoint.
*/
int sqlite3WalCheckpoint(
Wal *pWal, /* Wal connection */
+ int eMode, /* PASSIVE, FULL or RESTART */
+ int (*xBusy)(void*), /* Function to call when busy */
+ void *pBusyArg, /* Context argument for xBusyHandler */
int sync_flags, /* Flags to sync db file with (or 0) */
int nBuf, /* Size of temporary buffer */
- u8 *zBuf /* Temporary buffer to use */
+ u8 *zBuf, /* Temporary buffer to use */
+ int *pnLog, /* OUT: Number of frames in WAL */
+ int *pnCkpt /* OUT: Number of backfilled frames in WAL */
){
int rc; /* Return code */
int isChanged = 0; /* True if a new wal-index header is loaded */
+ int eMode2 = eMode; /* Mode to pass to walCheckpoint() */
assert( pWal->ckptLock==0 );
+ assert( pWal->writeLock==0 );
WALTRACE(("WAL%p: checkpoint begins\n", pWal));
rc = walLockExclusive(pWal, WAL_CKPT_LOCK, 1);
@@ -2641,11 +2702,40 @@
}
pWal->ckptLock = 1;
- /* Copy data from the log to the database file. */
- rc = walIndexReadHdr(pWal, &isChanged);
- if( rc==SQLITE_OK ){
- rc = walCheckpoint(pWal, sync_flags, nBuf, zBuf);
+ /* If this is a blocking-checkpoint, then obtain the write-lock as well
+ ** to prevent any writers from running while the checkpoint is underway.
+ ** This has to be done before the call to walIndexReadHdr() below.
+ **
+ ** If the writer lock cannot be obtained, then a passive checkpoint is
+ ** run instead. Since the checkpointer is not holding the writer lock,
+ ** there is no point in blocking waiting for any readers. Assuming no
+ ** other error occurs, this function will return SQLITE_BUSY to the caller.
+ */
+ if( eMode!=SQLITE_CHECKPOINT_PASSIVE ){
+ rc = walBusyLock(pWal, xBusy, pBusyArg, WAL_WRITE_LOCK, 1);
+ if( rc==SQLITE_OK ){
+ pWal->writeLock = 1;
+ }else if( rc==SQLITE_BUSY ){
+ eMode2 = SQLITE_CHECKPOINT_PASSIVE;
+ rc = SQLITE_OK;
+ }
}
+
+ /* Read the wal-index header. */
+ if( rc==SQLITE_OK ){
+ rc = walIndexReadHdr(pWal, &isChanged);
+ }
+
+ /* Copy data from the log to the database file. */
+ if( rc==SQLITE_OK && pWal->hdr.mxFrame ){
+ if( walPagesize(pWal)!=nBuf ){
+ rc = SQLITE_CORRUPT_BKPT;
+ }else{
+ if( pnLog ) *pnLog = (int)pWal->hdr.mxFrame;
+ rc = walCheckpoint(pWal, eMode2, xBusy, pBusyArg, sync_flags,zBuf,pnCkpt);
+ }
+ }
+
if( isChanged ){
/* If a new wal-index header was loaded before the checkpoint was
** performed, then the pager-cache associated with pWal is now
@@ -2657,10 +2747,11 @@
}
/* Release the locks. */
+ sqlite3WalEndWriteTransaction(pWal);
walUnlockExclusive(pWal, WAL_CKPT_LOCK, 1);
pWal->ckptLock = 0;
WALTRACE(("WAL%p: checkpoint %s\n", pWal, rc ? "failed" : "ok"));
- return rc;
+ return (rc==SQLITE_OK && eMode!=eMode2 ? SQLITE_BUSY : rc);
}
/* Return the value to pass to a sqlite3_wal_hook callback, the
diff --git a/src/wal.h b/src/wal.h
index 35f695c..2039c70 100644
--- a/src/wal.h
+++ b/src/wal.h
@@ -20,22 +20,22 @@
#include "sqliteInt.h"
#ifdef SQLITE_OMIT_WAL
-# define sqlite3WalOpen(x,y,z) 0
-# define sqlite3WalClose(w,x,y,z) 0
-# define sqlite3WalBeginReadTransaction(y,z) 0
+# define sqlite3WalOpen(x,y,z) 0
+# define sqlite3WalClose(w,x,y,z) 0
+# define sqlite3WalBeginReadTransaction(y,z) 0
# define sqlite3WalEndReadTransaction(z)
-# define sqlite3WalRead(v,w,x,y,z) 0
-# define sqlite3WalDbsize(y) 0
-# define sqlite3WalBeginWriteTransaction(y) 0
-# define sqlite3WalEndWriteTransaction(x) 0
-# define sqlite3WalUndo(x,y,z) 0
+# define sqlite3WalRead(v,w,x,y,z) 0
+# define sqlite3WalDbsize(y) 0
+# define sqlite3WalBeginWriteTransaction(y) 0
+# define sqlite3WalEndWriteTransaction(x) 0
+# define sqlite3WalUndo(x,y,z) 0
# define sqlite3WalSavepoint(y,z)
-# define sqlite3WalSavepointUndo(y,z) 0
-# define sqlite3WalFrames(u,v,w,x,y,z) 0
-# define sqlite3WalCheckpoint(u,v,w,x) 0
-# define sqlite3WalCallback(z) 0
-# define sqlite3WalExclusiveMode(y,z) 0
-# define sqlite3WalHeapMemory(z) 0
+# define sqlite3WalSavepointUndo(y,z) 0
+# define sqlite3WalFrames(u,v,w,x,y,z) 0
+# define sqlite3WalCheckpoint(r,s,t,u,v,w,x,y,z) 0
+# define sqlite3WalCallback(z) 0
+# define sqlite3WalExclusiveMode(y,z) 0
+# define sqlite3WalHeapMemory(z) 0
#else
#define WAL_SAVEPOINT_NDATA 4
@@ -86,9 +86,14 @@
/* Copy pages from the log to the database file */
int sqlite3WalCheckpoint(
Wal *pWal, /* Write-ahead log connection */
+ int eMode, /* One of PASSIVE, FULL and RESTART */
+ int (*xBusy)(void*), /* Function to call when busy */
+ void *pBusyArg, /* Context argument for xBusyHandler */
int sync_flags, /* Flags to sync db file with (or 0) */
int nBuf, /* Size of buffer nBuf */
- u8 *zBuf /* Temporary buffer to use */
+ u8 *zBuf, /* Temporary buffer to use */
+ int *pnLog, /* OUT: Number of frames in WAL */
+ int *pnCkpt /* OUT: Number of backfilled frames in WAL */
);
/* Return the value to pass to a sqlite3_wal_hook callback, the
diff --git a/test/attach.test b/test/attach.test
index 61e1272..a5af8f7 100644
--- a/test/attach.test
+++ b/test/attach.test
@@ -835,4 +835,5 @@
PRAGMA database_list;
}] 9 end
} {4 noname {} 5 inmem {}}
+
finish_test
diff --git a/test/pager1.test b/test/pager1.test
index dfc49af..79a04ca 100644
--- a/test/pager1.test
+++ b/test/pager1.test
@@ -1992,7 +1992,7 @@
INSERT INTO ko DEFAULT VALUES;
}
execsql { PRAGMA wal_checkpoint }
-} {}
+} {0 -1 -1}
do_test pager1-22.2.1 {
testvfs tv -default 1
tv filter xSync
diff --git a/test/threadtest3.c b/test/threadtest3.c
index 82b4708..cb7e2fa 100644
--- a/test/threadtest3.c
+++ b/test/threadtest3.c
@@ -1394,6 +1394,7 @@
print_and_free_err(&err);
}
+#include "tt3_checkpoint.c"
int main(int argc, char **argv){
struct ThreadTest {
@@ -1408,8 +1409,11 @@
{ walthread5, "walthread5", 1000 },
{ walthread5, "walthread5", 1000 },
- { cgt_pager_1, "cgt_pager_1", 0 },
+ { cgt_pager_1, "cgt_pager_1", 0 },
{ dynamic_triggers, "dynamic_triggers", 20000 },
+
+ { checkpoint_starvation_1, "checkpoint_starvation_1", 10000 },
+ { checkpoint_starvation_2, "checkpoint_starvation_2", 10000 },
};
int i;
diff --git a/test/tt3_checkpoint.c b/test/tt3_checkpoint.c
new file mode 100644
index 0000000..3c28f0d
--- /dev/null
+++ b/test/tt3_checkpoint.c
@@ -0,0 +1,150 @@
+/*
+** 2001 September 15
+**
+** The author disclaims copyright to this source code. In place of
+** a legal notice, here is a blessing:
+**
+** May you do good and not evil.
+** May you find forgiveness for yourself and forgive others.
+** May you share freely, never taking more than you give.
+**
+*************************************************************************
+** This file is part of the test program "threadtest3". Despite being a C
+** file it is not compiled separately, but included by threadtest3.c using
+** the #include directive normally used with header files.
+**
+** This file contains the implementation of test cases:
+**
+** checkpoint_starvation_1
+** checkpoint_starvation_2
+*/
+
+/*
+** Both test cases involve 1 writer/checkpointer thread and N reader threads.
+**
+** Each reader thread performs a series of read transactions, one after
+** another. Each read transaction lasts for 100 ms.
+**
+** The writer writes transactions as fast as possible. It uses a callback
+** registered with sqlite3_wal_hook() to try to keep the WAL-size limited to
+** around 50 pages.
+**
+** In test case checkpoint_starvation_1, the auto-checkpoint uses
+** SQLITE_CHECKPOINT_PASSIVE. In checkpoint_starvation_2, it uses RESTART.
+** The expectation is that in the first case the WAL file will grow very
+** large, and in the second will be limited to the 50 pages or thereabouts.
+** However, the overall transaction throughput will be lower for
+** checkpoint_starvation_2, as every checkpoint will block for up to 200 ms
+** waiting for readers to clear.
+*/
+
+/* Frame limit used by the WAL hook for these tests. */
+#define CHECKPOINT_STARVATION_FRAMELIMIT 50
+
+/* Duration in ms of each read transaction */
+#define CHECKPOINT_STARVATION_READMS 100
+
+struct CheckpointStarvationCtx {
+ int eMode;
+ int nMaxFrame;
+};
+typedef struct CheckpointStarvationCtx CheckpointStarvationCtx;
+
+static int checkpoint_starvation_walhook(
+ void *pCtx,
+ sqlite3 *db,
+ const char *zDb,
+ int nFrame
+){
+ CheckpointStarvationCtx *p = (CheckpointStarvationCtx *)pCtx;
+ if( nFrame>p->nMaxFrame ){
+ p->nMaxFrame = nFrame;
+ }
+ if( nFrame>=CHECKPOINT_STARVATION_FRAMELIMIT ){
+ sqlite3_wal_checkpoint_v2(db, zDb, p->eMode, 0, 0);
+ }
+ return SQLITE_OK;
+}
+
+static char *checkpoint_starvation_reader(int iTid, int iArg){
+ Error err = {0};
+ Sqlite db = {0};
+
+ opendb(&err, &db, "test.db", 0);
+ while( !timetostop(&err) ){
+ i64 iCount1, iCount2;
+ sql_script(&err, &db, "BEGIN");
+ iCount1 = execsql_i64(&err, &db, "SELECT count(x) FROM t1");
+ usleep(CHECKPOINT_STARVATION_READMS*1000);
+ iCount2 = execsql_i64(&err, &db, "SELECT count(x) FROM t1");
+ sql_script(&err, &db, "COMMIT");
+
+ if( iCount1!=iCount2 ){
+ test_error(&err, "Isolation failure - %lld %lld", iCount1, iCount2);
+ }
+ }
+ closedb(&err, &db);
+
+ print_and_free_err(&err);
+ return 0;
+}
+
+static void checkpoint_starvation_main(int nMs, CheckpointStarvationCtx *p){
+ Error err = {0};
+ Sqlite db = {0};
+ Threadset threads = {0};
+ int nInsert = 0;
+ int i;
+
+ opendb(&err, &db, "test.db", 1);
+ sql_script(&err, &db,
+ "PRAGMA page_size = 1024;"
+ "PRAGMA journal_mode = WAL;"
+ "CREATE TABLE t1(x);"
+ );
+
+ setstoptime(&err, nMs);
+
+ for(i=0; i<4; i++){
+ launch_thread(&err, &threads, checkpoint_starvation_reader, 0);
+ usleep(CHECKPOINT_STARVATION_READMS*1000/4);
+ }
+
+ sqlite3_wal_hook(db.db, checkpoint_starvation_walhook, (void *)p);
+ while( !timetostop(&err) ){
+ sql_script(&err, &db, "INSERT INTO t1 VALUES(randomblob(1200))");
+ nInsert++;
+ }
+
+ printf(" Checkpoint mode : %s\n",
+ p->eMode==SQLITE_CHECKPOINT_PASSIVE ? "PASSIVE" : "RESTART"
+ );
+ printf(" Peak WAL : %d frames\n", p->nMaxFrame);
+ printf(" Transaction count: %d transactions\n", nInsert);
+
+ join_all_threads(&err, &threads);
+ closedb(&err, &db);
+ print_and_free_err(&err);
+}
+
+static void checkpoint_starvation_1(int nMs){
+ Error err = {0};
+ CheckpointStarvationCtx ctx = { SQLITE_CHECKPOINT_PASSIVE, 0 };
+ checkpoint_starvation_main(nMs, &ctx);
+ if( ctx.nMaxFrame<(CHECKPOINT_STARVATION_FRAMELIMIT*10) ){
+ test_error(&err, "WAL failed to grow - %d frames", ctx.nMaxFrame);
+ }
+ print_and_free_err(&err);
+}
+
+static void checkpoint_starvation_2(int nMs){
+ Error err = {0};
+ CheckpointStarvationCtx ctx = { SQLITE_CHECKPOINT_RESTART, 0 };
+ checkpoint_starvation_main(nMs, &ctx);
+ if( ctx.nMaxFrame>CHECKPOINT_STARVATION_FRAMELIMIT+10 ){
+ test_error(&err, "WAL grew too large - %d frames", ctx.nMaxFrame);
+ }
+ print_and_free_err(&err);
+}
+
+
diff --git a/test/wal.test b/test/wal.test
index 1db08b5..339661e 100644
--- a/test/wal.test
+++ b/test/wal.test
@@ -287,8 +287,8 @@
INSERT INTO t2 VALUES('y', 'z');
ROLLBACK TO save;
COMMIT;
- SELECT * FROM t2;
}
+ execsql { SELECT * FROM t2 }
} {w x}
@@ -542,7 +542,7 @@
} {1 2 3 4 5 6 7 8 9 10}
do_test wal-10.$tn.12 {
catchsql { PRAGMA wal_checkpoint }
- } {0 {}} ;# Reader no longer block checkpoints
+ } {0 {0 13 13}} ;# Reader no longer block checkpoints
do_test wal-10.$tn.13 {
execsql { INSERT INTO t1 VALUES(11, 12) }
sql2 {SELECT * FROM t1}
@@ -552,7 +552,7 @@
#
do_test wal-10.$tn.14 {
catchsql { PRAGMA wal_checkpoint }
- } {0 {}}
+ } {0 {0 15 13}}
# The following series of test cases used to verify another blocking
# case in WAL - a case which no longer blocks.
@@ -562,10 +562,10 @@
} {1 2 3 4 5 6 7 8 9 10 11 12}
do_test wal-10.$tn.16 {
catchsql { PRAGMA wal_checkpoint }
- } {0 {}}
+ } {0 {0 15 15}}
do_test wal-10.$tn.17 {
execsql { PRAGMA wal_checkpoint }
- } {}
+ } {0 15 15}
do_test wal-10.$tn.18 {
sql3 { BEGIN; SELECT * FROM t1 }
} {1 2 3 4 5 6 7 8 9 10 11 12}
@@ -588,13 +588,13 @@
#
do_test wal-10.$tn.23 {
execsql { PRAGMA wal_checkpoint }
- } {}
+ } {0 17 17}
do_test wal-10.$tn.24 {
sql2 { BEGIN; SELECT * FROM t1; }
} {1 2 3 4 5 6 7 8 9 10 11 12 13 14}
do_test wal-10.$tn.25 {
execsql { PRAGMA wal_checkpoint }
- } {}
+ } {0 17 17}
do_test wal-10.$tn.26 {
catchsql { INSERT INTO t1 VALUES(15, 16) }
} {0 {}}
@@ -611,11 +611,11 @@
do_test wal-10.$tn.29 {
execsql { INSERT INTO t1 VALUES(19, 20) }
catchsql { PRAGMA wal_checkpoint }
- } {0 {}}
+ } {0 {0 6 0}}
do_test wal-10.$tn.30 {
code3 { sqlite3_finalize $::STMT }
execsql { PRAGMA wal_checkpoint }
- } {}
+ } {0 6 0}
# At one point, if a reader failed to upgrade to a writer because it
# was reading an old snapshot, the write-locks were not being released.
@@ -654,7 +654,7 @@
} {a b c d}
do_test wal-10.$tn.36 {
catchsql { PRAGMA wal_checkpoint }
- } {0 {}}
+ } {0 {0 16 16}}
do_test wal-10.$tn.36 {
sql3 { INSERT INTO t1 VALUES('e', 'f') }
sql2 { SELECT * FROM t1 }
@@ -662,7 +662,7 @@
do_test wal-10.$tn.37 {
sql2 COMMIT
execsql { PRAGMA wal_checkpoint }
- } {}
+ } {0 18 18}
}
#-------------------------------------------------------------------------
@@ -797,8 +797,8 @@
UPDATE t1 SET y = 1 WHERE x = 'A';
PRAGMA wal_checkpoint;
UPDATE t1 SET y = 0 WHERE x = 'A';
- SELECT * FROM t2;
}
+ execsql { SELECT * FROM t2 }
} {B 2}
do_test wal-12.6 {
file copy -force test.db test2.db
@@ -845,6 +845,7 @@
sqlite3 db test.db
execsql { SELECT * FROM t2 }
} {B 2}
+breakpoint
do_test wal-13.1.3 {
db close
file exists test.db-wal
@@ -1029,14 +1030,14 @@
foreach {tn ckpt_cmd ckpt_res ckpt_main ckpt_aux} {
1 {sqlite3_wal_checkpoint db} SQLITE_OK 1 1
2 {sqlite3_wal_checkpoint db ""} SQLITE_OK 1 1
- 3 {db eval "PRAGMA wal_checkpoint"} {} 1 1
+ 3 {db eval "PRAGMA wal_checkpoint"} {0 10 10} 1 1
4 {sqlite3_wal_checkpoint db main} SQLITE_OK 1 0
5 {sqlite3_wal_checkpoint db aux} SQLITE_OK 0 1
6 {sqlite3_wal_checkpoint db temp} SQLITE_OK 0 0
- 7 {db eval "PRAGMA main.wal_checkpoint"} {} 1 0
- 8 {db eval "PRAGMA aux.wal_checkpoint"} {} 0 1
- 9 {db eval "PRAGMA temp.wal_checkpoint"} {} 0 0
+ 7 {db eval "PRAGMA main.wal_checkpoint"} {0 10 10} 1 0
+ 8 {db eval "PRAGMA aux.wal_checkpoint"} {0 16 16} 0 1
+ 9 {db eval "PRAGMA temp.wal_checkpoint"} {0 -1 -1} 0 0
} {
do_test wal-16.$tn.1 {
file delete -force test2.db test2.db-wal test2.db-journal
@@ -1400,10 +1401,8 @@
} {0}
do_test wal-20.3 {
close $::buddy
- execsql {
- PRAGMA wal_checkpoint;
- SELECT count(*) FROM t1;
- }
+ execsql { PRAGMA wal_checkpoint }
+ execsql { SELECT count(*) FROM t1 }
} {16384}
do_test wal-20.4 {
db close
@@ -1437,8 +1436,8 @@
INSERT INTO t1 SELECT randomblob(900), randomblob(900) FROM t1;
ROLLBACK TO s;
COMMIT;
- SELECT * FROM t1;
}
+ execsql { SELECT * FROM t1 }
} {1 2 3 4 5 6 7 8 9 10 11 12}
do_test wal-21.3 {
execsql { PRAGMA integrity_check }
diff --git a/test/wal2.test b/test/wal2.test
index a62e9b9..299afa4 100644
--- a/test/wal2.test
+++ b/test/wal2.test
@@ -346,7 +346,7 @@
INSERT INTO data VALUES('need xShmOpen to see this');
PRAGMA wal_checkpoint;
}
-} {wal}
+} {wal 0 5 5}
do_test wal2-4.2 {
db close
testvfs tvfs -noshm 1
@@ -712,7 +712,7 @@
INSERT INTO t2 VALUES('I', 'II');
PRAGMA journal_mode;
}
-} {wal exclusive wal}
+} {wal exclusive 0 3 3 wal}
do_test wal2-6.5.2 {
execsql {
PRAGMA locking_mode = normal;
@@ -723,7 +723,7 @@
} {normal exclusive I II III IV}
do_test wal2-6.5.3 {
execsql { PRAGMA wal_checkpoint }
-} {}
+} {0 4 4}
db close
proc lock_control {method filename handle spec} {
@@ -807,9 +807,9 @@
CREATE TABLE t1(x);
INSERT INTO t1 VALUES(zeroblob(8188*1020));
CREATE TABLE t2(y);
+ PRAGMA wal_checkpoint;
}
execsql {
- PRAGMA wal_checkpoint;
SELECT rootpage>=8192 FROM sqlite_master WHERE tbl_name = 't2';
}
} {1}
diff --git a/test/wal3.test b/test/wal3.test
index c57132e..84e321c 100644
--- a/test/wal3.test
+++ b/test/wal3.test
@@ -427,7 +427,7 @@
} {o t t f}
do_test wal3-6.1.3 {
execsql { PRAGMA wal_checkpoint } db2
-} {}
+} {0 7 7}
# At this point the log file has been fully checkpointed. However,
# connection [db3] holds a lock that prevents the log from being wrapped.
@@ -515,7 +515,7 @@
}
do_test wal3-6.2.2 {
execsql { PRAGMA wal_checkpoint }
-} {}
+} {0 7 7}
do_test wal3-6.2.3 {
set ::R
} {h h l b}
@@ -624,7 +624,7 @@
INSERT INTO b VALUES('Markazi');
PRAGMA wal_checkpoint;
}
-} {wal}
+} {wal 0 9 9}
do_test wal3-8.2 {
execsql { SELECT * FROM b }
} {Tehran Qom Markazi}
diff --git a/test/wal5.test b/test/wal5.test
new file mode 100644
index 0000000..2ea4805
--- /dev/null
+++ b/test/wal5.test
@@ -0,0 +1,262 @@
+# 2010 April 13
+#
+# The author disclaims copyright to this source code. In place of
+# a legal notice, here is a blessing:
+#
+# May you do good and not evil.
+# May you find forgiveness for yourself and forgive others.
+# May you share freely, never taking more than you give.
+#
+#***********************************************************************
+# This file implements regression tests for SQLite library. The
+# focus of this file is testing the operation of "blocking-checkpoint"
+# operations.
+#
+
+set testdir [file dirname $argv0]
+source $testdir/tester.tcl
+source $testdir/lock_common.tcl
+source $testdir/wal_common.tcl
+ifcapable !wal {finish_test ; return }
+
+set testprefix wal5
+
+proc db_page_count {{file test.db}} { expr [file size $file] / 1024 }
+proc wal_page_count {{file test.db}} { wal_frame_count ${file}-wal 1024 }
+
+
+do_multiclient_test tn {
+
+
+ set ::nBusyHandler 0
+ set ::busy_handler_script ""
+ proc busyhandler {n} {
+ incr ::nBusyHandler
+ eval $::busy_handler_script
+ return 0
+ }
+
+ proc reopen_all {} {
+ code1 {db close}
+ code2 {db2 close}
+ code3 {db3 close}
+ code1 {sqlite3 db test.db}
+ code2 {sqlite3 db2 test.db}
+ code3 {sqlite3 db3 test.db}
+ sql1 { PRAGMA synchronous = NORMAL }
+ code1 { db busy busyhandler }
+ }
+
+ do_test 1.$tn.1 {
+ reopen_all
+ sql1 {
+ PRAGMA page_size = 1024;
+ PRAGMA auto_vacuum = 0;
+ CREATE TABLE t1(x, y);
+ PRAGMA journal_mode = WAL;
+ INSERT INTO t1 VALUES(1, zeroblob(1200));
+ INSERT INTO t1 VALUES(2, zeroblob(1200));
+ INSERT INTO t1 VALUES(3, zeroblob(1200));
+ }
+ expr [file size test.db] / 1024
+ } {2}
+
+ # Have connection 2 grab a read-lock on the current snapshot.
+ do_test 1.$tn.2 { sql2 { BEGIN; SELECT x FROM t1 } } {1 2 3}
+
+ # Attempt a checkpoint.
+ do_test 1.$tn.3 {
+ sql1 { PRAGMA wal_checkpoint }
+ list [db_page_count] [wal_page_count]
+ } {5 9}
+
+ # Write to the db again. The log cannot wrap because of the lock still
+ # held by connection 2. The busy-handler has not yet been invoked.
+ do_test 1.$tn.4 {
+ sql1 { INSERT INTO t1 VALUES(4, zeroblob(1200)) }
+ list [db_page_count] [wal_page_count] $::nBusyHandler
+ } {5 12 0}
+
+ # Now do a blocking-checkpoint. Set the busy-handler up so that connection
+ # 2 releases its lock on the 6th invocation. The checkpointer should then
+ # proceed to checkpoint the entire log file. Next write should go to the
+ # start of the log file.
+ #
+ set ::busy_handler_script { if {$n==5} { sql2 COMMIT } }
+ do_test 1.$tn.5 {
+ sql1 { PRAGMA wal_checkpoint = RESTART }
+ list [db_page_count] [wal_page_count] $::nBusyHandler
+ } {6 12 6}
+ do_test 1.$tn.6 {
+ set ::nBusyHandler 0
+ sql1 { INSERT INTO t1 VALUES(5, zeroblob(1200)) }
+ list [db_page_count] [wal_page_count] $::nBusyHandler
+ } {6 12 0}
+
+ do_test 1.$tn.7 {
+ reopen_all
+ list [db_page_count] [wal_page_count] $::nBusyHandler
+ } {7 0 0}
+
+ do_test 1.$tn.8 { sql2 { BEGIN ; SELECT x FROM t1 } } {1 2 3 4 5}
+ do_test 1.$tn.9 {
+ sql1 { INSERT INTO t1 VALUES(6, zeroblob(1200)) }
+ list [db_page_count] [wal_page_count] $::nBusyHandler
+ } {7 5 0}
+ do_test 1.$tn.10 { sql3 { BEGIN ; SELECT x FROM t1 } } {1 2 3 4 5 6}
+
+ set ::busy_handler_script {
+ if {$n==5} { sql2 COMMIT }
+ if {$n==6} { set ::db_file_size [db_page_count] }
+ if {$n==7} { sql3 COMMIT }
+ }
+ do_test 1.$tn.11 {
+ sql1 { PRAGMA wal_checkpoint = RESTART }
+ list [db_page_count] [wal_page_count] $::nBusyHandler
+ } {10 5 8}
+ do_test 1.$tn.12 { set ::db_file_size } 10
+}
+
+
+#-------------------------------------------------------------------------
+# This block of tests explores checkpoint operations on more than one
+# database file.
+#
+proc setup_and_attach_aux {} {
+ sql1 { ATTACH 'test.db2' AS aux }
+ sql2 { ATTACH 'test.db2' AS aux }
+ sql3 { ATTACH 'test.db2' AS aux }
+ sql1 {
+ PRAGMA main.page_size=1024; PRAGMA main.journal_mode=WAL;
+ PRAGMA aux.page_size=1024; PRAGMA aux.journal_mode=WAL;
+ }
+}
+
+proc file_page_counts {} {
+ list [db_page_count test.db ] \
+ [wal_page_count test.db ] \
+ [db_page_count test.db2] \
+ [wal_page_count test.db2]
+}
+
+# Test that executing "PRAGMA wal_checkpoint" checkpoints all attached
+# databases, not just the main db.
+#
+do_multiclient_test tn {
+ setup_and_attach_aux
+ do_test 2.1.$tn.1 {
+ sql1 {
+ CREATE TABLE t1(a, b);
+ INSERT INTO t1 VALUES(1, 2);
+ CREATE TABLE aux.t2(a, b);
+ INSERT INTO t2 VALUES(1, 2);
+ }
+ } {}
+ do_test 2.2.$tn.2 { file_page_counts } {1 5 1 5}
+ do_test 2.1.$tn.3 { sql1 { PRAGMA wal_checkpoint } } {0 5 5}
+ do_test 2.1.$tn.4 { file_page_counts } {2 5 2 5}
+}
+
+do_multiclient_test tn {
+ setup_and_attach_aux
+ do_test 2.2.$tn.1 {
+ execsql {
+ CREATE TABLE t1(a, b);
+ INSERT INTO t1 VALUES(1, 2);
+ CREATE TABLE aux.t2(a, b);
+ INSERT INTO t2 VALUES(1, 2);
+ INSERT INTO t2 VALUES(3, 4);
+ }
+ } {}
+ do_test 2.2.$tn.2 { file_page_counts } {1 5 1 7}
+ do_test 2.2.$tn.3 { sql2 { BEGIN; SELECT * FROM t1 } } {1 2}
+ do_test 2.2.$tn.4 { sql1 { PRAGMA wal_checkpoint = RESTART } } {1 5 5}
+ do_test 2.2.$tn.5 { file_page_counts } {2 5 2 7}
+}
+
+do_multiclient_test tn {
+ setup_and_attach_aux
+ do_test 2.3.$tn.1 {
+ execsql {
+ CREATE TABLE t1(a, b);
+ INSERT INTO t1 VALUES(1, 2);
+ CREATE TABLE aux.t2(a, b);
+ INSERT INTO t2 VALUES(1, 2);
+ }
+ } {}
+ do_test 2.3.$tn.2 { file_page_counts } {1 5 1 5}
+ do_test 2.3.$tn.3 { sql2 { BEGIN; SELECT * FROM t1 } } {1 2}
+ do_test 2.3.$tn.4 { sql1 { INSERT INTO t1 VALUES(3, 4) } } {}
+ do_test 2.3.$tn.5 { sql1 { INSERT INTO t2 VALUES(3, 4) } } {}
+ do_test 2.3.$tn.6 { file_page_counts } {1 7 1 7}
+ do_test 2.3.$tn.7 { sql1 { PRAGMA wal_checkpoint = FULL } } {1 7 5}
+ do_test 2.3.$tn.8 { file_page_counts } {1 7 2 7}
+}
+
+# Check that checkpoints block on the correct locks. And respond correctly
+# if they cannot obtain those locks. There are three locks that a checkpoint
+# may block on (in the following order):
+#
+# 1. The writer lock: FULL and RESTART checkpoints block until any writer
+# process releases its lock.
+#
+# 2. Readers using part of the log file. FULL and RESTART checkpoints block
+# until readers using part (but not all) of the log file have finished.
+#
+# 3. Readers using any of the log file. After copying data into the
+# database file, RESTART checkpoints block until readers using any part
+# of the log file have finished.
+#
+# This test case involves running a checkpoint while there exist other
+# processes holding all three types of locks.
+#
+foreach {tn1 checkpoint busy_on ckpt_expected expected} {
+ 1 PASSIVE - {0 5 5} -
+ 2 TYPO - {0 5 5} -
+
+ 3 FULL - {0 7 7} 2
+ 4 FULL 1 {1 5 5} 1
+ 5 FULL 2 {1 7 5} 2
+ 6 FULL 3 {0 7 7} 2
+
+ 7 RESTART - {0 7 7} 3
+ 8 RESTART 1 {1 5 5} 1
+ 9 RESTART 2 {1 7 5} 2
+ 10 RESTART 3 {1 7 7} 3
+
+} {
+ do_multiclient_test tn {
+ setup_and_attach_aux
+
+ proc busyhandler {x} {
+ set ::max_busyhandler $x
+ if {$::busy_on!="-" && $x==$::busy_on} { return 1 }
+ switch -- $x {
+ 1 { sql2 "COMMIT ; BEGIN ; SELECT * FROM t1" }
+ 2 { sql3 "COMMIT" }
+ 3 { sql2 "COMMIT" }
+ }
+ return 0
+ }
+ set ::max_busyhandler -
+
+ do_test 2.4.$tn1.$tn.1 {
+ sql1 {
+ CREATE TABLE t1(a, b);
+ INSERT INTO t1 VALUES(1, 2);
+ }
+ sql2 { BEGIN; INSERT INTO t1 VALUES(3, 4) }
+ sql3 { BEGIN; SELECT * FROM t1 }
+ } {1 2}
+
+ do_test 2.4.$tn1.$tn.2 {
+ code1 { db busy busyhandler }
+ sql1 "PRAGMA wal_checkpoint = $checkpoint"
+ } $ckpt_expected
+ do_test 2.4.$tn1.$tn.3 { set ::max_busyhandler } $expected
+ }
+}
+
+
+finish_test
+
diff --git a/test/wal_common.tcl b/test/wal_common.tcl
index a5b165d..917ad59 100644
--- a/test/wal_common.tcl
+++ b/test/wal_common.tcl
@@ -18,7 +18,9 @@
}
proc wal_frame_count {zFile pgsz} {
+ if {[file exists $zFile]==0} { return 0 }
set f [file size $zFile]
+ if {$f < 32} { return 0 }
expr {($f - 32) / ($pgsz+24)}
}
