Skip to content

Commit f087ed7

Browse files
mugikhanmugikhan
authored
Merge pull request #7 from powersync-ja/chore/update-wa-sqlite
Feat: Full text search and update wa-sqlite
2 parents 1d1d391 + 0077436 commit f087ed7

13 files changed

+1848
-1507
lines changed

‎.changeset/stupid-jokes-add.md

+6
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
---
2+
"@journeyapps/wa-sqlite": minor
3+
---
4+
5+
Added full text search (FTS5) support out of the box by default. There should be no configuration required.
6+
Updated from upstream changes

‎Makefile

+61-4
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,13 @@ CFILES = \
1717
libvfs.c \
1818
$(CFILES_EXTRA)
1919

20+
JSFILES = \
21+
src/libauthorizer.js \
22+
src/libfunction.js \
23+
src/libmodule.js \
24+
src/libprogress.js \
25+
src/libvfs.js
26+
2027
vpath %.c src
2128
vpath %.c deps
2229
vpath %.c deps/$(SQLITE_VERSION)
@@ -56,7 +63,7 @@ EMFLAGS_COMMON = \
5663

5764
EMFLAGS_DEBUG = \
5865
-s ASSERTIONS=1 \
59-
-g -Oz \
66+
-g -Oz -Oz \
6067
$(EMFLAGS_COMMON)
6168

6269
EMFLAGS_DIST = \
@@ -104,6 +111,7 @@ WASQLITE_DEFINES = \
104111
-DSQLITE_THREADSAFE=0 \
105112
-DSQLITE_USE_ALLOCA \
106113
-DSQLITE_ENABLE_BATCH_ATOMIC_WRITE \
114+
-DSQLITE_ENABLE_FTS5 \
107115
$(WASQLITE_EXTRA_DEFINES)
108116

109117
# directories
@@ -112,11 +120,11 @@ all: dist
112120

113121
.PHONY: clean
114122
clean:
115-
rm -rf dist dist-xl debug tmp
123+
rm -rf dist debug tmp
116124

117125
.PHONY: spotless
118126
spotless:
119-
rm -rf dist dist-xl debug tmp deps cache
127+
rm -rf dist debug tmp deps cache
120128

121129
## cache
122130
.PHONY: clean-cache
@@ -194,6 +202,27 @@ debug/wa-sqlite-async.mjs: $(OBJ_FILES_DEBUG) $(RS_DEBUG_BC) $(EXPORTED_FUNCTION
194202
$(RS_WASM_TGT_DIR)/debug/deps/*.bc \
195203
$(OBJ_FILES_DEBUG) *.o -o $@
196204

205+
## Debug FTS builds
206+
# .PHONY: debug
207+
# debug: debug/wa-sqlite.mjs debug/wa-sqlite-async.mjs
208+
209+
# debug/wa-sqlite.mjs: $(OBJ_FILES_DEBUG_FTS) $(RS_DEBUG_BC) $(EXPORTED_FUNCTIONS) $(EXPORTED_RUNTIME_METHODS)
210+
# mkdir -p debug
211+
# $(EMCC) $(EMFLAGS_DEBUG) \
212+
# $(EMFLAGS_INTERFACES) \
213+
# $(EMFLAGS_LIBRARIES) \
214+
# $(RS_WASM_TGT_DIR)/debug/deps/*.bc \
215+
# $(OBJ_FILES_DEBUG_FTS) *.o -o $@
216+
217+
# debug/wa-sqlite-async.mjs: $(OBJ_FILES_DEBUG_FTS) $(RS_DEBUG_BC) $(EXPORTED_FUNCTIONS) $(EXPORTED_RUNTIME_METHODS) $(ASYNCIFY_IMPORTS)
218+
# mkdir -p debug
219+
# $(EMCC) $(EMFLAGS_DEBUG) \
220+
# $(EMFLAGS_INTERFACES) \
221+
# $(EMFLAGS_LIBRARIES) \
222+
# $(EMFLAGS_ASYNCIFY_DEBUG) \
223+
# $(RS_WASM_TGT_DIR)/debug/deps/*.bc \
224+
# $(OBJ_FILES_DEBUG_FTS) *.o -o $@
225+
197226
## dist
198227
.PHONY: clean-dist
199228
clean-dist:
@@ -220,4 +249,32 @@ dist/wa-sqlite-async.mjs: $(OBJ_FILES_DIST) $(RS_RELEASE_BC) $(EXPORTED_FUNCTION
220249
$(RS_WASM_TGT_DIR)/wasm/deps/*.bc \
221250
$(OBJ_FILES_DIST) -o $@
222251

223-
FORCE: ;
252+
FORCE: ;
253+
254+
# FTS builds
255+
# .PHONY: clean-dist
256+
# clean-dist:
257+
# rm -rf dist
258+
259+
# .PHONY: dist
260+
# dist: dist/wa-sqlite.mjs dist/wa-sqlite-async.mjs
261+
262+
# dist/wa-sqlite.mjs: $(OBJ_FILES_DIST_FTS) $(RS_RELEASE_BC) $(EXPORTED_FUNCTIONS) $(EXPORTED_RUNTIME_METHODS)
263+
# mkdir -p dist
264+
# $(EMCC) $(EMFLAGS_DIST) \
265+
# $(EMFLAGS_INTERFACES) \
266+
# $(EMFLAGS_LIBRARIES) \
267+
# $(RS_WASM_TGT_DIR)/wasm/deps/*.bc \
268+
# $(OBJ_FILES_DIST_FTS) -o $@
269+
270+
# dist/wa-sqlite-async.mjs: $(OBJ_FILES_DIST_FTS) $(RS_RELEASE_BC) $(EXPORTED_FUNCTIONS) $(EXPORTED_RUNTIME_METHODS) $(ASYNCIFY_IMPORTS)
271+
# mkdir -p dist
272+
# $(EMCC) $(EMFLAGS_DIST) \
273+
# $(EMFLAGS_INTERFACES) \
274+
# $(EMFLAGS_LIBRARIES) \
275+
# $(EMFLAGS_ASYNCIFY_DIST) \
276+
# $(CFLAGS_DIST) \
277+
# $(RS_WASM_TGT_DIR)/wasm/deps/*.bc \
278+
# $(OBJ_FILES_DIST_FTS) -o $@
279+
280+
# FORCE: ;

‎README.md

+2-2
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ Here are the build steps:
2929
* `git clone [email protected]:rhashimoto/wa-sqlite.git`
3030
* `cd wa-sqlite`
3131
* `yarn install`
32-
* `make` (or `yarn prepack`)
32+
* `make`
3333

3434
The default build produces ES6 modules + WASM, [synchronous and asynchronous](https://github.com/rhashimoto/wa-sqlite/issues/7) (using Asyncify) in `dist/`.
3535

@@ -51,7 +51,7 @@ git submodule init
5151
```
5252

5353
```bash
54-
git submodule update --recursive
54+
git submodule update --init --recursive
5555
```
5656

5757
```bash

‎dist/wa-sqlite-async.mjs

+132-136
Large diffs are not rendered by default.

‎dist/wa-sqlite-async.wasm

188 KB
Binary file not shown.

‎dist/wa-sqlite.mjs

+129-132
Large diffs are not rendered by default.

‎dist/wa-sqlite.wasm

87.6 KB
Binary file not shown.

‎package.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@
3737
"karma-jasmine": "^5.1.0",
3838
"monaco-editor": "^0.34.1",
3939
"puppeteer": "^19.6.3",
40-
"sinon": "^15.0.1",
40+
"sinon": "15.0.1",
4141
"typedoc": "^0.22.11",
4242
"typescript": "^4.2.4"
4343
},

‎scripts/docker-setup.sh

+4-3
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
1-
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
2-
source ~/.bashrc
3-
rustup component add rust-src --toolchain nightly-2023-08-28-aarch64-unknown-linux-gnu -y
1+
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y && \
2+
. $HOME/.cargo/env && \
3+
rustup toolchain install nightly-2023-08-28-aarch64-unknown-linux-gnu && \
4+
rustup component add rust-src --toolchain nightly-2023-08-28-aarch64-unknown-linux-gnu
45

56
sudo apt-get update -y
67
sudo apt-get install -y tclsh

‎src/libvfs.c

+56-48
Original file line numberDiff line numberDiff line change
@@ -6,71 +6,75 @@
66

77
// sqlite3_io_methods javascript handlers
88
// 64-bit integer parameters are passed by pointer.
9-
extern int vfsClose(sqlite3_file* file);
10-
extern int vfsRead(sqlite3_file* file, void* pData, int iAmt, sqlite3_int64 iOffset);
11-
extern int vfsWrite(sqlite3_file* file, const void* pData, int iAmt, sqlite3_int64 iOffset);
12-
extern int vfsTruncate(sqlite3_file* file, sqlite3_int64 size);
13-
extern int vfsSync(sqlite3_file* file, int flags);
14-
extern int vfsFileSize(sqlite3_file* file, sqlite3_int64* pSize);
15-
extern int vfsLock(sqlite3_file* file, int flags);
16-
extern int vfsUnlock(sqlite3_file* file, int flags);
17-
extern int vfsCheckReservedLock(sqlite3_file* file, int* pResOut);
18-
extern int vfsFileControl(sqlite3_file* file, int flags, void* pOut);
19-
extern int vfsSectorSize(sqlite3_file* file);
20-
extern int vfsDeviceCharacteristics(sqlite3_file* file);
9+
extern int vfsClose(sqlite3_file *file);
10+
extern int vfsRead(sqlite3_file *file, void *pData, int iAmt, sqlite3_int64 iOffset);
11+
extern int vfsWrite(sqlite3_file *file, const void *pData, int iAmt, sqlite3_int64 iOffset);
12+
extern int vfsTruncate(sqlite3_file *file, sqlite3_int64 size);
13+
extern int vfsSync(sqlite3_file *file, int flags);
14+
extern int vfsFileSize(sqlite3_file *file, sqlite3_int64 *pSize);
15+
extern int vfsLock(sqlite3_file *file, int flags);
16+
extern int vfsUnlock(sqlite3_file *file, int flags);
17+
extern int vfsCheckReservedLock(sqlite3_file *file, int *pResOut);
18+
extern int vfsFileControl(sqlite3_file *file, int flags, void *pOut);
19+
extern int vfsSectorSize(sqlite3_file *file);
20+
extern int vfsDeviceCharacteristics(sqlite3_file *file);
2121

22-
extern int vfsOpen(sqlite3_vfs* vfs, const char *zName, sqlite3_file* file, int flags, int *pOutFlags);
23-
extern int vfsDelete(sqlite3_vfs* vfs, const char *zName, int syncDir);
24-
extern int vfsAccess(sqlite3_vfs* vfs, const char *zName, int flags, int *pResOut);
22+
extern int vfsOpen(sqlite3_vfs *vfs, const char *zName, sqlite3_file *file, int flags, int *pOutFlags);
23+
extern int vfsDelete(sqlite3_vfs *vfs, const char *zName, int syncDir);
24+
extern int vfsAccess(sqlite3_vfs *vfs, const char *zName, int flags, int *pResOut);
2525

2626
// This is undefined in the WASM linker step if not specified
2727
extern int __rust_no_alloc_shim_is_unstable = 0;
2828
extern int sqlite3_powersync_init(sqlite3 *db, char **pzErrMsg,
29-
const sqlite3_api_routines *pApi);
29+
const sqlite3_api_routines *pApi);
3030

31-
static int xOpen(sqlite3_vfs* vfs, const char* zName, sqlite3_file* file, int flags, int* pOutFlags) {
31+
static int xOpen(sqlite3_vfs *vfs, const char *zName, sqlite3_file *file, int flags, int *pOutFlags)
32+
{
3233
static sqlite3_io_methods io_methods = {
33-
1,
34-
vfsClose,
35-
vfsRead,
36-
vfsWrite,
37-
vfsTruncate,
38-
vfsSync,
39-
vfsFileSize,
40-
vfsLock,
41-
vfsUnlock,
42-
vfsCheckReservedLock,
43-
vfsFileControl,
44-
vfsSectorSize,
45-
vfsDeviceCharacteristics
46-
};
34+
1,
35+
vfsClose,
36+
vfsRead,
37+
vfsWrite,
38+
vfsTruncate,
39+
vfsSync,
40+
vfsFileSize,
41+
vfsLock,
42+
vfsUnlock,
43+
vfsCheckReservedLock,
44+
vfsFileControl,
45+
vfsSectorSize,
46+
vfsDeviceCharacteristics};
4747
file->pMethods = &io_methods;
4848

4949
return vfsOpen(vfs, zName, file, flags, pOutFlags);
5050
}
5151

52-
static int xFullPathname(sqlite3_vfs* vfs, const char* zName, int nOut, char* zOut) {
52+
static int xFullPathname(sqlite3_vfs *vfs, const char *zName, int nOut, char *zOut)
53+
{
5354
strncpy(zOut, zName, nOut);
5455
return SQLITE_OK;
5556
}
5657

57-
static int xCurrentTime(sqlite3_vfs* vfs, double* pJulianDay) {
58+
static int xCurrentTime(sqlite3_vfs *vfs, double *pJulianDay)
59+
{
5860
// UNIX epoch 1/1/1970 is Julian day 2440587.5
59-
static const sqlite3_int64 unixEpoch = 24405875*(sqlite3_int64)8640000;
61+
static const sqlite3_int64 unixEpoch = 24405875 * (sqlite3_int64)8640000;
6062
struct timeval sNow;
6163
gettimeofday(&sNow, 0);
62-
sqlite3_int64 julianMillis = unixEpoch + 1000*(sqlite3_int64)sNow.tv_sec + sNow.tv_usec/1000;
64+
sqlite3_int64 julianMillis = unixEpoch + 1000 * (sqlite3_int64)sNow.tv_sec + sNow.tv_usec / 1000;
6365
*pJulianDay = julianMillis / 86400000.0;
6466
return SQLITE_OK;
6567
}
6668

6769
const int EMSCRIPTEN_KEEPALIVE register_vfs(
68-
const char* zName,
69-
int mxPathName,
70-
int makeDefault,
71-
sqlite3_vfs** ppVFS) {
72-
sqlite3_vfs* vfs = *ppVFS = (sqlite3_vfs*)sqlite3_malloc(sizeof(sqlite3_vfs));
73-
if (!vfs) {
70+
const char *zName,
71+
int mxPathName,
72+
int makeDefault,
73+
sqlite3_vfs **ppVFS)
74+
{
75+
sqlite3_vfs *vfs = *ppVFS = (sqlite3_vfs *)sqlite3_malloc(sizeof(sqlite3_vfs));
76+
if (!vfs)
77+
{
7478
return SQLITE_NOMEM;
7579
}
7680

@@ -85,9 +89,9 @@ const int EMSCRIPTEN_KEEPALIVE register_vfs(
8589
vfs->xAccess = vfsAccess;
8690
vfs->xFullPathname = xFullPathname;
8791
vfs->xCurrentTime = xCurrentTime;
88-
92+
8993
// Get remaining functionality from the default VFS.
90-
sqlite3_vfs* defer = sqlite3_vfs_find(0);
94+
sqlite3_vfs *defer = sqlite3_vfs_find(0);
9195
#define COPY_FIELD(NAME) vfs->NAME = defer->NAME
9296
COPY_FIELD(xDlOpen);
9397
COPY_FIELD(xDlError);
@@ -99,22 +103,26 @@ const int EMSCRIPTEN_KEEPALIVE register_vfs(
99103
#undef COPY_FIELD
100104

101105
const int result = sqlite3_vfs_register(vfs, makeDefault);
102-
if (result != SQLITE_OK) {
106+
if (result != SQLITE_OK)
107+
{
103108
*ppVFS = 0;
104109
sqlite3_free(vfs);
105110
}
106111
return result;
107112
}
108113

109-
void* EMSCRIPTEN_KEEPALIVE getSqliteFree() {
114+
void *EMSCRIPTEN_KEEPALIVE getSqliteFree()
115+
{
110116
return sqlite3_free;
111117
}
112118

113-
int main() {
119+
int main()
120+
{
114121
sqlite3_initialize();
115122
return 0;
116123
}
117124

118-
int setup_powersync() {
119-
return sqlite3_auto_extension((void (*)(void)) &sqlite3_powersync_init);
125+
int setup_powersync()
126+
{
127+
return sqlite3_auto_extension((void (*)(void)) & sqlite3_powersync_init);
120128
}

‎src/libvfs.js

+1-1
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ const vfs_methods = {
2222
vfs['handleAsync'] = Asyncify.handleAsync;
2323
}
2424

25-
const mxPathName = vfs.mxPathName ?? 64;
25+
const mxPathName = vfs['mxPathName'] ?? 64;
2626
const out = Module['_malloc'](4);
2727
const result = ccall('register_vfs', 'number', ['string', 'number', 'number', 'number'],
2828
[vfs.name, mxPathName, makeDefault ? 1 : 0, out]);

‎src/types/index.d.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -1692,4 +1692,4 @@ declare module 'wa-sqlite/src/examples/tag.js' {
16921692
* @returns {function(TemplateStringsArray, ...any): Promise<object[]>}
16931693
*/
16941694
export function tag(sqlite3: any, db: number): (arg0: TemplateStringsArray, ...args: any[]) => Promise<object[]>;
1695-
}
1695+
}

0 commit comments

Comments
 (0)