From e84dc809e261191cc5feb094e0e3728a0fafe2b7 Mon Sep 17 00:00:00 2001 From: BogDan Vatra Date: Tue, 13 Jun 2023 16:10:23 +0300 Subject: [PATCH] Say hello to QtVFS for SQLite3 This patch allows to open databases using QFile. This way it can open databases from RW locations as android shared storage or even from RO resources e.g. qrc or android assets. [ChangeLog][QtSql][SQLite3 driver] QtVFS for SQLite3 allows to open databases using QFile. This way it can open databases from RW locations such as android shared storage, or even from read-only resources e.g. qrc or android assets. Fixes: QTBUG-107120 Change-Id: I889ad44de966c96105fe1954ee4eda175dd5a886 Reviewed-by: Christian Ehrlicher --- src/plugins/sqldrivers/sqlite/CMakeLists.txt | 1 + src/plugins/sqldrivers/sqlite/qsql_sqlite.cpp | 5 +- .../sqldrivers/sqlite/qsql_sqlite_vfs.cpp | 255 ++++++++++++++++++ .../sqldrivers/sqlite/qsql_sqlite_vfs_p.h | 21 ++ src/plugins/sqldrivers/sqlite/smain.cpp | 2 + src/sql/doc/src/sql-driver.qdoc | 11 + tests/auto/sql/kernel/CMakeLists.txt | 1 + tests/auto/sql/kernel/qvfssql/CMakeLists.txt | 24 ++ tests/auto/sql/kernel/qvfssql/sample.db | Bin 0 -> 49152 bytes tests/auto/sql/kernel/qvfssql/tst_qvfssql.cpp | 94 +++++++ 10 files changed, 413 insertions(+), 1 deletion(-) create mode 100644 src/plugins/sqldrivers/sqlite/qsql_sqlite_vfs.cpp create mode 100644 src/plugins/sqldrivers/sqlite/qsql_sqlite_vfs_p.h create mode 100644 tests/auto/sql/kernel/qvfssql/CMakeLists.txt create mode 100644 tests/auto/sql/kernel/qvfssql/sample.db create mode 100644 tests/auto/sql/kernel/qvfssql/tst_qvfssql.cpp diff --git a/src/plugins/sqldrivers/sqlite/CMakeLists.txt b/src/plugins/sqldrivers/sqlite/CMakeLists.txt index add9dff9fdd..6b31b1018c4 100644 --- a/src/plugins/sqldrivers/sqlite/CMakeLists.txt +++ b/src/plugins/sqldrivers/sqlite/CMakeLists.txt @@ -10,6 +10,7 @@ qt_internal_add_plugin(QSQLiteDriverPlugin PLUGIN_TYPE sqldrivers SOURCES qsql_sqlite.cpp qsql_sqlite_p.h + qsql_sqlite_vfs.cpp qsql_sqlite_vfs_p.h smain.cpp DEFINES QT_NO_CAST_FROM_ASCII diff --git a/src/plugins/sqldrivers/sqlite/qsql_sqlite.cpp b/src/plugins/sqldrivers/sqlite/qsql_sqlite.cpp index 60661e68243..c5d1e7a2c71 100644 --- a/src/plugins/sqldrivers/sqlite/qsql_sqlite.cpp +++ b/src/plugins/sqldrivers/sqlite/qsql_sqlite.cpp @@ -691,6 +691,7 @@ bool QSQLiteDriver::open(const QString & db, const QString &, const QString &, c bool openReadOnlyOption = false; bool openUriOption = false; bool useExtendedResultCodes = true; + bool useQtVfs = false; #if QT_CONFIG(regularexpression) static const auto regexpConnectOption = "QSQLITE_ENABLE_REGEXP"_L1; bool defineRegexp = false; @@ -708,6 +709,8 @@ bool QSQLiteDriver::open(const QString & db, const QString &, const QString &, c if (ok) timeOut = nt; } + } else if (option == "QSQLITE_USE_QT_VFS"_L1) { + useQtVfs = true; } else if (option == "QSQLITE_OPEN_READONLY"_L1) { openReadOnlyOption = true; } else if (option == "QSQLITE_OPEN_URI"_L1) { @@ -742,7 +745,7 @@ bool QSQLiteDriver::open(const QString & db, const QString &, const QString &, c openMode |= SQLITE_OPEN_NOMUTEX; - const int res = sqlite3_open_v2(db.toUtf8().constData(), &d->access, openMode, nullptr); + const int res = sqlite3_open_v2(db.toUtf8().constData(), &d->access, openMode, useQtVfs ? "QtVFS" : nullptr); if (res == SQLITE_OK) { sqlite3_busy_timeout(d->access, timeOut); diff --git a/src/plugins/sqldrivers/sqlite/qsql_sqlite_vfs.cpp b/src/plugins/sqldrivers/sqlite/qsql_sqlite_vfs.cpp new file mode 100644 index 00000000000..be5807c6e2d --- /dev/null +++ b/src/plugins/sqldrivers/sqlite/qsql_sqlite_vfs.cpp @@ -0,0 +1,255 @@ +// Copyright (C) 2023 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#include "qsql_sqlite_vfs_p.h" + +#include + +#include // defines PATH_MAX on unix +#include +#include // defines FILENAME_MAX everywhere + +#ifndef PATH_MAX +# define PATH_MAX FILENAME_MAX +#endif + +namespace { +struct Vfs : sqlite3_vfs { + sqlite3_vfs *pVfs; + sqlite3_io_methods ioMethods; +}; + +struct File : sqlite3_file { + class QtFile : public QFile { + public: + QtFile(const QString &name, bool removeOnClose) + : QFile(name) + , removeOnClose(removeOnClose) + {} + + ~QtFile() override + { + if (removeOnClose) + remove(); + } + private: + bool removeOnClose; + }; + QtFile *pFile; +}; + + +int xClose(sqlite3_file *sfile) +{ + auto file = static_cast(sfile); + delete file->pFile; + file->pFile = nullptr; + return SQLITE_OK; +} + +int xRead(sqlite3_file *sfile, void *ptr, int iAmt, sqlite3_int64 iOfst) +{ + auto file = static_cast(sfile); + if (!file->pFile->seek(iOfst)) + return SQLITE_IOERR_READ; + + auto sz = file->pFile->read(static_cast(ptr), iAmt); + if (sz < iAmt) { + memset(static_cast(ptr) + sz, 0, size_t(iAmt - sz)); + return SQLITE_IOERR_SHORT_READ; + } + return SQLITE_OK; +} + +int xWrite(sqlite3_file *sfile, const void *data, int iAmt, sqlite3_int64 iOfst) +{ + auto file = static_cast(sfile); + if (!file->pFile->seek(iOfst)) + return SQLITE_IOERR_SEEK; + return file->pFile->write(reinterpret_cast(data), iAmt) == iAmt ? SQLITE_OK : SQLITE_IOERR_WRITE; +} + +int xTruncate(sqlite3_file *sfile, sqlite3_int64 size) +{ + auto file = static_cast(sfile); + return file->pFile->resize(size) ? SQLITE_OK : SQLITE_IOERR_TRUNCATE; +} + +int xSync(sqlite3_file *sfile, int /*flags*/) +{ + static_cast(sfile)->pFile->flush(); + return SQLITE_OK; +} + +int xFileSize(sqlite3_file *sfile, sqlite3_int64 *pSize) +{ + auto file = static_cast(sfile); + *pSize = file->pFile->size(); + return SQLITE_OK; +} + +// No lock/unlock for QFile, QLockFile doesn't work for me + +int xLock(sqlite3_file *, int) { return SQLITE_OK; } + +int xUnlock(sqlite3_file *, int) { return SQLITE_OK; } + +int xCheckReservedLock(sqlite3_file *, int *pResOut) +{ + *pResOut = 0; + return SQLITE_OK; +} + +int xFileControl(sqlite3_file *, int, void *) { return SQLITE_NOTFOUND; } + +int xSectorSize(sqlite3_file *) +{ + return 4096; +} + +int xDeviceCharacteristics(sqlite3_file *) +{ + return 0; // no SQLITE_IOCAP_XXX +} + +int xOpen(sqlite3_vfs *svfs, sqlite3_filename zName, sqlite3_file *sfile, + int flags, int *pOutFlags) +{ + auto vfs = static_cast(svfs); + auto file = static_cast(sfile); + memset(file, 0, sizeof(File)); + QIODeviceBase::OpenMode mode = QIODeviceBase::NotOpen; + if (!zName || (flags & SQLITE_OPEN_MEMORY)) + return SQLITE_PERM; + if ((flags & SQLITE_OPEN_READONLY) && + !(flags & SQLITE_OPEN_READWRITE) && + !(flags & SQLITE_OPEN_CREATE) && + !(flags & SQLITE_OPEN_DELETEONCLOSE)) { + mode |= QIODeviceBase::OpenModeFlag::ReadOnly; + } else { + /* + ** ^The [SQLITE_OPEN_EXCLUSIVE] flag is always used in conjunction + ** with the [SQLITE_OPEN_CREATE] flag, which are both directly + ** analogous to the O_EXCL and O_CREAT flags of the POSIX open() + ** API. The SQLITE_OPEN_EXCLUSIVE flag, when paired with the + ** SQLITE_OPEN_CREATE, is used to indicate that file should always + ** be created, and that it is an error if it already exists. + ** It is not used to indicate the file should be opened + ** for exclusive access. + */ + if ((flags & SQLITE_OPEN_CREATE) && (flags & SQLITE_OPEN_EXCLUSIVE)) + mode |= QIODeviceBase::OpenModeFlag::NewOnly; + + if (flags & SQLITE_OPEN_READWRITE) + mode |= QIODeviceBase::OpenModeFlag::ReadWrite; + } + + file->pMethods = &vfs->ioMethods; + file->pFile = new File::QtFile{QString::fromUtf8(zName), bool(flags & SQLITE_OPEN_DELETEONCLOSE)}; + if (!file->pFile->open(mode)) + return SQLITE_CANTOPEN; + if (pOutFlags) + *pOutFlags = flags; + + return SQLITE_OK; +} + +int xDelete(sqlite3_vfs *, const char *zName, int) +{ + return QFile::remove(QString::fromUtf8(zName)) ? SQLITE_OK : SQLITE_ERROR; +} + +int xAccess(sqlite3_vfs */*svfs*/, const char *zName, int flags, int *pResOut) +{ + *pResOut = 0; + switch (flags) { + case SQLITE_ACCESS_EXISTS: + case SQLITE_ACCESS_READ: + *pResOut = QFile::exists(QString::fromUtf8(zName)); + break; + default: + break; + } + return SQLITE_OK; +} + +int xFullPathname(sqlite3_vfs *, const char *zName, int nOut, char *zOut) +{ + if (!zName) + return SQLITE_ERROR; + + int i = 0; + for (;zName[i] && i < nOut; ++i) + zOut[i] = zName[i]; + + if (i >= nOut) + return SQLITE_ERROR; + + zOut[i] = '\0'; + return SQLITE_OK; +} + +int xRandomness(sqlite3_vfs *svfs, int nByte, char *zOut) +{ + auto vfs = static_cast(svfs)->pVfs; + return vfs->xRandomness(vfs, nByte, zOut); +} + +int xSleep(sqlite3_vfs *svfs, int microseconds) +{ + auto vfs = static_cast(svfs)->pVfs; + return vfs->xSleep(vfs, microseconds); +} + +int xCurrentTime(sqlite3_vfs *svfs, double *zOut) +{ + auto vfs = static_cast(svfs)->pVfs; + return vfs->xCurrentTime(vfs, zOut); +} + +int xGetLastError(sqlite3_vfs *, int, char *) +{ + return 0; +} + +int xCurrentTimeInt64(sqlite3_vfs *svfs, sqlite3_int64 *zOut) +{ + auto vfs = static_cast(svfs)->pVfs; + return vfs->xCurrentTimeInt64(vfs, zOut); +} +} // namespace { + +void register_qt_vfs() +{ + static Vfs vfs; + memset(&vfs, 0, sizeof(Vfs)); + vfs.iVersion = 1; + vfs.szOsFile = sizeof(File); + vfs.mxPathname = PATH_MAX; + vfs.zName = "QtVFS"; + vfs.xOpen = &xOpen; + vfs.xDelete = &xDelete; + vfs.xAccess = &xAccess; + vfs.xFullPathname = &xFullPathname; + vfs.xRandomness = &xRandomness; + vfs.xSleep = &xSleep; + vfs.xCurrentTime = &xCurrentTime; + vfs.xGetLastError = &xGetLastError; + vfs.xCurrentTimeInt64 = &xCurrentTimeInt64; + vfs.pVfs = sqlite3_vfs_find(nullptr); + vfs.ioMethods.iVersion = 1; + vfs.ioMethods.xClose = &xClose; + vfs.ioMethods.xRead = &xRead; + vfs.ioMethods.xWrite = &xWrite; + vfs.ioMethods.xTruncate = &xTruncate; + vfs.ioMethods.xSync = &xSync; + vfs.ioMethods.xFileSize = &xFileSize; + vfs.ioMethods.xLock = &xLock; + vfs.ioMethods.xUnlock = &xUnlock; + vfs.ioMethods.xCheckReservedLock = &xCheckReservedLock; + vfs.ioMethods.xFileControl = &xFileControl; + vfs.ioMethods.xSectorSize = &xSectorSize; + vfs.ioMethods.xDeviceCharacteristics = &xDeviceCharacteristics; + + sqlite3_vfs_register(&vfs, 0); +} diff --git a/src/plugins/sqldrivers/sqlite/qsql_sqlite_vfs_p.h b/src/plugins/sqldrivers/sqlite/qsql_sqlite_vfs_p.h new file mode 100644 index 00000000000..56024b3ecb0 --- /dev/null +++ b/src/plugins/sqldrivers/sqlite/qsql_sqlite_vfs_p.h @@ -0,0 +1,21 @@ +// Copyright (C) 2023 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#ifndef QSQL_SQLITE_VFS_H +#define QSQL_SQLITE_VFS_H + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +void register_qt_vfs(); + + +#endif // QSQL_SQLITE_VFS_H diff --git a/src/plugins/sqldrivers/sqlite/smain.cpp b/src/plugins/sqldrivers/sqlite/smain.cpp index f84a256bc8e..0d201c38d30 100644 --- a/src/plugins/sqldrivers/sqlite/smain.cpp +++ b/src/plugins/sqldrivers/sqlite/smain.cpp @@ -4,6 +4,7 @@ #include #include #include "qsql_sqlite_p.h" +#include "qsql_sqlite_vfs_p.h" QT_BEGIN_NAMESPACE @@ -23,6 +24,7 @@ public: QSQLiteDriverPlugin::QSQLiteDriverPlugin() : QSqlDriverPlugin() { + register_qt_vfs(); } QSqlDriver* QSQLiteDriverPlugin::create(const QString &name) diff --git a/src/sql/doc/src/sql-driver.qdoc b/src/sql/doc/src/sql-driver.qdoc index 97af8c620d5..3cac1e8fe31 100644 --- a/src/sql/doc/src/sql-driver.qdoc +++ b/src/sql/doc/src/sql-driver.qdoc @@ -723,6 +723,17 @@ \li Busy handler timeout in milliseconds (val <= 0: disabled), see \l {https://www.sqlite.org/c3ref/busy_timeout.html} {SQLite documentation} for more information + + \row + \li QSQLITE_USE_QT_VFS + \li If set, the database is opened using Qt's VFS which allows to + open databases using QFile. This way it can open databases from + any read-write locations (e.g.android shared storage) but also + from read-only resources (e.g. qrc or android assets). Be aware + that when opening databases from read-only resources make sure + you add QSQLITE_OPEN_READONLY attribute as well. + Otherwise it will fail to open it. + \row \li QSQLITE_OPEN_READONLY \li If set, the database is open in read-only mode which will fail diff --git a/tests/auto/sql/kernel/CMakeLists.txt b/tests/auto/sql/kernel/CMakeLists.txt index 0a2b5dfd422..d51cb75f31b 100644 --- a/tests/auto/sql/kernel/CMakeLists.txt +++ b/tests/auto/sql/kernel/CMakeLists.txt @@ -11,3 +11,4 @@ add_subdirectory(qsqlrecord) add_subdirectory(qsqlthread) add_subdirectory(qsql) add_subdirectory(qsqlresult) +add_subdirectory(qvfssql) diff --git a/tests/auto/sql/kernel/qvfssql/CMakeLists.txt b/tests/auto/sql/kernel/qvfssql/CMakeLists.txt new file mode 100644 index 00000000000..184f0a578d6 --- /dev/null +++ b/tests/auto/sql/kernel/qvfssql/CMakeLists.txt @@ -0,0 +1,24 @@ +# Copyright (C) 2023 The Qt Company Ltd. +# SPDX-License-Identifier: BSD-3-Clause + +##################################################################### +## tst_qsqlfield Test: +##################################################################### + +qt_internal_add_test(tst_qvfssql + SOURCES + tst_qvfssql.cpp + LIBRARIES + Qt::SqlPrivate +) + +set(qvfssql_resource_files + "sample.db" +) + +qt_internal_add_resource(tst_qvfssql "tst_qvfssql" + PREFIX + "/ro/" + FILES + ${qvfssql_resource_files} +) diff --git a/tests/auto/sql/kernel/qvfssql/sample.db b/tests/auto/sql/kernel/qvfssql/sample.db new file mode 100644 index 0000000000000000000000000000000000000000..56e6427e3c715a3abf3c55c44f6cc83b4be4e3a4 GIT binary patch literal 49152 zcmeI*O>5go0LJl|ktIid$!rP1WFd5pF@*K{rN=^F2<@R|**140H!2ZpTU%MlHphOO zp8ElMEj{&A*wda`TDlY#y4&rchmIUOxT!G?DI|u!Kx0WG*^ho>ALOydcYnPVrJ=bW zC%qswP4R<}Qd}`c2(e=4lAXtsYZr2(m2A5lSIl@^5kIwmEbC9APj>Q9TVh-7pEeX*ftPjUS8G z?`&P&-ZI-)uie@*^RjnZyt_Jic!Zjfv=ey*B+oSOb#w`+b+qF#_3nBT&KO%rrF zNw`0`W3^WBD%aPgi26I>!@+~Dy_jkTducpeZ;xN1T^}!%I{CwdmC1yyQFY6omat_u zq8$_UZKSEO`a$oj^wrwZ9HClAp<0F5ox;`}g=*%fg=(HG)26u>r3Y=xn4L(q=FO3? zd6clZ)N!V;HAV>=?zDuB*$KaC^lB|PN2tb8sD}3_FAq}%lZ`oU>$)>7Reg4<<88TV zhB?P@_C09d1G{0;{%UizrsjxVKZ;(@uH%=o57O`={zS#Bj~{giAbn67)_>~f`l(%ELI42-5I_I{1Q0*~0R#|00D&_WD0r@15z-5~ zq4M2qS-AZmE$3WWkuukblW3q+(Q)OfFP;5(H&P{))%hw-;{J{*O4qOW(*D#XIZy?A zHlF@V=+FA2{zt#luj~pF0tg_000IagfB*srAbqzN&S?Ri2nagFR<7A~$ zj^q1T|G(0I3;n+>m=Hh!0R#|0009ILKmY**5I|tj1q#ZMYpNFw(lE(O$6vL>2qanm zztSIv{r`)8g5*a40R#|0009ILKmY**5I_KdNrB`2|8Io;SHGFugJlE|KmY**5I_I{ z1Q0*~0R#|OIDw+d*%1KL%`n=DR6*waHUGxmAPx5E|6jPaBrpO9Abe^G5`e*mx>M!n(v z|Gx~nG5!DNw84ml00IagfB*srAb<{9 literal 0 HcmV?d00001 diff --git a/tests/auto/sql/kernel/qvfssql/tst_qvfssql.cpp b/tests/auto/sql/kernel/qvfssql/tst_qvfssql.cpp new file mode 100644 index 00000000000..e2c093c46d3 --- /dev/null +++ b/tests/auto/sql/kernel/qvfssql/tst_qvfssql.cpp @@ -0,0 +1,94 @@ +// Copyright (C) 2023 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0 + +#include + +#include +#include + +#include "../qsqldatabase/tst_databases.h" + +using namespace Qt::StringLiterals; + +class tst_QVfsSql : public QObject +{ + Q_OBJECT +private slots: + void testRoDb(); + void testRwDb(); +}; + +void tst_QVfsSql::testRoDb() +{ + QVERIFY(QSqlDatabase::drivers().contains("QSQLITE"_L1)); + QSqlDatabase::addDatabase("QSQLITE"_L1, "ro_db"_L1); + QSqlDatabase db = QSqlDatabase::database("ro_db"_L1, false); + QVERIFY_SQL(db, isValid()); + db.setDatabaseName(":/ro/sample.db"_L1); + + db.setConnectOptions("QSQLITE_USE_QT_VFS"_L1); + QVERIFY(!db.open()); // can not open as the QSQLITE_OPEN_READONLY attribute is missing + + db.setConnectOptions("QSQLITE_USE_QT_VFS;QSQLITE_OPEN_READONLY"_L1); + QVERIFY_SQL(db, open()); + + QStringList tables = db.tables(); + QSqlQuery q{db}; + for (auto table : {"reltest1"_L1, "reltest2"_L1, "reltest3"_L1, "reltest4"_L1, "reltest5"_L1}) { + QVERIFY(tables.contains(table)); + QVERIFY_SQL(q, exec("select * from " + table)); + QVERIFY(q.next()); + } + QVERIFY_SQL(q, exec("select * from reltest1 where id = 4"_L1)); + QVERIFY_SQL(q, first()); + QVERIFY(q.value(0).toInt() == 4); + QVERIFY(q.value(1).toString() == "boris"_L1); + QVERIFY(q.value(2).toInt() == 2); + QVERIFY(q.value(3).toInt() == 2); +} + +void tst_QVfsSql::testRwDb() +{ + QSqlDatabase::addDatabase("QSQLITE"_L1, "rw_db"_L1); + QSqlDatabase db = QSqlDatabase::database("rw_db"_L1, false); + QVERIFY_SQL(db, isValid()); + const auto dbPath = QStandardPaths::writableLocation(QStandardPaths::TempLocation) + "/test_qt_vfs.db"_L1; + db.setDatabaseName(dbPath); + QFile::remove(dbPath); + + db.setConnectOptions("QSQLITE_USE_QT_VFS;QSQLITE_OPEN_READONLY"_L1); + QVERIFY(!db.open()); // can not open as the QSQLITE_OPEN_READONLY attribute is set and the file is missing + + db.setConnectOptions("QSQLITE_USE_QT_VFS"_L1); + QVERIFY_SQL(db, open()); + + QVERIFY(db.tables().isEmpty()); + QSqlQuery q{db}; + QVERIFY_SQL(q, exec("CREATE TABLE test (id INTEGER PRIMARY KEY AUTOINCREMENT, val INTEGER)"_L1)); + QVERIFY_SQL(q, exec("BEGIN"_L1)); + for (int i = 0; i < 1000; ++i) { + q.prepare("INSERT INTO test (val) VALUES (:val)"_L1); + q.bindValue(":val"_L1, i); + QVERIFY_SQL(q, exec()); + } + QVERIFY_SQL(q, exec("COMMIT"_L1)); + QVERIFY_SQL(q, exec("SELECT val FROM test ORDER BY val"_L1)); + for (int i = 0; i < 1000; ++i) { + QVERIFY_SQL(q, next()); + QCOMPARE(q.value(0).toInt() , i); + } + QVERIFY_SQL(q, exec("DELETE FROM test WHERE val < 500"_L1)); + auto fileSize = QFileInfo{dbPath}.size(); + QVERIFY_SQL(q, exec("VACUUM"_L1)); + QVERIFY(QFileInfo{dbPath}.size() < fileSize); // TEST xTruncate VFS + QVERIFY_SQL(q, exec("SELECT val FROM test ORDER BY val"_L1)); + for (int i = 500; i < 1000; ++i) { + QVERIFY_SQL(q, next()); + QCOMPARE(q.value(0).toInt() , i); + } + db.close(); + QFile::remove(dbPath); +} + +QTEST_APPLESS_MAIN(tst_QVfsSql) +#include "tst_qvfssql.moc"