Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Update expensify_prod branch #1913

Merged
merged 2 commits into from
Oct 22, 2024
Merged

Update expensify_prod branch #1913

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
e54d93f
Update SQLite with more logs, v4
flodnv Oct 22, 2024
2ce076a
Merge pull request #1912 from Expensify/flo_SQLiteLog4
johnmlee101 Oct 22, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ INCLUDE = -I$(PROJECT) -I$(PROJECT)/mbedtls/include
CXXFLAGS = -g -std=c++20 -fPIC -DSQLITE_ENABLE_NORMALIZE $(BEDROCK_OPTIM_COMPILE_FLAG) -Wall -Werror -Wformat-security -Wno-unqualified-std-cast-call -Wno-error=deprecated-declarations $(INCLUDE)

# Amalgamation flags
AMALGAMATION_FLAGS = -Wno-unused-but-set-variable -DSQLITE_ENABLE_FTS5 -DSQLITE_ENABLE_STAT4 -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_SESSION -DSQLITE_ENABLE_PREUPDATE_HOOK -DSQLITE_ENABLE_UPDATE_DELETE_LIMIT -DSQLITE_ENABLE_NOOP_UPDATE -DSQLITE_MUTEX_ALERT_MILLISECONDS=20 -DHAVE_USLEEP=1 -DSQLITE_MAX_MMAP_SIZE=17592186044416ull -DSQLITE_SHARED_MAPPING -DSQLITE_ENABLE_NORMALIZE -DSQLITE_MAX_PAGE_COUNT=4294967294 -DSQLITE_DISABLE_PAGECACHE_OVERFLOW_STATS -DSQLITE_ENABLE_STMT_SCANSTATUS=1
AMALGAMATION_FLAGS = -Wno-unused-but-set-variable -DSQLITE_ENABLE_FTS5 -DSQLITE_ENABLE_STAT4 -DSQLITE_ENABLE_JSON1 -DSQLITE_ENABLE_SESSION -DSQLITE_ENABLE_PREUPDATE_HOOK -DSQLITE_ENABLE_UPDATE_DELETE_LIMIT -DSQLITE_ENABLE_NOOP_UPDATE -DSQLITE_MUTEX_ALERT_MILLISECONDS=20 -DHAVE_USLEEP=1 -DSQLITE_MAX_MMAP_SIZE=17592186044416ull -DSQLITE_SHARED_MAPPING -DSQLITE_ENABLE_NORMALIZE -DSQLITE_MAX_PAGE_COUNT=4294967294 -DSQLITE_DISABLE_PAGECACHE_OVERFLOW_STATS

# All our intermediate, dependency, object, etc files get hidden in here.
INTERMEDIATEDIR = .build
Expand Down
96 changes: 65 additions & 31 deletions libstuff/sqlite3.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
** separate file. This file contains only code for the core SQLite library.
**
** The content in this amalgamation comes from Fossil check-in
** 08e1dea9c070c9a2d668d39ceb153bc8b6d1.
** d02ec06e5bde7c8dc0f21ca2bfc44597aca0.
*/
#define SQLITE_CORE 1
#define SQLITE_AMALGAMATION 1
Expand Down Expand Up @@ -465,7 +465,7 @@ extern "C" {
*/
#define SQLITE_VERSION "3.47.0"
#define SQLITE_VERSION_NUMBER 3047000
#define SQLITE_SOURCE_ID "2024-10-21 11:49:04 08e1dea9c070c9a2d668d39ceb153bc8b6d172e7273f6564a374c43055e84461"
#define SQLITE_SOURCE_ID "2024-10-22 16:26:14 d02ec06e5bde7c8dc0f21ca2bfc44597aca015854d5656f851f4518476d04381"

/*
** CAPI3REF: Run-Time Library Version Numbers
Expand Down Expand Up @@ -18161,6 +18161,7 @@ struct sqlite3 {
#endif

u64 *aPrepareTime;
u64 *aSchemaTime;
};

#define PREPARE_TIME_START 0
Expand All @@ -18183,9 +18184,30 @@ struct sqlite3 {

#define PREPARE_TIME_N 14



#define SCHEMA_TIME_START 0
#define SCHEMA_TIME_AFTER_CREATE_1 1
#define SCHEMA_TIME_AFTER_OPEN_TRANS 2
#define SCHEMA_TIME_AFTER_GET_META 3
#define SCHEMA_TIME_AFTER_FIX_ENCODING 4
#define SCHEMA_TIME_AFTER_SETCACHESIZE 5
#define SCHEMA_TIME_BEGIN_EXEC 6
#define SCHEMA_TIME_BEFORE_STEP 7
#define SCHEMA_TIME_BEFORE_PREPARE 8
#define SCHEMA_TIME_BEFORE_FINALIZE 9
#define SCHEMA_TIME_BEGIN_ANALYZE_LOAD 10
#define SCHEMA_TIME_END_ANALYZE_LOAD 11
#define SCHEMA_TIME_FINISH 12

#define SCHEMA_TIME_N 13
#define SCHEMA_TIME_TIMEOUT (1 * 1000 * 1000)



#define sqlite3PrepareTimeSet(x,y) sqlite3CommitTimeSet(x,y)
SQLITE_PRIVATE void sqlite3PrepareTimeLog(const char *zSql, int nSql, u64 *aPrepareTime);
SQLITE_PRIVATE void sqlite3SchemaTimeLog(Vdbe *pVdbe);
SQLITE_PRIVATE void sqlite3SchemaTimeLog(u64 *aSchemaTime);

#define PREPARE_TIME_TIMEOUT (2 * 1000 * 1000) /* 2 second timeout */

Expand Down Expand Up @@ -93188,7 +93210,7 @@ SQLITE_PRIVATE void sqlite3CommitTimeLog(u64 *aCommit){
(aCommit[ii]==0 ? 0 : (int)(aCommit[ii] - i1))
);
}
sqlite3_log(SQLITE_WARNING, "slow commit (v=3): (%s)", zStr);
sqlite3_log(SQLITE_WARNING, "slow commit (v=4): (%s)", zStr);
sqlite3_free(zStr);
}
}
Expand Down Expand Up @@ -93216,24 +93238,25 @@ SQLITE_PRIVATE void sqlite3PrepareTimeLog(const char *zSql, int nSql, u64 *aPrep
}
if( nByte<0 ){ nByte = sqlite3Strlen30(zSql); }
sqlite3_log(SQLITE_WARNING,
"slow prepare (v=3): (%s) [%.*s]", zStr, nByte, zSql
"slow prepare (v=4): (%s) [%.*s]", zStr, nByte, zSql
);
sqlite3_free(zStr);
}
}
SQLITE_PRIVATE void sqlite3SchemaTimeLog(Vdbe *pVdbe){
int ii;
char *zStr = 0;
for(ii=0; ii<pVdbe->nOp; ii++){
VdbeOp *pOp = &pVdbe->aOp[ii];
zStr = sqlite3_mprintf("%z%s(%s %lld %lld)", zStr, (zStr?" ":""),
(char*)sqlite3OpcodeName(pOp->opcode), pOp->nExec, pOp->nCycle
);
SQLITE_PRIVATE void sqlite3SchemaTimeLog(u64 *aSchema){
u64 i1 = aSchema[SCHEMA_TIME_START];
assert( SCHEMA_TIME_START==0 && SCHEMA_TIME_FINISH==SCHEMA_TIME_N-1 );
if( aSchema[SCHEMA_TIME_FINISH]>(i1+SCHEMA_TIME_TIMEOUT) ){
char *zStr = 0;
int ii;
for(ii=1; ii<SCHEMA_TIME_N; ii++){
zStr = sqlite3_mprintf("%z%s%d", zStr, (zStr?", ":""),
(aSchema[ii]==0 ? 0 : (int)(aSchema[ii] - i1))
);
}
sqlite3_log(SQLITE_WARNING, "slow schema (v=4): (%s)", zStr);
sqlite3_free(zStr);
}
sqlite3_log(SQLITE_WARNING,
"slow schema query (v=3): (%s) [%s]", zStr, pVdbe->zSql
);
sqlite3_free(zStr);
}


Expand Down Expand Up @@ -139696,12 +139719,8 @@ SQLITE_API int sqlite3_exec(
while( rc==SQLITE_OK && zSql[0] ){
int nCol = 0;
char **azVals = 0;
i64 tmStart = sqlite3STimeNow();
u64 svFlags = db->flags;
if( db->init.busy ){
db->flags |= SQLITE_StmtScanStatus;
}

sqlite3PrepareTimeSet(db->aSchemaTime, SCHEMA_TIME_BEFORE_PREPARE);
pStmt = 0;
rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, &zLeftover);
assert( rc==SQLITE_OK || pStmt==0 );
Expand All @@ -139715,6 +139734,7 @@ SQLITE_API int sqlite3_exec(
}
callbackIsInit = 0;

sqlite3PrepareTimeSet(db->aSchemaTime, SCHEMA_TIME_BEFORE_STEP);
while( 1 ){
int i;
rc = sqlite3_step(pStmt);
Expand Down Expand Up @@ -139760,10 +139780,8 @@ SQLITE_API int sqlite3_exec(
}
}

sqlite3PrepareTimeSet(db->aSchemaTime, SCHEMA_TIME_BEFORE_FINALIZE);
if( rc!=SQLITE_ROW ){
if( db->init.busy && (sqlite3STimeNow()-tmStart)>PREPARE_TIME_TIMEOUT ){
sqlite3SchemaTimeLog((Vdbe*)pStmt);
}
rc = sqlite3VdbeFinalize((Vdbe *)pStmt);
pStmt = 0;
zSql = zLeftover;
Expand All @@ -139772,11 +139790,6 @@ SQLITE_API int sqlite3_exec(
}
}

if( db->init.busy ){
/* Clear the SQLITE_StmtScanStatus flag if it was clear at the top
** of this loop. */
db->flags &= (~SQLITE_StmtScanStatus | svFlags);
}
sqlite3DbFree(db, azCols);
azCols = 0;
}
Expand Down Expand Up @@ -145422,6 +145435,11 @@ SQLITE_PRIVATE int sqlite3InitOne(sqlite3 *db, int iDb, char **pzErrMsg, u32 mFl
int openedTransaction = 0;
int mask = ((db->mDbFlags & DBFLAG_EncodingFixed) | ~DBFLAG_EncodingFixed);

u64 aSchemaTime[SCHEMA_TIME_N];
memset(aSchemaTime, 0, sizeof(aSchemaTime));
db->aSchemaTime = aSchemaTime;
sqlite3PrepareTimeSet(aSchemaTime, SCHEMA_TIME_START);

assert( (db->mDbFlags & DBFLAG_SchemaKnownOk)==0 );
assert( iDb>=0 && iDb<db->nDb );
assert( db->aDb[iDb].pSchema );
Expand Down Expand Up @@ -145456,6 +145474,8 @@ SQLITE_PRIVATE int sqlite3InitOne(sqlite3 *db, int iDb, char **pzErrMsg, u32 mFl
goto error_out;
}

sqlite3PrepareTimeSet(aSchemaTime, SCHEMA_TIME_AFTER_CREATE_1);

/* Create a cursor to hold the database open
*/
pDb = &db->aDb[iDb];
Expand All @@ -145479,6 +145499,8 @@ SQLITE_PRIVATE int sqlite3InitOne(sqlite3 *db, int iDb, char **pzErrMsg, u32 mFl
openedTransaction = 1;
}

sqlite3PrepareTimeSet(aSchemaTime, SCHEMA_TIME_AFTER_OPEN_TRANS);

/* Get the database meta information.
**
** Meta values are as follows:
Expand All @@ -145504,6 +145526,8 @@ SQLITE_PRIVATE int sqlite3InitOne(sqlite3 *db, int iDb, char **pzErrMsg, u32 mFl
}
pDb->pSchema->schema_cookie = meta[BTREE_SCHEMA_VERSION-1];

sqlite3PrepareTimeSet(aSchemaTime, SCHEMA_TIME_AFTER_GET_META);

/* If opening a non-empty database, check the text encoding. For the
** main database, set sqlite3.enc to the encoding of the main database.
** For an attached db, it is an error if the encoding is not the same
Expand Down Expand Up @@ -145539,6 +145563,8 @@ SQLITE_PRIVATE int sqlite3InitOne(sqlite3 *db, int iDb, char **pzErrMsg, u32 mFl
}
pDb->pSchema->enc = ENC(db);

sqlite3PrepareTimeSet(aSchemaTime, SCHEMA_TIME_AFTER_FIX_ENCODING);

if( pDb->pSchema->cache_size==0 ){
#ifndef SQLITE_OMIT_DEPRECATED
size = sqlite3AbsInt32(meta[BTREE_DEFAULT_CACHE_SIZE-1]);
Expand All @@ -145550,6 +145576,8 @@ SQLITE_PRIVATE int sqlite3InitOne(sqlite3 *db, int iDb, char **pzErrMsg, u32 mFl
sqlite3BtreeSetCacheSize(pDb->pBt, pDb->pSchema->cache_size);
}

sqlite3PrepareTimeSet(aSchemaTime, SCHEMA_TIME_AFTER_SETCACHESIZE);

/*
** file_format==1 Version 3.0.0.
** file_format==2 Version 3.1.3. // ALTER TABLE ADD COLUMN
Expand Down Expand Up @@ -145590,18 +145618,21 @@ SQLITE_PRIVATE int sqlite3InitOne(sqlite3 *db, int iDb, char **pzErrMsg, u32 mFl
xAuth = db->xAuth;
db->xAuth = 0;
#endif
sqlite3PrepareTimeSet(aSchemaTime, SCHEMA_TIME_BEGIN_EXEC);
rc = sqlite3_exec(db, zSql, sqlite3InitCallback, &initData, 0);
#ifndef SQLITE_OMIT_AUTHORIZATION
db->xAuth = xAuth;
}
#endif
if( rc==SQLITE_OK ) rc = initData.rc;
sqlite3DbFree(db, zSql);
sqlite3PrepareTimeSet(aSchemaTime, SCHEMA_TIME_BEGIN_ANALYZE_LOAD);
#ifndef SQLITE_OMIT_ANALYZE
if( rc==SQLITE_OK ){
sqlite3AnalysisLoad(db, iDb);
}
#endif
sqlite3PrepareTimeSet(aSchemaTime, SCHEMA_TIME_END_ANALYZE_LOAD);
}
assert( pDb == &(db->aDb[iDb]) );
if( db->mallocFailed ){
Expand Down Expand Up @@ -145635,6 +145666,9 @@ SQLITE_PRIVATE int sqlite3InitOne(sqlite3 *db, int iDb, char **pzErrMsg, u32 mFl
sqlite3BtreeLeave(pDb->pBt);

error_out:
db->aSchemaTime = 0;
sqlite3PrepareTimeSet(aSchemaTime, SCHEMA_TIME_FINISH);
sqlite3SchemaTimeLog(aSchemaTime);
if( rc ){
if( rc==SQLITE_NOMEM || rc==SQLITE_IOERR_NOMEM ){
sqlite3OomFault(db);
Expand Down Expand Up @@ -257598,7 +257632,7 @@ static void fts5SourceIdFunc(
){
assert( nArg==0 );
UNUSED_PARAM2(nArg, apUnused);
sqlite3_result_text(pCtx, "fts5: 2024-10-21 11:49:04 08e1dea9c070c9a2d668d39ceb153bc8b6d172e7273f6564a374c43055e84461", -1, SQLITE_TRANSIENT);
sqlite3_result_text(pCtx, "fts5: 2024-10-22 16:26:14 d02ec06e5bde7c8dc0f21ca2bfc44597aca015854d5656f851f4518476d04381", -1, SQLITE_TRANSIENT);
}

/*
Expand Down
2 changes: 1 addition & 1 deletion libstuff/sqlite3.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -148,7 +148,7 @@ extern "C" {
*/
#define SQLITE_VERSION "3.47.0"
#define SQLITE_VERSION_NUMBER 3047000
#define SQLITE_SOURCE_ID "2024-10-21 11:49:04 08e1dea9c070c9a2d668d39ceb153bc8b6d172e7273f6564a374c43055e84461"
#define SQLITE_SOURCE_ID "2024-10-22 16:26:14 d02ec06e5bde7c8dc0f21ca2bfc44597aca015854d5656f851f4518476d04381"

/*
** CAPI3REF: Run-Time Library Version Numbers
Expand Down
Loading