Matrix SQLiteLint使用及源碼分析

前言

這篇文章差很少是去年這個時候寫的,寫好了一直忘了發出來,如今補發一下...html

SQLiteLint。雖然名帶「lint」,但並非代碼的靜態檢查,而是在APP運行時對sql語句、執行序列、表信息等進行分析檢測。而和「lint」有點相似的是:在開發階段就介入,並運用一些最佳實踐的規則來檢測,從而發現潛在的、可疑的SQLite使用問題。java

SQLiteLint的用處

在真正使用SQLiteLint以前先說明一下SQLiteLint的用處,如下用處提煉自SQLiteLint 官方wikiandroid

  1. 檢測索引使用問題
  • 未建索引致使的全表掃描
  • 索引未生效致使的全表掃描
  • 沒必要要的臨時建樹排序
  • 不足夠的索引組合
  1. 檢測冗餘索引問題
  2. 檢測select * 問題
  3. 檢測Autoincrement問題
  4. 檢測建議使用prepared statement
  5. 檢測建議使用without rowid特性

快速接入

1.添加依賴

  • gralde.propeties指定Matrix版本 MATRIX_VERSION=0.5.2(編寫該文檔時的最新版本)
  • 添加 dependencies
dependencies {
    debugCompile "com.tencent.matrix:matrix-sqlite-lint-android-sdk:${MATRIX_VERSION}"
    releaseCompile "com.tencent.matrix:matrix-sqlite-lint-android-sdk-no-op:${MATRIX_VERSION}"
}
複製代碼

2.SQLiteLint初始化並添加關注的database

Application#onCreate方法中初始化git

SQLiteLintConfig config =new SQLiteLintConfig(SQLiteLint.SqlExecutionCallbackMode.HOOK);
            SQLiteLintPlugin sqLiteLintPlugin = new SQLiteLintPlugin(config);
            builder.plugin(sqLiteLintPlugin);

            Matrix.init(builder.build());
            sqLiteLintPlugin.start();
            SQLiteLintPlugin plugin = (SQLiteLintPlugin) Matrix.with().getPluginByClass(SQLiteLintPlugin.class);
            if (plugin == null) {
                return;
            }
            if (!plugin.isPluginStarted()) {
                plugin.start();
            }
            plugin.addConcernedDB(new SQLiteLintConfig.ConcernDb(TestDBHelper.get().getWritableDatabase())
                    //.setWhiteListXml(R.xml.sqlite_lint_whitelist)//disable white list by default
                    .enableAllCheckers());
複製代碼

實際測試使用

這裏結合Matrix 官方例子來看。github

public class TestDBHelper extends SQLiteOpenHelper {
    private static final int DB_VERSION = 1;
    private static final String DB_NAME = "sqliteLintTest.db";
    public static final String TABLE_NAME = "testTable";
    public static final String TABLE_NAME_AUTO_INCREMENT = "testTableAutoIncrement";
    public static final String TABLE_NAME_WITHOUT_ROWID_BETTER = "testTableWithoutRowid";
    public static final String TABLE_NAME_Redundant_index = "testTableRedundantIndex";
    public static final String TABLE_NAME_CONTACT = "contact";
    private static TestDBHelper mHelper = null;
	...
    @Override
    public void onCreate(SQLiteDatabase sqLiteDatabase) {
        String sql = "create table if not exists " + TABLE_NAME + " (Id integer primary key, name text, age integer)";
        sqLiteDatabase.execSQL(sql);
        String sqlAutoIncrement = "create table if not exists " + TABLE_NAME_AUTO_INCREMENT + " (Id integer primary key AUTOINCREMENT, name text, age integer)";
        sqLiteDatabase.execSQL(sqlAutoIncrement);
        String sqlWithoutRowId = "create table if not exists " + TABLE_NAME_WITHOUT_ROWID_BETTER + " (Id text primary key, name integer, age integer)";
        sqLiteDatabase.execSQL(sqlWithoutRowId);
        String sqlRedundantIndex = "create table if not exists " + TABLE_NAME_Redundant_index + " (Id text, name text, age integer, gender integer)";
        sqLiteDatabase.execSQL(sqlRedundantIndex);
        String indexSql = "create index if not exists index_age on " + TABLE_NAME_Redundant_index + "(age);";
        String indexSql2 = "create index if not exists index_age_name on " + TABLE_NAME_Redundant_index + "(age, name);";
        String indexSql3 = "create index if not exists index_name_age on " + TABLE_NAME_Redundant_index + "(name,age);";
        String indexSql4 = "create index if not exists index_id on " + TABLE_NAME_Redundant_index + "(Id);";

        sqLiteDatabase.execSQL(indexSql);
        sqLiteDatabase.execSQL(indexSql2);
        sqLiteDatabase.execSQL(indexSql3);
        sqLiteDatabase.execSQL(indexSql4);

        String contact = "create table if not exists " + TABLE_NAME_CONTACT + " (Id integer primary key, name text, age integer, gender integer, status integer)";
        sqLiteDatabase.execSQL(contact);
        String contactIndex = "create index if not exists index_age_name_status on " + TABLE_NAME_CONTACT + "(age, name, status);";
        String contactIndex2 = "create index if not exists index_name_age_status on " + TABLE_NAME_CONTACT + "(name, age, status);";
        String contactStatusIndex = "create index if not exists index_status on " + TABLE_NAME_CONTACT + "(status);";
        sqLiteDatabase.execSQL(contactIndex);
        sqLiteDatabase.execSQL(contactIndex2);
        sqLiteDatabase.execSQL(contactStatusIndex);
    }
	...
}
複製代碼

這裏新建了幾個數據庫表:sql

testTable:正常的表,integer 類型的id做爲主鍵數據庫

testTableAutoIncrement:id 做爲主鍵,且是自增屬性api

testTableWithoutRowid:設置了withoutRowid屬性性能優化

testTableRedundantIndex:設置多個索引app

contact:主要用於多條件查詢

如下爲測試的sql語句:

public static String[] getTestSqlList() {
        String[] list = new String[]{
                "select * from testTable",//select *
                "select name from testTable where age>10",//no index
                "select name from testTableRedundantIndex where age&2 != 0",//not use index
                "select name from testTableRedundantIndex where name like 'j%'",//not use index
                "select name from testTableRedundantIndex where name = 'jack' and age > 20",
                "select testTable.name from testTable, testTableAutoIncrement where testTableAutoIncrement.age=testTable.age",
                "select Id from testTable where age = 10 union select Id from testTableRedundantIndex where age > 10",//union
                "select name from testTable order by age",//use tmp tree
                "select name from testTableRedundantIndex where gender=1 and age=5",//bigger index
                "select name, case when age>=18 then 'Adult' else 'child' end LifeStage from testTableRedundantIndex where age > 20 order by age,name,gender",
                "select name,age,gender from testTableRedundantIndex where age > 10 and age < 20 or id between 30 and 40 or id = 1000 ORDER BY name,age,gender desc limit 10 offset 2;",
                "select * from (select * from testTable where age = 18 order by age limit 10) as tb where age = 18 " +
                        "UNION select m.* from testTable AS m, testTableRedundantIndex AS c where m.age = c.age;",
                "SELECT name FROM testTable WHERE name not LIKE 'rt%' OR name LIKE 'rc%' AND age > 20 GROUP BY name ORDER BY age;",
                "SELECT id AS id_alias FROM testTable AS test_alias WHERE id_alias = 1 or id = 2",
                "SELECT name FROM testTable WHERE id = (SELECT id FROM testTableRedundantIndex WHERE name = 'hello world')",
                "SELECT * FROM testTable where name = 'rc' UNION SELECT * FROM testTableWithoutRowid UNION SELECT * FROM testTableAutoIncrement",
                "SELECT name FROM testTable WHERE AGE GLOB '2*';",
                "SELECT DISTINCT name FROM testTable GROUP BY name HAVING count(name) < 2;",
                "SELECT name FROM contact WHERE status = 2;",
                "select rowid from contact where name = 'rr' or age > 12",
                "select t1.name ,t2.age from testTable as t1,testTableRedundantIndex as t2 where t1.id = t2.id and (t1.age=23 or t2.age=12);",
                "select t1.name ,t2.age from testTable as t1,testTableRedundantIndex as t2 where t1.id = t2.id and (t1.age=23 and t2.age=12);",
                "select name,age from contact where name like 'w%' and age > 12",
                "select name,age from contact where name >= 'rc' and age&2=1",
                "select name,age from contact where name = 'r' or age > 12 or status = 1",
        };
        return list;
    }
複製代碼

最終檢測結果以下圖:

共檢查出31項建議項或者提示項,每一個建議或提示點擊後能夠查看詳細結果,詳細結果頁面會展現檢查的sql語句,sql語句explain qurey plan結果,優化建議,部分詳情頁會有堆棧信息。以下圖:

根據優化建議,咱們能夠知道SQLiteLint建議咱們創建name和age的複合索引。

SQLiteLint 存在問題

  1. targetSdkVersion>=28 hook失效

    targetsdk>=28後,SqliteDebug屬於Hide類型,反射沒法訪問。建議若是能夠,暫時修改targetSdkVersion用以分析數據庫,修改完後再改回來。

  2. 部分app接入後出現using JNI after critical get崩潰。經排查發現是項目中存在虛擬數據表,後面在Application的onCreate方法中初始化並啓動SQLiteLint,再添加關注的數據庫後解決,注意,這裏不要添加含有虛擬表的數據庫。

SQLiteLint源碼分析

從啓動SQLiteLintPlugin開始看起,

SQLiteLintConfig config = initSQLiteLintConfig();
            SQLiteLintPlugin sqLiteLintPlugin = new SQLiteLintPlugin(config);
            builder.plugin(sqLiteLintPlugin);

            Matrix.init(builder.build());
            sqLiteLintPlugin.start();

    private static SQLiteLintConfig initSQLiteLintConfig() {
        try {
            /** * * HOOK模式下,SQLiteLint會本身去獲取全部已執行的sql語句及其耗時(by hooking sqlite3_profile) * @see 而另外一個模式:SQLiteLint.SqlExecutionCallbackMode.CUSTOM_NOTIFY , 則須要調用 {@link SQLiteLint#notifySqlExecution(String, String, int)}來通知 * SQLiteLint 須要分析的、已執行的sql語句及其耗時 * @see TestSQLiteLintActivity#doTest() */
            return new SQLiteLintConfig(SQLiteLint.SqlExecutionCallbackMode.HOOK);
        } catch (Throwable t) {
            return new SQLiteLintConfig(SQLiteLint.SqlExecutionCallbackMode.HOOK);
        }
    }
複製代碼

在這裏初始化了一個SQLiteLintConfig,設置了sql執行語句的分析回調模式爲Hook方式。

public SQLiteLintConfig(SQLiteLint.SqlExecutionCallbackMode sqlExecutionCallbackMode) {
        SQLiteLint.setSqlExecutionCallbackMode(sqlExecutionCallbackMode);
        sConcernDbList = new ArrayList<>();
    }
	
	//com.tencent.sqlitelint.SQLiteLint#setSqlExecutionCallbackMode
	public static void setSqlExecutionCallbackMode(SqlExecutionCallbackMode sqlExecutionCallbackMode){
        if (sSqlExecutionCallbackMode != null) {
            return;
        }

        sSqlExecutionCallbackMode = sqlExecutionCallbackMode;
        if (sSqlExecutionCallbackMode == SqlExecutionCallbackMode.HOOK) {
            // hook must called before open the database
            SQLite3ProfileHooker.hook();
        }
    }
複製代碼

SQLiteLintConfig的構造方法主要就是設置了SqlExecutionCallbackMode,繼續跟下去能夠看到,若是是HOOK方式,則調用SQLite3ProfileHooker.hook()執行hook操做。hook方法內部最終會調用到com.tencent.sqlitelint.util.SQLite3ProfileHooker#doHook

private static boolean doHook() {
		//hookOpenSQLite3Profile方法主要做用是將SQLiteDebug類中的DEBUG_SQL_TIME變量設置爲true
        if (!hookOpenSQLite3Profile()) {
            SLog.i(TAG, "doHook hookOpenSQLite3Profile failed");
            return false;
        }
		//nativeDoHook是個native方法,hook住native層sqlite3_profile方法註冊sql執行結果回調
        return nativeDoHook();
    }
複製代碼

nativeDoHook 是native方法,先按下不表,繼續來看SQLiteLintPlugin調用start方法以後作了什麼。 com.tencent.sqlitelint.SQLiteLintPlugin#start

@Override
    public void start() {
        super.start();
        if (!isSupported()) {
            return;
        }

        SQLiteLint.setReportDelegate(new IssueReportBehaviour.IReportDelegate() {
            @Override
            public void report(SQLiteLintIssue issue) {
                if (issue == null) {
                    return;
                }
				//issue上報
                reportMatrixIssue(issue);
            }
        });

        List<SQLiteLintConfig.ConcernDb> concernDbList = mConfig.getConcernDbList();
        for (int i = 0; i < concernDbList.size(); i++) {
            SQLiteLintConfig.ConcernDb concernDb = concernDbList.get(i);
            String concernedDbPath = concernDb.getInstallEnv().getConcernedDbPath();
            SQLiteLint.install(mContext, concernDb.getInstallEnv(), concernDb.getOptions());
			//設置sql語句不進行檢查的白名單
            SQLiteLint.setWhiteList(concernedDbPath, concernDb.getWhiteListXmlResId());
			//設置檢查項
            SQLiteLint.enableCheckers(concernedDbPath, concernDb.getEnableCheckerList());
        }
    }
複製代碼

SQLiteLint檢查項簡單說明:

  1. ExplainQueryPlanChecker :在sqlite命令行中可經過explain query plan sql語句獲取sqlite 特色SQL查詢的策略或計劃的高級描述,最重要的是可報告查詢使用數據庫索引的方式
  2. AvoidSelectAllChecker: sql語句使用select * 檢查
  3. WithoutRowIdBetterChecker: sql語句建表時without rowid檢查
  4. PreparedStatementBetterChecker: sql 語句PreparedStatement檢查
  5. RedundantIndexChecker:冗餘索引檢查

具體檢查項以及檢查依據請查看Matrix Android SQLiteLint

SQLiteLint install後執行的操做:

//com.tencent.sqlitelint.SQLiteLint#install
    public static void install(Context context, InstallEnv installEnv, Options options) {
        assert installEnv != null;
        assert sSqlExecutionCallbackMode != null
                : "SqlExecutionCallbackMode is UNKNOWN!setSqlExecutionCallbackMode must be called before install";

        options = (options == null) ? Options.LAX : options;

        SQLiteLintAndroidCoreManager.INSTANCE.install(context, installEnv, options);
    }
	
	
	//com.tencent.sqlitelint.SQLiteLintAndroidCoreManager#install
    public void install(Context context, SQLiteLint.InstallEnv installEnv, SQLiteLint.Options options) {
        String concernedDbPath = installEnv.getConcernedDbPath();
        if (mCoresMap.containsKey(concernedDbPath)) {
            SLog.w(TAG, "install twice!! ignore");
            return;
        }

        SQLiteLintAndroidCore core = new SQLiteLintAndroidCore(context, installEnv, options);
        mCoresMap.put(concernedDbPath, core);
    }

	//com.tencent.sqlitelint.SQLiteLintAndroidCore#SQLiteLintAndroidCore
    SQLiteLintAndroidCore(Context context, SQLiteLint.InstallEnv installEnv, SQLiteLint.Options options) {
        mContext = context;
        //初始化SQLiteLintInternal.db,用於存放檢查發現的issue
        SQLiteLintDbHelper.INSTANCE.initialize(context);
        mConcernedDbPath = installEnv.getConcernedDbPath();
        mSQLiteExecutionDelegate = installEnv.getSQLiteExecutionDelegate();

        if (SQLiteLint.getSqlExecutionCallbackMode() == SQLiteLint.SqlExecutionCallbackMode.HOOK) {
            //hook sqlite3_profile api
            SQLite3ProfileHooker.hook();
        }
		//開啓檢查,下面會繼續分析
        SQLiteLintNativeBridge.nativeInstall(mConcernedDbPath);

        //設置發現issue後的行爲
        mBehaviors = new ArrayList<>();
        /*PersistenceBehaviour is a default pre-behaviour */
        mBehaviors.add(new PersistenceBehaviour());
        if (options.isAlertBehaviourEnable()) {
            mBehaviors.add(new IssueAlertBehaviour(context, mConcernedDbPath));
        }
        if (options.isReportBehaviourEnable()) {
            mBehaviors.add(new IssueReportBehaviour(SQLiteLint.sReportDelegate));
        }
    }
複製代碼

上面所說的nativeDoHook以下:

JNIEXPORT jboolean JNICALL Java_com_tencent_sqlitelint_util_SQLite3ProfileHooker_nativeDoHook(JNIEnv *env, jobject /* this */) {
        LOGI("SQLiteLintHooker_nativeDoHook");
        if (!kInitSuc) {
            LOGW("SQLiteLintHooker_nativeDoHook kInitSuc failed");
            return false;
        }
        loaded_soinfo* soinfo = elfhook_open("libandroid_runtime.so");
        if (!soinfo) {
            LOGW("Failure to open libandroid_runtime.so");
            return false;
        }
        if (!elfhook_replace(soinfo, "c", (void*)hooked_sqlite3_profile, (void**)&original_sqlite3_profile)) {
            LOGW("Failure to hook sqlite3_profile");
            elfhook_close(soinfo);
            soinfo = nullptr;
            return false;
        }
        elfhook_close(soinfo);
        soinfo = nullptr;

        kStop = false;

        return true;
    }
複製代碼

重點關注elfhook_replace方法所作的事情,這裏採用PLT(GOT) Hook的方式hook了系統android_runtime.so中的sqlite3_profile方法。

爲何hook了sqlite3_profile方法就能夠達到咱們的要求? 每一個SQL語句完成後,將調用由sqlite3_profile註冊的回調函數。配置文件回調包含原始語句文本以及該語句運行多長時間的掛鐘時間的估計值.

再來看看hooked_sqlite3_profile中的邏輯:

void* hooked_sqlite3_profile(sqlite3* db, void(*xProfile)(void*, const char*, sqlite_uint64), void* p) {
        LOGI("hooked_sqlite3_profile call");
        return original_sqlite3_profile(db, SQLiteLintSqlite3ProfileCallback, p);
    }
複製代碼

hooked_sqlite3_profile中作的事情很是簡單,使用原有sqlite3_profile方法並設置了一個SQLiteLintSqlite3ProfileCallback回調,由於這個SQLiteLintSqlite3ProfileCallback咱們才能夠拿到sqlite profile的結果。

void Java_com_tencent_sqlitelint_SQLiteLintNativeBridge_nativeNotifySqlExecute(JNIEnv *env, jobject, jstring dbPath , jstring sql, jlong executeTime, jstring extInfo) {
        char *filename = jstringToChars(env, dbPath);
        char *ext_info = jstringToChars(env, extInfo);
        char *jsql = jstringToChars(env, sql);

        NotifySqlExecution(filename, jsql, executeTime, ext_info);

        free(jsql);
        free(ext_info);
        free(filename);
    }
複製代碼

以前所說的SqlExecutionCallbackMode 一個是Hook,一個是NotifySqlExecution,能夠看到sql語句執行後hook方式會主動調用NotifySqlExecution, 傳入須要分析的、已執行的sql語句及其耗時參數,因此咱們纔不須要手動調用SQLiteLint#notifySqlExecution(String, String, int)來通知。

SQLiteLintNativeBridge.nativeInstall(mConcernedDbPath)調用了native方法nativeInstall

void Java_com_tencent_sqlitelint_SQLiteLintNativeBridge_nativeInstall(JNIEnv *env, jobject, jstring name) {
        char *filename = jstringToChars(env,name);
        InstallSQLiteLint(filename, OnIssuePublish);
        free(filename);
        SetSqlExecutionDelegate(SqliteLintExecSql);
    }
	
	//sqlitelint::InstallSQLiteLint
    void InstallSQLiteLint(const char* db_path, OnPublishIssueCallback issue_callback) {
        LintManager::Get()->Install(db_path, issue_callback);
    }
	
	//LintManager::Install
	void LintManager::Install(const char* db_path, OnPublishIssueCallback issued_callback) {
        sInfo("LintManager::Install dbPath:%s", db_path);
        std::unique_lock<std::mutex> lock(lints_mutex_);
        std::map<const std::string, Lint*>::iterator it = lints_.find(db_path);
        if (it != lints_.end()) {
            lock.unlock();
            sWarn("Install already installed; dbPath: %s", db_path);
            return;
        }

        Lint* lint = new Lint(db_path, issued_callback);
        lints_.insert(std::pair<const std::string, Lint*>(db_path, lint));
        lock.unlock();
    }

複製代碼

LintManager::Install方法中新建了一個Lint,在其構造方法開啓了一個線程,死循環執行check任務。TakeSqlInfo不斷從執行的sql語句中獲取執行結果信息,當沒有sql語句時會進入wait狀態,當有sql語句執行完後,系統會調用sqlite3_profile,而咱們在sqlite3_profile回調中執行了Lint::NotifySqlExecution 方法,這個方法會執行notify方法,使得從新進入運行狀態,使用可用的檢查器開始sql語句檢查。

void Lint::Check() {
        init_check_thread_ = new std::thread(&Lint::InitCheck, this);

        std::vector<Issue>* published_issues = new std::vector<Issue>;
        std::unique_ptr<SqlInfo> sql_info;
        SqlInfo simple_sql_info;
        while (true) {
            int ret = TakeSqlInfo(sql_info);
            if (ret != 0) {
                sError("check exit");
                break;
            }
            //sql語句計數
            env_.IncSqlCnt();
            //sql語句預處理,去除多餘空格,sql語句轉成小寫
            PreProcessSqlString(sql_info->sql_);
            sDebug("Lint::Check checked cnt=%d", env_.GetSqlCnt());
            //是否支持檢查,只支持select,update,delete,insert,replace類型的sql語句
            if (!IsSqlSupportCheck(sql_info->sql_)) {
                sDebug("Lint::Check Sql not support");
                env_.AddToSqlHistory(*sql_info);
                sql_info = nullptr;
                continue;
            }

            //預處理,按sql語句類別不一樣分別作處理
            if (!PreProcessSqlInfo(sql_info.get())) {
                sWarn("Lint::Check PreProcessSqlInfo failed");
                env_.AddToSqlHistory(*sql_info);
                sql_info = nullptr;
                continue;
            }

            sql_info->CopyWithoutParse(simple_sql_info);
            env_.AddToSqlHistory(simple_sql_info);

            published_issues->clear();

            //各種檢查器檢查sql語句
            ScheduleCheckers(CheckScene::kSample, *sql_info, published_issues);

            const std::string& wildcard_sql = sql_info->wildcard_sql_.empty() ? sql_info->sql_ : sql_info->wildcard_sql_;
            bool checked = false;
            if (!checked_sql_cache_.Get(wildcard_sql, checked)) {
                ScheduleCheckers(CheckScene::kUncheckedSql, *sql_info, published_issues);
                checked_sql_cache_.Put(wildcard_sql, true);
            } else {
                sVerbose("Lint::Check() already checked recently");
            }

            if (!published_issues->empty()) {
                sInfo("New check some diagnosis out!, sql=%s", sql_info->sql_.c_str());
                if (issued_callback_) {
                    //issue上報
                    issued_callback_(env_.GetDbPath().c_str(), *published_issues);
                }
            }

            sql_info = nullptr;
            env_.CheckReleaseHistory();
        }

        sError("check break");
        delete published_issues;
    }
複製代碼

至此,SQLiteLint流程基本已經串通了。用兩張圖來講明整個流程:

初始化流程

sql檢查流程

總結

經過Matrix SQLiteLint,可讓咱們在開發、測試或者灰度階段進行sql語句檢查,能夠寫出更加高效性能更好的sql語句,特別是對數據庫使用和優化沒有太多經驗的開發人員來講是很是有用的。固然,SQLiteLint也不是完美的,SQLiteLint檢測只是建議性質,特別是關於索引的檢測有可能會有誤報,而且有些狀況下是沒法避免索引失效的,所以是否須要修改要結合實際狀況。

參考文章: Query Planning Sqlite索引優化 數據庫索引原理及優化 性能優化之數據庫優化 索引在什麼狀況下會失效

相關文章
相關標籤/搜索