HI I am use SQLite version 3.3.6
default page_size is 4096 , i set 1024 i have four question ,i don't understand 1: before i use insert cmd ,the linux memory and sqlite3_memory_used is linux: tota=29126656 used=16998400 free=12128256 sqlite3_memory_used:1372576 after i use inset cmd ,the linux memory and sqlite3_memory_used is linux: tota=29126656 used=17608704 free=11517952 sqlite3_memory_used:1710016 why 1710016-1372576=337440 != 12128256-11517952=610304 2: i keep the sqlite3 run two days,i found when i use sqlite select cmd or insert cmd i found sqlite3_memory_used = 2315024 is not change but linux free memory is reduce(use free cmd) if i comment out the select cmd and insert cmd ,linux free memory is not change this whether or not a Normal behavior ? 3: sqlite3_memory_used return value whether or not include the memory leak? 4: i found anther situation,when linux free memory reduce to about 1M,free memory immediately increase to about 8M my code: void insert_table_data(mtype_group_str table,uint8_t class_type,char time_str[]) { char sql_cmd[MAX_BUFF_LEN]; char meter_addr_str[CJT188_ADDR_LEN*2+100]; int32_t result; sqlite3_stmt *stmt=NULL; uint8_t id_index; uint16_t meter_index; uint32_t di_offset=0; printf_mem_infor("insert_table_data1"); begin_transaction(); printf_mem_infor("insert_table_data2"); make_insert_cmd(table,class_type,sql_cmd); printf_mem_infor("insert_table_data3"); result = sqlite3_prepare_v2(sqlite_mdb.mdb,sql_cmd,-1,&stmt,NULL); if(result != SQLITE_OK) { elog_d(SYS_LOG_MDB,"insert_table_data:result=%d",result); sqlite3_finalize(stmt); rollback_transaction(); return ; } printf_mem_infor("insert_table_data4"); for(meter_index = 0; meter_index < table.m_num; meter_index++) { result = sqlite3_bind_text(stmt,1,xxxxxxx,-1,SQLITE_STATIC); if(result != SQLITE_OK) { elog_d(SYS_LOG_MDB,"insert_table_data:result1=%d",result); } di_offset = 0; if(class_type == REAL_DATA) { result = sqlite3_bind_text(stmt,2,time_str,-1,SQLITE_STATIC); if(result != SQLITE_OK) { elog_d(SYS_LOG_MDB,"insert_table_data:result2=%d",result); } } else if(class_type == DAY_DATA) { result = sqlite3_bind_text(stmt,2,time_str,-1,SQLITE_STATIC); if(result != SQLITE_OK) { elog_d(SYS_LOG_MDB,"insert_table_data:result3=%d",result); } } else if(class_type == MONTH_DATA) { result = sqlite3_bind_text(stmt,2,time_str,-1,SQLITE_STATIC); if(result != SQLITE_OK) { elog_d(SYS_LOG_MDB,"insert_table_data:result4=%d",result); } } for(id_index = 0; id_index < table.mtype_infor[class_type].di_sum; id_index++) { di_offset += ID_INDEX_SIZE; result = sqlite3_bind_blob(stmt,id_index+3,xxxxxxx,SQLITE_STATIC); if(result != SQLITE_OK) { elog_d(SYS_LOG_MDB,"insert_table_data:result5=%d",result); } di_offset += DATA_TIME_SIZE + table.mtype_infor[class_type].mtype_di[id_index].data_len; } printf_mem_infor("insert_table_data5"); result = sqlite3_step(stmt); printf_mem_infor("insert_table_data6"); if(result != SQLITE_DONE) { elog_d(SYS_LOG_MDB,"result = sqlite3_step(stmt)=%d",result); rollback_transaction(); } printf_mem_infor("insert_table_data7"); result = sqlite3_reset(stmt); printf_mem_infor("insert_table_data8"); if(result != SQLITE_OK) { elog_d(SYS_LOG_MDB,"result = sqlite3_reset(stmt)=%d",result); } printf_mem_infor("insert_table_data9"); result = sqlite3_clear_bindings(stmt); printf_mem_infor("insert_table_data10"); if(result != SQLITE_OK) { elog_d(SYS_LOG_MDB,"result = sqlite3_clear_bindings(stmt)=%d",result); } } printf_mem_infor("insert_table_data11"); sqlite3_finalize(stmt); printf_mem_infor("insert_table_data12"); commit_transaction(); printf_mem_infor("insert_table_data13"); } uint8_t get_fre_infor(mtype_group_str table,uint8_t class_type,char time_str[]) { char sql_cmd[MAX_BUFF_LEN]; int32_t result; char *errmsg=NULL,**dbresult=NULL; int16_t amr_time_index = 0; int32_t nrow = 0,ncol=0; sprintf(sql_cmd,"select distinct amr_time from tab_%d_%d_%c order by amr_time asc",table.big_type,table.sml_type,data_type_mark[class_type]); result = sqlite3_get_table(sqlite_mdb.mdb,sql_cmd,&dbresult,&nrow,&ncol,&errmsg); elog_d(SYS_LOG_MDB,"get_fre_infor:ncol=%d nrow=%d",ncol,nrow); if(result == SQLITE_OK) { for(amr_time_index = 0; amr_time_index < nrow; amr_time_index++) { elog_d(SYS_LOG_MDB,"dbresult[%d]=%s",amr_time_index,dbresult[amr_time_index+1]); if(strcmp(time_str,dbresult[amr_time_index+1])==0) { elog_d(SYS_LOG_MDB,"find freeze time"); sqlite3_free_table(dbresult); //if(errmsg!=NULL) sqlite3_free(errmsg); return RET_OK_P1; } } if(amr_time_index == nrow) { if(nrow >= data_save_max[class_type]) { elog_d(SYS_LOG_MDB,"save cnt full"); sprintf(time_str,"%s",dbresult[1]); //if(errmsg!=NULL) sqlite3_free(errmsg); sqlite3_free_table(dbresult); elog_d(SYS_LOG_MDB,"free finish"); return RET_OK_P1; } else { elog_d(SYS_LOG_MDB,"not find freeze time and save cnt not full"); //if(errmsg!=NULL) sqlite3_free(errmsg); sqlite3_free_table(dbresult); elog_d(SYS_LOG_MDB,"free finish"); return RET_FAIL_Z0; } } } else { elog_d(SYS_LOG_MDB,"get_fre_infor:result=%d",result); sqlite3_free_table(dbresult); sqlite3_free(errmsg); return RET_FAIL_Z0; } return RET_FAIL_Z0; } /** Disk-Space Utilization Report For sfssjzq.db Page size in bytes................................ 1024 Pages in the whole file (measured)................ 1858 Pages in the whole file (calculated).............. 1858 Pages that store data............................. 1858 100.0% Pages on the freelist (per header)................ 0 0.0% Pages on the freelist (calculated)................ 0 0.0% Pages of auto-vacuum overhead..................... 0 0.0% Number of tables in the database.................. 4 Number of indices................................. 3 Number of defined indices......................... 3 Number of implied indices......................... 0 Size of the file in bytes......................... 1902592 Bytes of user payload stored...................... 767040 40.3% *** Page counts for all tables with their indices ***************************** TAB_25_22_R....................................... 1362 73.3% TAB_25_22_D....................................... 254 13.7% TAB_25_22_M....................................... 241 13.0% SQLITE_MASTER..................................... 1 0.054% *** Page counts for all tables and indices separately ************************* TAB_25_22_R_INDEX................................. 718 38.6% TAB_25_22_R....................................... 644 34.7% TAB_25_22_D_INDEX................................. 133 7.2% TAB_25_22_M_INDEX................................. 127 6.8% TAB_25_22_D....................................... 121 6.5% TAB_25_22_M....................................... 114 6.1% SQLITE_MASTER..................................... 1 0.054% *** All tables and indices **************************************************** Percentage of total database...................... 100.0% Number of entries................................. 28566 Bytes of storage consumed......................... 1902592 Bytes of payload.................................. 1577410 82.9% Bytes of metadata................................. 139702 7.3% Average payload per entry......................... 55.22 Average unused bytes per entry.................... 6.49 Average metadata per entry........................ 4.89 Average fanout.................................... 23.00 Maximum payload per entry......................... 175 Entries that use overflow......................... 0 0.0% Index pages used.................................. 79 Primary pages used................................ 1779 Overflow pages used............................... 0 Total pages used.................................. 1858 Unused bytes on index pages....................... 15303 18.9% Unused bytes on primary pages..................... 170177 9.3% Unused bytes on overflow pages.................... 0 Unused bytes on all pages......................... 185480 9.7% *** All tables **************************************************************** Percentage of total database...................... 47.4% Number of entries................................. 14286 Bytes of storage consumed......................... 901120 Bytes of payload.................................. 767914 85.2% Bytes of metadata................................. 85138 9.4% Average payload per entry......................... 53.75 Average unused bytes per entry.................... 3.36 Average metadata per entry........................ 5.96 Average fanout.................................... 97.00 Maximum payload per entry......................... 175 Entries that use overflow......................... 0 0.0% Index pages used.................................. 9 Primary pages used................................ 871 Overflow pages used............................... 0 Total pages used.................................. 880 Unused bytes on index pages....................... 2192 23.8% Unused bytes on primary pages..................... 45876 5.1% Unused bytes on overflow pages.................... 0 Unused bytes on all pages......................... 48068 5.3% *** All indices *************************************************************** Percentage of total database...................... 52.6% Number of entries................................. 14280 Bytes of storage consumed......................... 1001472 Bytes of payload.................................. 809496 80.8% Bytes of metadata................................. 54564 5.4% Average payload per entry......................... 56.69 Average unused bytes per entry.................... 9.62 Average metadata per entry........................ 3.82 Average fanout.................................... 13.00 Maximum payload per entry......................... 58 Entries that use overflow......................... 0 0.0% Index pages used.................................. 70 Primary pages used................................ 908 Overflow pages used............................... 0 Total pages used.................................. 978 Unused bytes on index pages....................... 13111 18.3% Unused bytes on primary pages..................... 124301 13.4% Unused bytes on overflow pages.................... 0 Unused bytes on all pages......................... 137412 13.7% *** Table SQLITE_MASTER ******************************************************* Percentage of total database...................... 0.054% Number of entries................................. 6 Bytes of storage consumed......................... 1024 Bytes of payload.................................. 874 85.4% Bytes of metadata................................. 135 13.2% B-tree depth...................................... 1 Average payload per entry......................... 145.67 Average unused bytes per entry.................... 2.50 Average metadata per entry........................ 22.50 Maximum payload per entry......................... 175 Entries that use overflow......................... 0 0.0% Primary pages used................................ 1 Overflow pages used............................... 0 Total pages used.................................. 1 Unused bytes on primary pages..................... 15 1.5% Unused bytes on overflow pages.................... 0 Unused bytes on all pages......................... 15 1.5% *** Table TAB_25_22_D and all its indices ************************************* Percentage of total database...................... 13.7% Number of entries................................. 4080 Bytes of storage consumed......................... 260096 Bytes of payload.................................. 218152 83.9% Bytes of metadata................................. 19702 7.6% Average payload per entry......................... 53.47 Average unused bytes per entry.................... 5.45 Average metadata per entry........................ 4.83 Average fanout.................................... 25.00 Maximum payload per entry......................... 55 Entries that use overflow......................... 0 0.0% Index pages used.................................. 10 Primary pages used................................ 244 Overflow pages used............................... 0 Total pages used.................................. 254 Unused bytes on index pages....................... 1559 15.2% Unused bytes on primary pages..................... 20683 8.3% Unused bytes on overflow pages.................... 0 Unused bytes on all pages......................... 22242 8.6% *** Table TAB_25_22_D w/o any indices ***************************************** Percentage of total database...................... 6.5% Number of entries................................. 2040 Bytes of storage consumed......................... 123904 Bytes of payload.................................. 106080 85.6% Bytes of metadata................................. 11990 9.7% B-tree depth...................................... 2 Average payload per entry......................... 52.00 Average unused bytes per entry.................... 2.86 Average metadata per entry........................ 5.88 Average fanout.................................... 120.00 Non-sequential pages.............................. 41 34.2% Maximum payload per entry......................... 52 Entries that use overflow......................... 0 0.0% Index pages used.................................. 1 Primary pages used................................ 120 Overflow pages used............................... 0 Total pages used.................................. 121 Unused bytes on index pages....................... 67 6.5% Unused bytes on primary pages..................... 5767 4.7% Unused bytes on overflow pages.................... 0 Unused bytes on all pages......................... 5834 4.7% *** Index TAB_25_22_D_INDEX of table TAB_25_22_D ****************************** Percentage of total database...................... 7.2% Number of entries................................. 2040 Bytes of storage consumed......................... 136192 Bytes of payload.................................. 112072 82.3% Bytes of metadata................................. 7712 5.7% B-tree depth...................................... 3 Average payload per entry......................... 54.94 Average unused bytes per entry.................... 8.04 Average metadata per entry........................ 3.78 Average fanout.................................... 14.00 Non-sequential pages.............................. 39 29.5% Maximum payload per entry......................... 55 Entries that use overflow......................... 0 0.0% Index pages used.................................. 9 Primary pages used................................ 124 Overflow pages used............................... 0 Total pages used.................................. 133 Unused bytes on index pages....................... 1492 16.2% Unused bytes on primary pages..................... 14916 11.7% Unused bytes on overflow pages.................... 0 Unused bytes on all pages......................... 16408 12.0% *** Table TAB_25_22_M and all its indices ************************************* Percentage of total database...................... 13.0% Number of entries................................. 4080 Bytes of storage consumed......................... 246784 Bytes of payload.................................. 205912 83.4% Bytes of metadata................................. 19519 7.9% Average payload per entry......................... 50.47 Average unused bytes per entry.................... 5.23 Average metadata per entry........................ 4.78 Average fanout.................................... 24.00 Maximum payload per entry......................... 52 Entries that use overflow......................... 0 0.0% Index pages used.................................. 10 Primary pages used................................ 231 Overflow pages used............................... 0 Total pages used.................................. 241 Unused bytes on index pages....................... 2334 22.8% Unused bytes on primary pages..................... 19019 8.0% Unused bytes on overflow pages.................... 0 Unused bytes on all pages......................... 21353 8.7% *** Table TAB_25_22_M w/o any indices ***************************************** Percentage of total database...................... 6.1% Number of entries................................. 2040 Bytes of storage consumed......................... 116736 Bytes of payload.................................. 99960 85.6% Bytes of metadata................................. 11879 10.2% B-tree depth...................................... 2 Average payload per entry......................... 49.00 Average unused bytes per entry.................... 2.40 Average metadata per entry........................ 5.82 Average fanout.................................... 113.00 Non-sequential pages.............................. 34 30.1% Maximum payload per entry......................... 49 Entries that use overflow......................... 0 0.0% Index pages used.................................. 1 Primary pages used................................ 113 Overflow pages used............................... 0 Total pages used.................................. 114 Unused bytes on index pages....................... 122 11.9% Unused bytes on primary pages..................... 4775 4.1% Unused bytes on overflow pages.................... 0 Unused bytes on all pages......................... 4897 4.2% *** Index TAB_25_22_M_INDEX of table TAB_25_22_M ****************************** Percentage of total database...................... 6.8% Number of entries................................. 2040 Bytes of storage consumed......................... 130048 Bytes of payload.................................. 105952 81.5% Bytes of metadata................................. 7640 5.9% B-tree depth...................................... 3 Average payload per entry......................... 51.94 Average unused bytes per entry.................... 8.07 Average metadata per entry........................ 3.75 Average fanout.................................... 14.00 Non-sequential pages.............................. 31 24.6% Maximum payload per entry......................... 52 Entries that use overflow......................... 0 0.0% Index pages used.................................. 9 Primary pages used................................ 118 Overflow pages used............................... 0 Total pages used.................................. 127 Unused bytes on index pages....................... 2212 24.0% Unused bytes on primary pages..................... 14244 11.8% Unused bytes on overflow pages.................... 0 Unused bytes on all pages......................... 16456 12.7% *** Table TAB_25_22_R and all its indices ************************************* Percentage of total database...................... 73.3% Number of entries................................. 20400 Bytes of storage consumed......................... 1394688 Bytes of payload.................................. 1152472 82.6% Bytes of metadata................................. 100346 7.2% Average payload per entry......................... 56.49 Average unused bytes per entry.................... 6.95 Average metadata per entry........................ 4.92 Average fanout.................................... 23.00 Maximum payload per entry......................... 58 Entries that use overflow......................... 0 0.0% Index pages used.................................. 59 Primary pages used................................ 1303 Overflow pages used............................... 0 Total pages used.................................. 1362 Unused bytes on index pages....................... 11410 18.9% Unused bytes on primary pages..................... 130460 9.8% Unused bytes on overflow pages.................... 0 Unused bytes on all pages......................... 141870 10.2% *** Table TAB_25_22_R w/o any indices ***************************************** Percentage of total database...................... 34.7% Number of entries................................. 10200 Bytes of storage consumed......................... 659456 Bytes of payload.................................. 561000 85.1% Bytes of metadata................................. 61134 9.3% B-tree depth...................................... 3 Average payload per entry......................... 55.00 Average unused bytes per entry.................... 3.66 Average metadata per entry........................ 5.99 Average fanout.................................... 91.00 Non-sequential pages.............................. 383 59.6% Maximum payload per entry......................... 55 Entries that use overflow......................... 0 0.0% Index pages used.................................. 7 Primary pages used................................ 637 Overflow pages used............................... 0 Total pages used.................................. 644 Unused bytes on index pages....................... 2003 27.9% Unused bytes on primary pages..................... 35319 5.4% Unused bytes on overflow pages.................... 0 Unused bytes on all pages......................... 37322 5.7% *** Index TAB_25_22_R_INDEX of table TAB_25_22_R ****************************** Percentage of total database...................... 38.6% Number of entries................................. 10200 Bytes of storage consumed......................... 735232 Bytes of payload.................................. 591472 80.4% Bytes of metadata................................. 39212 5.3% B-tree depth...................................... 4 Average payload per entry......................... 57.99 Average unused bytes per entry.................... 10.25 Average metadata per entry........................ 3.84 Average fanout.................................... 13.00 Non-sequential pages.............................. 378 52.7% Maximum payload per entry......................... 58 Entries that use overflow......................... 0 0.0% Index pages used.................................. 52 Primary pages used................................ 666 Overflow pages used............................... 0 Total pages used.................................. 718 Unused bytes on index pages....................... 9407 17.7% Unused bytes on primary pages..................... 95141 14.0% Unused bytes on overflow pages.................... 0 Unused bytes on all pages......................... 104548 14.2% *** Definitions *************************************************************** Page size in bytes The number of bytes in a single page of the database file. Usually 1024. Number of pages in the whole file The number of 1024-byte pages that go into forming the complete database Pages that store data The number of pages that store data, either as primary B*Tree pages or as overflow pages. The number at the right is the data pages divided by the total number of pages in the file. Pages on the freelist The number of pages that are not currently in use but are reserved for future use. The percentage at the right is the number of freelist pages divided by the total number of pages in the file. Pages of auto-vacuum overhead The number of pages that store data used by the database to facilitate auto-vacuum. This is zero for databases that do not support auto-vacuum. Number of tables in the database The number of tables in the database, including the SQLITE_MASTER table used to store schema information. Number of indices The total number of indices in the database. Number of defined indices The number of indices created using an explicit CREATE INDEX statement. Number of implied indices The number of indices used to implement PRIMARY KEY or UNIQUE constraints on tables. Size of the file in bytes The total amount of disk space used by the entire database files. Bytes of user payload stored The total number of bytes of user payload stored in the database. The schema information in the SQLITE_MASTER table is not counted when computing this number. The percentage at the right shows the payload divided by the total file size. Percentage of total database The amount of the complete database file that is devoted to storing information described by this category. Number of entries The total number of B-Tree key/value pairs stored under this category. Bytes of storage consumed The total amount of disk space required to store all B-Tree entries under this category. The is the total number of pages used times the pages size. Bytes of payload The amount of payload stored under this category. Payload is the data part of table entries and the key part of index entries. The percentage at the right is the bytes of payload divided by the bytes of storage consumed. Bytes of metadata The amount of formatting and structural information stored in the table or index. Metadata includes the btree page header, the cell pointer array, the size field for each cell, the left child pointer or non-leaf cells, the overflow pointers for overflow cells, and the rowid value for rowid table cells. In other words, metadata is everything that is neither unused space nor content. The record header in the payload is counted as content, not metadata. Average payload per entry The average amount of payload on each entry. This is just the bytes of payload divided by the number of entries. Average unused bytes per entry The average amount of free space remaining on all pages under this category on a per-entry basis. This is the number of unused bytes on all pages divided by the number of entries. Non-sequential pages The number of pages in the table or index that are out of sequence. Many filesystems are optimized for sequential file access so a small number of non-sequential pages might result in faster queries, especially for larger database files that do not fit in the disk cache. Note that after running VACUUM, the root page of each table or index is at the beginning of the database file and all other pages are in a separate part of the database file, resulting in a single non- sequential page. Maximum payload per entry The largest payload size of any entry. Entries that use overflow The number of entries that user one or more overflow pages. Total pages used This is the number of pages used to hold all information in the current category. This is the sum of index, primary, and overflow pages. Index pages used This is the number of pages in a table B-tree that hold only key (rowid) information and no data. Primary pages used This is the number of B-tree pages that hold both key and data. Overflow pages used The total number of overflow pages used for this category. Unused bytes on index pages The total number of bytes of unused space on all index pages. The percentage at the right is the number of unused bytes divided by the total number of bytes on index pages. Unused bytes on primary pages The total number of bytes of unused space on all primary pages. The percentage at the right is the number of unused bytes divided by the total number of bytes on primary pages. Unused bytes on overflow pages The total number of bytes of unused space on all overflow pages. The percentage at the right is the number of unused bytes divided by the total number of bytes on overflow pages. Unused bytes on all pages The total number of bytes of unused space on all primary and overflow pages. The percentage at the right is the number of unused bytes divided by the total number of bytes. ******************************************************************************* The entire text of this report can be sourced into any SQL database engine for further analysis. All of the text above is an SQL comment. The data used to generate this report follows: */ BEGIN; CREATE TABLE space_used( name clob, -- Name of a table or index in the database file tblname clob, -- Name of associated table is_index boolean, -- TRUE if it is an index, false for a table is_without_rowid boolean, -- TRUE if WITHOUT ROWID table nentry int, -- Number of entries in the BTree leaf_entries int, -- Number of leaf entries depth int, -- Depth of the b-tree payload int, -- Total amount of data stored in this table or index ovfl_payload int, -- Total amount of data stored on overflow pages ovfl_cnt int, -- Number of entries that use overflow mx_payload int, -- Maximum payload size int_pages int, -- Number of interior pages used leaf_pages int, -- Number of leaf pages used ovfl_pages int, -- Number of overflow pages used int_unused int, -- Number of unused bytes on interior pages leaf_unused int, -- Number of unused bytes on primary pages ovfl_unused int, -- Number of unused bytes on overflow pages gap_cnt int, -- Number of gaps in the page layout compressed_size int -- Total bytes stored on disk ); INSERT INTO space_used VALUES('sqlite_master','sqlite_master',0,0,6,6,1,874,0,0,175,0,1,0,0,15,0,0,1024); INSERT INTO space_used VALUES('tab_25_22_R','tab_25_22_R',0,0,10836,10200,3,561000,0,0,55,7,637,0,2003,35319,0,383,659456); INSERT INTO space_used VALUES('tab_25_22_R_index','tab_25_22_R',1,0,10200,9535,4,591472,0,0,58,52,666,0,9407,95141,0,378,735232); INSERT INTO space_used VALUES('tab_25_22_D','tab_25_22_D',0,0,2159,2040,2,106080,0,0,52,1,120,0,67,5767,0,41,123904); INSERT INTO space_used VALUES('tab_25_22_D_index','tab_25_22_D',1,0,2040,1917,3,112072,0,0,55,9,124,0,1492,14916,0,39,136192); INSERT INTO space_used VALUES('tab_25_22_M','tab_25_22_M',0,0,2152,2040,2,99960,0,0,49,1,113,0,122,4775,0,34,116736); INSERT INTO space_used VALUES('tab_25_22_M_index','tab_25_22_M',1,0,2040,1923,3,105952,0,0,52,9,118,0,2212,14244,0,31,130048); COMMIT; -- Sent from: http://sqlite.1065341.n5.nabble.com/ _______________________________________________ sqlite-users mailing list sqlite-users@mailinglists.sqlite.org http://mailinglists.sqlite.org/cgi-bin/mailman/listinfo/sqlite-users