[feature] [tokudb] Issue: #63 Add compression ratio statistics per FT

TokuDB has status to expose the compression ratio:

    mysql> show global status like 'tokudb%COMPRESSION_RATIO';
    +---------------------------------------+----------+
    | Variable_name                         | Value    |
    +---------------------------------------+----------+
    | Tokudb_LEAF_NODE_COMPRESSION_RATIO    | 5.850260 |
    | Tokudb_NONLEAF_NODE_COMPRESSION_RATIO | 0.564453 |
    | Tokudb_OVERALL_NODE_COMPRESSION_RATIO | 4.528809 |
    +---------------------------------------+----------+

but this a global level ratio, and sometimes the users may be interested
in table level compression ratio.

This patch address this request by exposing FT level compresstion ratio
through information_schema.TokuDB_fractal_tree_info table. We add two
columns, leaf_compress_ratio and internal_compress_ratio, which indicate
the compression ratio of leaf nodes and internal nodes accordingly.

Limitations, the comppression ratio infos are maintained in memeory, and
will lost when server is restarted or ft get evicted from memory.
Maybe in future there infos will be persisted in ft file, when this feature
is accepted by PerconaFT upstream.

Author: AliSQL

mysql-test/suite/funcs_1/r/is_columns_is.result

@@ -350,7 +350,9 @@ def	information_schema	TokuDB_fractal_tree_info	bt_num_blocks_in_use	4	0	NO	bigi
 def	information_schema	TokuDB_fractal_tree_info	bt_size_allocated	5	0	NO	bigint	NULL	NULL	19	0	NULL	NULL	NULL	bigint(0)			select	
 def	information_schema	TokuDB_fractal_tree_info	bt_size_in_use	6	0	NO	bigint	NULL	NULL	19	0	NULL	NULL	NULL	bigint(0)			select	
 def	information_schema	TokuDB_fractal_tree_info	dictionary_name	1		NO	varchar	256	768	NULL	NULL	NULL	utf8	utf8_general_ci	varchar(256)			select	
+def	information_schema	TokuDB_fractal_tree_info	internal_compress_ratio	11	0	NO	double	NULL	NULL	12	NULL	NULL	NULL	NULL	double			select
 def	information_schema	TokuDB_fractal_tree_info	internal_file_name	2		NO	varchar	256	768	NULL	NULL	NULL	utf8	utf8_general_ci	varchar(256)			select	
+def	information_schema	TokuDB_fractal_tree_info	leaf_compress_ratio	10	0	NO	double	NULL	NULL	12	NULL	NULL	NULL	NULL	double			select
 def	information_schema	TokuDB_fractal_tree_info	table_dictionary_name	9		NO	varchar	256	768	NULL	NULL	NULL	utf8	utf8_general_ci	varchar(256)			select	
 def	information_schema	TokuDB_fractal_tree_info	table_name	8		NO	varchar	256	768	NULL	NULL	NULL	utf8	utf8_general_ci	varchar(256)			select	
 def	information_schema	TokuDB_fractal_tree_info	table_schema	7		NO	varchar	256	768	NULL	NULL	NULL	utf8	utf8_general_ci	varchar(256)			select	
@@ -449,6 +451,7 @@ ORDER BY CHARACTER_SET_NAME, COLLATION_NAME, COL_CML;
 COL_CML	DATA_TYPE	CHARACTER_SET_NAME	COLLATION_NAME
 NULL	bigint	NULL	NULL
 NULL	datetime	NULL	NULL
+NULL	double	NULL	NULL
 NULL	int	NULL	NULL
 NULL	tinyint	NULL	NULL
 --> CHAR(0) is allowed (see manual), and here both CHARACHTER_* values
@@ -819,6 +822,8 @@ NULL	information_schema	TokuDB_fractal_tree_info	bt_size_in_use	bigint	NULL	NULL
 3.0000	information_schema	TokuDB_fractal_tree_info	table_schema	varchar	256	768	utf8	utf8_general_ci	varchar(256)
 3.0000	information_schema	TokuDB_fractal_tree_info	table_name	varchar	256	768	utf8	utf8_general_ci	varchar(256)
 3.0000	information_schema	TokuDB_fractal_tree_info	table_dictionary_name	varchar	256	768	utf8	utf8_general_ci	varchar(256)
+NULL	information_schema	TokuDB_fractal_tree_info	leaf_compress_ratio	double	NULL	NULL	NULL	NULL	double
+NULL	information_schema	TokuDB_fractal_tree_info	internal_compress_ratio	double	NULL	NULL	NULL	NULL	double
 NULL	information_schema	TokuDB_locks	locks_trx_id	bigint	NULL	NULL	NULL	NULL	bigint(0)
 NULL	information_schema	TokuDB_locks	locks_mysql_thread_id	bigint	NULL	NULL	NULL	NULL	bigint(0)
 3.0000	information_schema	TokuDB_locks	locks_dname	varchar	256	768	utf8	utf8_general_ci	varchar(256)

mysql-test/suite/rds/r/tokudb_show_compress_ratio_per_ft.result

@@ -0,0 +1,26 @@
+create table t1(c1 int) engine = tokudb;
+create table t2(c1 int) engine = tokudb;
+insert into t1 values (rand());
+select count(*) from t1;
+count(*)
+3145726
+select count(*) from t2;
+count(*)
+6291410
+set global tokudb_checkpoint_on_flush_logs=ON;
+flush logs;
+set global tokudb_checkpoint_on_flush_logs=OFF;
+select leaf_compress_ratio > 3, internal_compress_ratio > 1 from information_schema.TokuDB_fractal_tree_info where table_dictionary_name = 'main';
+leaf_compress_ratio > 3	internal_compress_ratio > 1
+1	0
+1	0
+flush tables;
+set global tokudb_checkpoint_on_flush_logs=ON;
+flush logs;
+set global tokudb_checkpoint_on_flush_logs=OFF;
+select leaf_compress_ratio > 3, internal_compress_ratio > 1 from information_schema.TokuDB_fractal_tree_info where table_dictionary_name = 'main';
+leaf_compress_ratio > 3	internal_compress_ratio > 1
+1	0
+1	0
+drop table t1;
+drop table t2;

mysql-test/suite/rds/t/tokudb_show_compress_ratio_per_ft.test

@@ -0,0 +1,39 @@
+--source include/have_tokudb.inc
+--source include/have_innodb.inc
+
+create table t1(c1 int) engine = tokudb;
+create table t2(c1 int) engine = tokudb;
+
+insert into t1 values (rand());
+
+--let $i=0
+--let $count=20
+--disable_query_log
+while ($i<$count)
+{
+  inc $i;
+  eval insert into t1 values (rand());
+  eval insert into t1 select * from t1;
+  eval insert into t2 select * from t1;
+}
+--enable_query_log
+
+select count(*) from t1;
+select count(*) from t2;
+
+# trigger checkpoint
+set global tokudb_checkpoint_on_flush_logs=ON;
+flush logs;
+set global tokudb_checkpoint_on_flush_logs=OFF;
+
+select leaf_compress_ratio > 3, internal_compress_ratio > 1 from information_schema.TokuDB_fractal_tree_info where table_dictionary_name = 'main';
+flush tables;
+
+set global tokudb_checkpoint_on_flush_logs=ON;
+flush logs;
+set global tokudb_checkpoint_on_flush_logs=OFF;
+
+select leaf_compress_ratio > 3, internal_compress_ratio > 1 from information_schema.TokuDB_fractal_tree_info where table_dictionary_name = 'main';
+
+drop table t1;
+drop table t2;

storage/tokudb/ft-index/buildheader/make_tdb.cc

@@ -553,7 +553,7 @@ static void print_db_struct (void) {
 			 "int (*verify_with_progress)(DB *, int (*progress_callback)(void *progress_extra, float progress), void *progress_extra, int verbose, int keep_going)",
 			 "int (*update)(DB *, DB_TXN*, const DBT *key, const DBT *extra, uint32_t flags)",
 			 "int (*update_broadcast)(DB *, DB_TXN*, const DBT *extra, uint32_t flags)",
-			 "int (*get_fractal_tree_info64)(DB*,uint64_t*,uint64_t*,uint64_t*,uint64_t*)",
+			 "int (*get_fractal_tree_info64)(DB*,uint64_t*,uint64_t*,uint64_t*,uint64_t*,double*,double*)",
 			 "int (*iterate_fractal_tree_block_map)(DB*,int(*)(uint64_t,int64_t,int64_t,int64_t,int64_t,void*),void*)",
                          "const char *(*get_dname)(DB *db)",
                          "int (*get_last_key)(DB *db, YDB_CALLBACK_FUNCTION func, void* extra)",

storage/tokudb/ft-index/ft/ft-internal.h

@@ -264,6 +264,12 @@ struct ft {
     // - if the score is high enough, we optimistically attempt to insert directly into the rightmost leaf
     // - if our attempt fails because the key was not in range of the rightmost leaf, we reset the score back to 0
     uint32_t seqinsert_score;
+
+    // counters for uncompressed/compressed size of leaf/internal nodes
+    unsigned long long leaf_uncompressed_bytes;
+    unsigned long long leaf_compressed_bytes;
+    unsigned long long internal_uncompressed_bytes;
+    unsigned long long internal_compressed_bytes;
 };
 
 // Allocate a DB struct off the stack and only set its comparison
@@ -627,6 +633,7 @@ typedef struct {
 
 void toku_ft_status_update_pivot_fetch_reason(ftnode_fetch_extra *bfe);
 void toku_ft_status_update_flush_reason(FTNODE node, uint64_t uncompressed_bytes_flushed, uint64_t bytes_written, tokutime_t write_time, bool for_checkpoint);
+void toku_ft_status_update_flush_for_ft(FTNODE node, FT ft, uint64_t uncompressed_bytes_flushed, uint64_t bytes_written);
 void toku_ft_status_update_serialize_times(FTNODE node, tokutime_t serialize_time, tokutime_t compress_time);
 void toku_ft_status_update_deserialize_times(FTNODE node, tokutime_t deserialize_time, tokutime_t decompress_time);
 void toku_ft_status_note_msn_discard(void);

storage/tokudb/ft-index/ft/ft-ops.cc

@@ -799,6 +799,17 @@ void toku_ft_status_update_flush_reason(FTNODE node,
     }
 }
 
+void toku_ft_status_update_flush_for_ft(FTNODE node, FT ft, uint64_t uncompressed_bytes_flushed, uint64_t bytes_written)
+{
+    if (node->height == 0) {
+        ft->leaf_uncompressed_bytes += uncompressed_bytes_flushed;
+        ft->leaf_compressed_bytes += bytes_written;
+    } else {
+        ft->internal_uncompressed_bytes += uncompressed_bytes_flushed;
+        ft->internal_compressed_bytes += bytes_written;
+    }
+}
+
 void toku_ftnode_checkpoint_complete_callback(void *value_data) {
     FTNODE node = static_cast<FTNODE>(value_data);
     if (node->height > 0) {

storage/tokudb/ft-index/ft/ft-ops.h

@@ -292,6 +292,8 @@ struct ftinfo64 {
     uint64_t num_blocks_in_use;     // number of blocks in use by most recent checkpoint
     uint64_t size_allocated;        // sum of sizes of blocks in blocktable
     uint64_t size_in_use;           // sum of sizes of blocks in use by most recent checkpoint
+    double leaf_ratio;              // compress ratio for leaf node
+    double internal_ratio;          // compress ratio for internal node
 };
 
 void toku_ft_handle_get_fractal_tree_info64(FT_HANDLE, struct ftinfo64 *);

storage/tokudb/ft-index/ft/ft.cc

@@ -409,6 +409,11 @@ static void ft_init(FT ft, FT_OPTIONS options, CACHEFILE cf) {
                                 ft_note_unpin_by_checkpoint);
 
     ft->blocktable.verify_no_free_blocknums();
+
+    ft->leaf_uncompressed_bytes = 0;
+    ft->leaf_compressed_bytes = 0;
+    ft->internal_uncompressed_bytes = 0;
+    ft->internal_compressed_bytes = 0;
 }
 
 
@@ -492,6 +497,10 @@ int toku_read_ft_and_store_in_cachefile (FT_HANDLE ft_handle, CACHEFILE cf, LSN
     ft->cmp.create(ft_handle->options.compare_fun, &ft->cmp_descriptor, ft_handle->options.memcmp_magic);
     ft->update_fun = ft_handle->options.update_fun;
     ft->cf = cf;
+    ft->leaf_uncompressed_bytes = 0;
+    ft->leaf_compressed_bytes = 0;
+    ft->internal_uncompressed_bytes = 0;
+    ft->internal_compressed_bytes = 0;
     toku_cachefile_set_userdata(cf,
                                 reinterpret_cast<void *>(ft),
                                 ft_log_fassociate_during_checkpoint,
@@ -878,6 +887,10 @@ toku_ft_stat64 (FT ft, struct ftstat64_s *s) {
 
 void toku_ft_get_fractal_tree_info64(FT ft, struct ftinfo64 *info) {
     ft->blocktable.get_info64(info);
+    info->leaf_ratio = ft->leaf_compressed_bytes > 0 ?
+                           (double)ft->leaf_uncompressed_bytes / (double)ft->leaf_compressed_bytes : 0;
+    info->internal_ratio = ft->internal_compressed_bytes > 0 ?
+                           (double)ft->internal_uncompressed_bytes / (double)ft->internal_compressed_bytes : 0;
 }
 
 int toku_ft_iterate_fractal_tree_block_map(FT ft, int (*iter)(uint64_t,int64_t,int64_t,int64_t,int64_t,void*), void *iter_extra) {

storage/tokudb/ft-index/ft/serialize/ft_node-serialize.cc

@@ -878,6 +878,7 @@ toku_serialize_ftnode_to (int fd, BLOCKNUM blocknum, FTNODE node, FTNODE_DISK_DA
 
     tokutime_t io_time = t1 - t0;
     toku_ft_status_update_flush_reason(node, n_uncompressed_bytes, n_to_write, io_time, for_checkpoint);
+    toku_ft_status_update_flush_for_ft(node, ft, n_uncompressed_bytes, n_to_write);
 
     toku_free(compressed_buf);
     node->dirty = 0;  // See #1957.   Must set the node to be clean after serializing it so that it doesn't get written again on the next checkpoint or eviction.

storage/tokudb/ft-index/src/ydb_db.cc

@@ -733,14 +733,16 @@ toku_db_set_memcmp_magic(DB *db, uint8_t magic) {
 }
 
 static int
-toku_db_get_fractal_tree_info64(DB *db, uint64_t *num_blocks_allocated, uint64_t *num_blocks_in_use, uint64_t *size_allocated, uint64_t *size_in_use) {
+toku_db_get_fractal_tree_info64(DB *db, uint64_t *num_blocks_allocated, uint64_t *num_blocks_in_use, uint64_t *size_allocated, uint64_t *size_in_use, double *leaf_compress_ratio, double *internal_compress_ratio) {
     HANDLE_PANICKED_DB(db);
     struct ftinfo64 ftinfo;
     toku_ft_handle_get_fractal_tree_info64(db->i->ft_handle, &ftinfo);
     *num_blocks_allocated = ftinfo.num_blocks_allocated;
     *num_blocks_in_use = ftinfo.num_blocks_in_use;
     *size_allocated = ftinfo.size_allocated;
     *size_in_use = ftinfo.size_in_use;
+    *leaf_compress_ratio = ftinfo.leaf_ratio;
+    *internal_compress_ratio = ftinfo.internal_ratio;
     return 0;
 }