From 66137009ba3d6fb9afbbe6c64d28804b8e271f93 Mon Sep 17 00:00:00 2001 From: Feng Liu Date: Tue, 24 Apr 2018 13:25:10 -0600 Subject: [PATCH 1/3] add hash_func for file token and string token --- include/module/pssm/module.h | 4 +++- include/runtime/api.h | 9 +++++++++ include/sched/rscope.h | 9 +++++++++ lib/pstd/include/pstd/scope.h | 9 +++++++++ lib/pstd/scope.c | 11 +++++++++++ lib/pstd/types/file.c | 22 +++++++++++++++++++++- lib/pstd/types/string.c | 23 ++++++++++++++++++++++- src/module/pssm/module.c | 23 +++++++++++++++++++++++ src/sched/rscope.c | 15 +++++++++++++++ 9 files changed, 122 insertions(+), 3 deletions(-) diff --git a/include/module/pssm/module.h b/include/module/pssm/module.h index 80451e43..9c294f97 100644 --- a/include/module/pssm/module.h +++ b/include/module/pssm/module.h @@ -1,5 +1,6 @@ /** * Copyright (C) 2017-2018, Hao Hou + * Copyright (C) 2018, Feng Liu **/ /** * @brief the module definition for the Plumber Standard Service Module @@ -35,7 +36,8 @@ enum { MODULE_PSSM_MODULE_OPCODE_SCOPE_STREAM_CLOSE, /*!< Close a RLS stream */ MODULE_PSSM_MODULE_OPCODE_SCOPE_STREAM_EOF, /*!< Check if the stream has reached the end */ MODULE_PSSM_MODULE_OPCODE_SCOPE_STREAM_READ, /*!< Read the stream */ - MODULE_PSSM_MODULE_OPCODE_SCOPE_STREAM_READY_EVENT /*!< Query the ready event */ + MODULE_PSSM_MODULE_OPCODE_SCOPE_STREAM_READY_EVENT, /*!< Query the ready event */ + MODULE_PSSM_MODULE_OPCODE_SCOPE_GET_HASH /*!< Get Hash */ }; #endif /* __PLUMBER_MODULE_PSSM_MODULE_H__ */ diff --git a/include/runtime/api.h b/include/runtime/api.h index 890ee0bd..c66c7c4f 100644 --- a/include/runtime/api.h +++ b/include/runtime/api.h @@ -1,5 +1,6 @@ /** * Copyright (C) 2017-2018, Hao Hou + * Copyright (C) 2018, Feng Liu **/ /** @@ -455,6 +456,14 @@ typedef struct { **/ int (*event_func)(void* __restrict handle, runtime_api_scope_ready_event_t* event_buf); + /** + * @brief Generate the hash code of the scope entity + * @param ptr the RLS pointer to hash + * @param out the generated hash code for ptr + * @return status code + **/ + int (*hash_func)(const void* ptr, uint64_t out[2]); + /** * @brief close a used stream handle * @param handle the handle to close diff --git a/include/sched/rscope.h b/include/sched/rscope.h index 329f99ac..a27c9aab 100644 --- a/include/sched/rscope.h +++ b/include/sched/rscope.h @@ -1,5 +1,6 @@ /** * Copyright (C) 2017-2018, Hao Hou + * Copyright (C) 2018, Feng Liu **/ /** * @brief The request local scope @@ -137,4 +138,12 @@ size_t sched_rscope_stream_read(sched_rscope_stream_t* stream, void* buffer, siz * @return The number of events has been returned, or error code **/ int sched_rscope_stream_get_event(sched_rscope_stream_t* stream, runtime_api_scope_ready_event_t* buf); + +/** + * @brief get the hash code of a token + * @param token the RLS token + * @param out the buffer for hash code + * @return The number of hash code has been returned or error code + **/ +int sched_rscope_get_hash(runtime_api_scope_token_t token, uint64_t out[2]); #endif /* __SCHED_RSCOPE_H__ */ diff --git a/lib/pstd/include/pstd/scope.h b/lib/pstd/include/pstd/scope.h index af112091..86cf015e 100644 --- a/lib/pstd/include/pstd/scope.h +++ b/lib/pstd/include/pstd/scope.h @@ -1,5 +1,6 @@ /** * Copyright (C) 2017-2018, Hao Hou + * Copyright (C) 2018, Feng Liu **/ /** * @brief The request local scope @@ -89,4 +90,12 @@ int pstd_scope_stream_close(pstd_scope_stream_t* stream); **/ int pstd_scope_stream_ready_event(pstd_scope_stream_t* stream, scope_ready_event_t* buf); +/** + * @brief Calculate the hash code of a token + * @param token The RLS token + * @param out The buffer for hash code + * @return number of hash code has been generated + **/ +int pstd_scope_get_hash(scope_token_t token, uint64_t out[2]); + #endif /* __PSTD_SCOPE_H__ */ diff --git a/lib/pstd/scope.c b/lib/pstd/scope.c index bfc3b3c4..83d226ba 100644 --- a/lib/pstd/scope.c +++ b/lib/pstd/scope.c @@ -1,5 +1,6 @@ /** * Copyright (C) 2017-2018, Hao Hou + * Copyright (C) 2018, Feng Liu **/ #include #include @@ -116,3 +117,13 @@ int pstd_scope_stream_ready_event(pstd_scope_stream_t* stream, scope_ready_event return ret; } + +int pstd_scope_get_hash(scope_token_t token, uint64_t out[2]) +{ + _ENSURE_PIPE(scope_get_hash, ERROR_CODE(int)); + int ret = 0; + + if(ERROR_CODE(int) == pipe_cntl(scope_get_hash, PIPE_CNTL_INVOKE, token, out, &ret)) + return ERROR_CODE(int); + return ret; +} diff --git a/lib/pstd/types/file.c b/lib/pstd/types/file.c index 7ac0022f..21153a54 100644 --- a/lib/pstd/types/file.c +++ b/lib/pstd/types/file.c @@ -1,5 +1,6 @@ /** * Copyright (C) 2017, Hao Hou + * Copyright (C) 2018, Feng Liu **/ #include #include @@ -12,6 +13,7 @@ #include #include +#include /** * @brief the actual data structure for a file reference @@ -358,6 +360,23 @@ static inline size_t _read(void* __restrict stream_mem, void* __restrict buf, si #endif } +/** + * @brief the callback for generating hash code for the file + * @param mem the RLS object + * @param out the output array + * @return status code + **/ +static inline int _hash(const void* mem, uint64_t out[2]) +{ + const pstd_file_t* file = (const pstd_file_t*)mem; + size_t len = strlen(file->filename); + /* use different seed for different type of RLS object */ + const uint32_t seed = 93578; + murmurhash3_128(file->filename, len, seed, out); + + return 0; +} + scope_token_t pstd_file_commit(pstd_file_t* file) { if(NULL == file || file->committed) @@ -370,7 +389,8 @@ scope_token_t pstd_file_commit(pstd_file_t* file) .open_func = _open, .close_func = _close, .eos_func = _eos, - .read_func = _read + .read_func = _read, + .hash_func = _hash }; scope_token_t ret = pstd_scope_add(&ent); diff --git a/lib/pstd/types/string.c b/lib/pstd/types/string.c index d2cf3486..9baf7d53 100644 --- a/lib/pstd/types/string.c +++ b/lib/pstd/types/string.c @@ -1,5 +1,6 @@ /** * Copyright (C) 2017, Hao Hou + * Copyright (C) 2018, Feng Liu **/ #include #include @@ -12,6 +13,7 @@ #include #include +#include /** * @brief the actuall data structure for the PSTD string type @@ -305,6 +307,24 @@ static inline size_t _read(void* __restrict stream_mem, void* __restrict buf, si return bytes_can_read; } +/** + * @brief Calculate the hash code for the string + * @param mem the RLS object + * @param out the output array + * @return status code + **/ +static inline int _hash(const void* mem, uint64_t out[2]) +{ + if(NULL == mem) + ERROR_RETURN_LOG(int, "Invalid arguments"); + + const pstd_string_t* str = (const pstd_string_t*)mem; + /* use different seed for different type of RLS object */ + const uint32_t seed = 222851856; + murmurhash3_128(str->buffer, str->length, seed, out); + return 0; +} + scope_token_t pstd_string_commit(pstd_string_t* str) { if(NULL == str) @@ -323,7 +343,8 @@ scope_token_t pstd_string_commit(pstd_string_t* str) .open_func = _open, .close_func = _close, .read_func = _read, - .eos_func = _eos + .eos_func = _eos, + .hash_func = _hash }; return pstd_scope_add(&ent); diff --git a/src/module/pssm/module.c b/src/module/pssm/module.c index 0794933e..b86a757f 100644 --- a/src/module/pssm/module.c +++ b/src/module/pssm/module.c @@ -1,5 +1,6 @@ /** * Copyright (C) 2017-2018, Hao Hou + * Copyright (C) 2018, Feng Liu **/ /** * @todo split this file to smaller files @@ -563,6 +564,16 @@ static inline int _rscope_stream_ready_event(sched_rscope_stream_t* stream, runt return sched_rscope_stream_get_event(stream, buf); } +static inline int _rscope_get_hash(uint32_t token, uint64_t out[2]) +{ + if(ERROR_CODE(runtime_api_scope_token_t) == token) + ERROR_RETURN_LOG(int, "Invalid arguments"); + + /* Be careful */ + runtime_api_scope_token_t internal_token = token - 1; + return sched_rscope_get_hash(internal_token, out); +} + static int _invoke(void* __restrict ctx, uint32_t opcode, va_list args) { (void)ctx; @@ -690,6 +701,17 @@ static int _invoke(void* __restrict ctx, uint32_t opcode, va_list args) return ERROR_CODE(int); return 0; } + case MODULE_PSSM_MODULE_OPCODE_SCOPE_GET_HASH: + { + uint32_t token = va_arg(args, uint32_t); + uint64_t* out = va_arg(args, uint64_t*); + int* ret = va_arg(args, int*); + if(NULL == ret || NULL == out) + ERROR_RETURN_LOG(int, "Invalid arguments"); + if(ERROR_CODE(int) == (*ret = _rscope_get_hash(token, out))) + return ERROR_CODE(int); + return 0; + } default: ERROR_RETURN_LOG(int, "Invalid opcode 0x%x", opcode); } @@ -716,6 +738,7 @@ static uint32_t _get_opcode(void* __restrict ctx, const char* name) if(strcmp(name, "scope_stream_eof") == 0) return MODULE_PSSM_MODULE_OPCODE_SCOPE_STREAM_EOF; if(strcmp(name, "scope_stream_read") == 0) return MODULE_PSSM_MODULE_OPCODE_SCOPE_STREAM_READ; if(strcmp(name, "scope_stream_ready_event") == 0) return MODULE_PSSM_MODULE_OPCODE_SCOPE_STREAM_READY_EVENT; + if(strcmp(name, "scope_get_hash") == 0) return MODULE_PSSM_MODULE_OPCODE_SCOPE_GET_HASH; ERROR_RETURN_LOG(uint32_t, "Invalid method name %s", name); } diff --git a/src/sched/rscope.c b/src/sched/rscope.c index cb1dee5b..aefd1bbc 100644 --- a/src/sched/rscope.c +++ b/src/sched/rscope.c @@ -1,5 +1,6 @@ /** * Copyright (C) 2017-2018, Hao Hou + * Copyright (C) 2018, Feng Liu **/ #include #include @@ -456,3 +457,17 @@ int sched_rscope_stream_get_event(sched_rscope_stream_t* stream, runtime_api_sco return ent->entity.event_func(stream->handle, buf); } +int sched_rscope_get_hash(runtime_api_scope_token_t token, uint64_t out[2]) +{ + if(_NULL_ENTRY == token || token >= _entry_table.capacity || _entry_table.data[token].data == NULL) + ERROR_RETURN_LOG(int, "Invalid arguments"); + + const _entry_t* target = _entry_table.data + token; + + if(target->data->entity.hash_func == NULL) + return 0; + + if(ERROR_CODE(int) == target->data->entity.hash_func(target->data->entity.data, out)) + return ERROR_CODE(int); + return 1; +} From bba27c6e4daf6f8070b9f5f8a9da0ca9e2231d2d Mon Sep 17 00:00:00 2001 From: Feng Liu Date: Thu, 26 Apr 2018 14:06:53 -0600 Subject: [PATCH 2/3] Change the return value of scope_entity's hash_func --- include/runtime/api.h | 2 +- lib/pstd/types/file.c | 6 +++--- lib/pstd/types/string.c | 4 ++-- src/sched/rscope.c | 4 +--- 4 files changed, 7 insertions(+), 9 deletions(-) diff --git a/include/runtime/api.h b/include/runtime/api.h index c66c7c4f..5fd729ba 100644 --- a/include/runtime/api.h +++ b/include/runtime/api.h @@ -460,7 +460,7 @@ typedef struct { * @brief Generate the hash code of the scope entity * @param ptr the RLS pointer to hash * @param out the generated hash code for ptr - * @return status code + * @return The number of hash code **/ int (*hash_func)(const void* ptr, uint64_t out[2]); diff --git a/lib/pstd/types/file.c b/lib/pstd/types/file.c index 21153a54..ab264af3 100644 --- a/lib/pstd/types/file.c +++ b/lib/pstd/types/file.c @@ -364,7 +364,7 @@ static inline size_t _read(void* __restrict stream_mem, void* __restrict buf, si * @brief the callback for generating hash code for the file * @param mem the RLS object * @param out the output array - * @return status code + * @return The number of hash code **/ static inline int _hash(const void* mem, uint64_t out[2]) { @@ -374,7 +374,7 @@ static inline int _hash(const void* mem, uint64_t out[2]) const uint32_t seed = 93578; murmurhash3_128(file->filename, len, seed, out); - return 0; + return 1; } scope_token_t pstd_file_commit(pstd_file_t* file) @@ -390,7 +390,7 @@ scope_token_t pstd_file_commit(pstd_file_t* file) .close_func = _close, .eos_func = _eos, .read_func = _read, - .hash_func = _hash + .hash_func = _hash }; scope_token_t ret = pstd_scope_add(&ent); diff --git a/lib/pstd/types/string.c b/lib/pstd/types/string.c index 9baf7d53..e20da3fe 100644 --- a/lib/pstd/types/string.c +++ b/lib/pstd/types/string.c @@ -311,7 +311,7 @@ static inline size_t _read(void* __restrict stream_mem, void* __restrict buf, si * @brief Calculate the hash code for the string * @param mem the RLS object * @param out the output array - * @return status code + * @return The number of hash code **/ static inline int _hash(const void* mem, uint64_t out[2]) { @@ -322,7 +322,7 @@ static inline int _hash(const void* mem, uint64_t out[2]) /* use different seed for different type of RLS object */ const uint32_t seed = 222851856; murmurhash3_128(str->buffer, str->length, seed, out); - return 0; + return 1; } scope_token_t pstd_string_commit(pstd_string_t* str) diff --git a/src/sched/rscope.c b/src/sched/rscope.c index aefd1bbc..91025c7e 100644 --- a/src/sched/rscope.c +++ b/src/sched/rscope.c @@ -467,7 +467,5 @@ int sched_rscope_get_hash(runtime_api_scope_token_t token, uint64_t out[2]) if(target->data->entity.hash_func == NULL) return 0; - if(ERROR_CODE(int) == target->data->entity.hash_func(target->data->entity.data, out)) - return ERROR_CODE(int); - return 1; + return target->data->entity.hash_func(target->data->entity.data, out); } From 3a7e12308e977531d7dec447ff0ccda69b4d933a Mon Sep 17 00:00:00 2001 From: Feng Liu Date: Thu, 26 Apr 2018 14:40:34 -0600 Subject: [PATCH 3/3] Add hash_func for trans token Note: The magic number for hashing file, string and zlib_token should be generated randomly before compiling in the future --- lib/pstd/include/pstd/types/trans.h | 1 + lib/pstd/types/trans.c | 19 ++++++++++++++++++- servlets/network/http/render/zlib_token.c | 2 ++ 3 files changed, 21 insertions(+), 1 deletion(-) diff --git a/lib/pstd/include/pstd/types/trans.h b/lib/pstd/include/pstd/types/trans.h index 5e361e8b..0468bfe4 100644 --- a/lib/pstd/include/pstd/types/trans.h +++ b/lib/pstd/include/pstd/types/trans.h @@ -33,6 +33,7 @@ typedef struct _pstd_trans_inst_t pstd_trans_inst_t; **/ typedef struct { void* data; /*!< The additional data for the processor callbacks */ + uint32_t hash; /*!< A magic number for the type of transformer */ /** * @brief Initialize the stream processor * @param data The addtional data to pass in diff --git a/lib/pstd/types/trans.c b/lib/pstd/types/trans.c index a3cae03b..2b49a746 100644 --- a/lib/pstd/types/trans.c +++ b/lib/pstd/types/trans.c @@ -1,5 +1,6 @@ /** * Copyright (C) 2018, Hao Hou + * Copyright (C) 2018, Feng Liu **/ #include #include @@ -13,6 +14,7 @@ #include #include #include +#include struct _pstd_trans_t { uint32_t commited:1; /*!< Indicates if the token is committed */ @@ -284,6 +286,20 @@ static int _event(void* __restrict trans_mem, runtime_api_scope_ready_event_t* e return 0; } +static int _hash(const void* ptr, uint64_t out[2]) +{ + const pstd_trans_t* trans = (const pstd_trans_t*)ptr; + + uint64_t tk_hash[2]; + int tk_res = pstd_scope_get_hash(trans->src_token, tk_hash); + if(ERROR_CODE(int) == tk_res || 0 == tk_res) + return tk_res; + + murmurhash3_128(tk_hash, sizeof(tk_hash), trans->ctx.hash, out); + + return 1; +} + scope_token_t pstd_trans_commit(pstd_trans_t* trans) { if(NULL == trans || trans->commited) @@ -297,7 +313,8 @@ scope_token_t pstd_trans_commit(pstd_trans_t* trans) .close_func = _close, .eos_func = _eos, .read_func = _read, - .event_func = _event + .event_func = _event, + .hash_func = _hash }; return pstd_scope_add(&ent); diff --git a/servlets/network/http/render/zlib_token.c b/servlets/network/http/render/zlib_token.c index de2fc7ea..23d1df3b 100644 --- a/servlets/network/http/render/zlib_token.c +++ b/servlets/network/http/render/zlib_token.c @@ -1,5 +1,6 @@ /** * Copyright (C) 2018, Hao Hou + * Copyright (C) 2018, Feng Liu **/ #if HAS_ZLIB #include @@ -158,6 +159,7 @@ scope_token_t zlib_token_encode(scope_token_t data_token, zlib_token_format_t fo pstd_trans_desc_t desc = { .data = zs, + .hash = 211840590, .init_func = _init, .feed_func = _feed, .fetch_func = _fetch,