Cache Token Support

include/module/pssm/module.h

@@ -1,5 +1,6 @@
 /**
  * Copyright (C) 2017-2018, Hao Hou
+ * Copyright (C) 2018, Feng Liu
  **/
 /**
  * @brief the module definition for the Plumber Standard Service Module
@@ -35,7 +36,8 @@ enum {
 	MODULE_PSSM_MODULE_OPCODE_SCOPE_STREAM_CLOSE,  /*!< Close a RLS stream */
 	MODULE_PSSM_MODULE_OPCODE_SCOPE_STREAM_EOF,    /*!< Check if the stream has reached the end */
 	MODULE_PSSM_MODULE_OPCODE_SCOPE_STREAM_READ,   /*!< Read the stream */
-	MODULE_PSSM_MODULE_OPCODE_SCOPE_STREAM_READY_EVENT  /*!< Query the ready event */
+	MODULE_PSSM_MODULE_OPCODE_SCOPE_STREAM_READY_EVENT,  /*!< Query the ready event */
+	MODULE_PSSM_MODULE_OPCODE_SCOPE_GET_HASH       /*!< Get Hash */
 };
 
 #endif /* __PLUMBER_MODULE_PSSM_MODULE_H__ */

include/runtime/api.h

@@ -1,5 +1,6 @@
 /**
  * Copyright (C) 2017-2018, Hao Hou
+ * Copyright (C) 2018, Feng Liu
  **/
 
 /**
@@ -455,6 +456,14 @@ typedef struct {
 	 **/
 	int (*event_func)(void* __restrict handle, runtime_api_scope_ready_event_t* event_buf);
 
+	/**
+	 * @brief Generate the hash code of the scope entity
+	 * @param ptr the RLS pointer to hash
+	 * @param out the generated hash code for ptr
+	 * @return The number of hash code
+	 **/
+	int (*hash_func)(const void* ptr, uint64_t out[2]);
+
 	/**
 	 * @brief close a used stream handle
 	 * @param handle the handle to close

include/sched/rscope.h

@@ -1,5 +1,6 @@
 /**
  * Copyright (C) 2017-2018, Hao Hou
+ * Copyright (C) 2018, Feng Liu
  **/
 /**
  * @brief The request local scope
@@ -137,4 +138,12 @@ size_t sched_rscope_stream_read(sched_rscope_stream_t* stream, void* buffer, siz
  * @return The number of events has been returned, or error code
  **/
 int sched_rscope_stream_get_event(sched_rscope_stream_t* stream, runtime_api_scope_ready_event_t* buf);
+
+/**
+ * @brief get the hash code of a token
+ * @param token the RLS token
+ * @param out the buffer for hash code
+ * @return The number of hash code has been returned or error code
+ **/
+int sched_rscope_get_hash(runtime_api_scope_token_t token, uint64_t out[2]);
 #endif /* __SCHED_RSCOPE_H__ */

lib/pstd/include/pstd/scope.h

@@ -1,5 +1,6 @@
 /**
  * Copyright (C) 2017-2018, Hao Hou
+ * Copyright (C) 2018, Feng Liu
  **/
 /**
  * @brief The request local scope
@@ -89,4 +90,12 @@ int pstd_scope_stream_close(pstd_scope_stream_t* stream);
  **/
 int pstd_scope_stream_ready_event(pstd_scope_stream_t* stream, scope_ready_event_t* buf);
 
+/**
+ * @brief Calculate the hash code of a token
+ * @param token The RLS token
+ * @param out The buffer for hash code
+ * @return number of hash code has been generated
+ **/
+int pstd_scope_get_hash(scope_token_t token, uint64_t out[2]);
+
 #endif /* __PSTD_SCOPE_H__ */

lib/pstd/include/pstd/types/trans.h

@@ -33,6 +33,7 @@ typedef struct _pstd_trans_inst_t pstd_trans_inst_t;
  **/
 typedef struct {
 	void*     data;         /*!< The additional data for the processor callbacks */
+	uint32_t  hash;         /*!< A magic number for the type of transformer */
 	/**
 	 * @brief Initialize the stream processor
 	 * @param data The addtional data to pass in

lib/pstd/scope.c

@@ -1,5 +1,6 @@
 /**
  * Copyright (C) 2017-2018, Hao Hou
+ * Copyright (C) 2018, Feng Liu
  **/
 #include <stdlib.h>
 #include <errno.h>
@@ -116,3 +117,13 @@ int pstd_scope_stream_ready_event(pstd_scope_stream_t* stream, scope_ready_event
 
 	return ret;
 }
+
+int pstd_scope_get_hash(scope_token_t token, uint64_t out[2])
+{
+	_ENSURE_PIPE(scope_get_hash, ERROR_CODE(int));
+	int ret = 0;
+
+	if(ERROR_CODE(int) == pipe_cntl(scope_get_hash, PIPE_CNTL_INVOKE, token, out, &ret))
+		return ERROR_CODE(int);
+	return ret;
+}

lib/pstd/types/file.c

@@ -1,5 +1,6 @@
 /**
  * Copyright (C) 2017, Hao Hou
+ * Copyright (C) 2018, Feng Liu
  **/
 #include <stdio.h>
 #include <string.h>
@@ -12,6 +13,7 @@
 
 #include <pstd.h>
 #include <pstd/types/file.h>
+#include <utils/hash/murmurhash3.h>
 
 /**
  * @brief the actual data structure for a file reference
@@ -358,6 +360,23 @@ static inline size_t _read(void* __restrict stream_mem, void* __restrict buf, si
 #endif
 }
 
+/**
+ * @brief the callback for generating hash code for the file
+ * @param mem the RLS object
+ * @param out the output array
+ * @return The number of hash code
+ **/
+static inline int _hash(const void* mem, uint64_t out[2])
+{
+	const pstd_file_t* file = (const pstd_file_t*)mem;
+	size_t len = strlen(file->filename);
+    /* use different seed for different type of RLS object */
+    const uint32_t seed = 93578;
+    murmurhash3_128(file->filename, len, seed, out);
+
+    return 1;
+}
+
 scope_token_t pstd_file_commit(pstd_file_t* file)
 {
 	if(NULL == file || file->committed)
@@ -370,7 +389,8 @@ scope_token_t pstd_file_commit(pstd_file_t* file)
 		.open_func = _open,
 		.close_func = _close,
 		.eos_func = _eos,
-		.read_func = _read
+		.read_func = _read,
+		.hash_func = _hash
 	};
 
 	scope_token_t ret = pstd_scope_add(&ent);

lib/pstd/types/string.c

@@ -1,5 +1,6 @@
 /**
  * Copyright (C) 2017, Hao Hou
+ * Copyright (C) 2018, Feng Liu
  **/
 #include <stdlib.h>
 #include <string.h>
@@ -12,6 +13,7 @@
 
 #include <pstd.h>
 #include <pstd/types/string.h>
+#include <utils/hash/murmurhash3.h>
 
 /**
  * @brief the actuall data structure for the PSTD string type
@@ -305,6 +307,24 @@ static inline size_t _read(void* __restrict stream_mem, void* __restrict buf, si
 	return bytes_can_read;
 }
 
+/**
+ * @brief Calculate the hash code for the string
+ * @param mem the RLS object
+ * @param out the output array
+ * @return The number of hash code
+ **/
+static inline int _hash(const void* mem, uint64_t out[2])
+{
+	if(NULL == mem)
+	    ERROR_RETURN_LOG(int, "Invalid arguments");
+
+	const pstd_string_t* str = (const pstd_string_t*)mem;
+    /* use different seed for different type of RLS object */
+	const uint32_t seed = 222851856;
+	murmurhash3_128(str->buffer, str->length, seed, out);
+	return 1;
+}
+
 scope_token_t pstd_string_commit(pstd_string_t* str)
 {
 	if(NULL == str)
@@ -323,7 +343,8 @@ scope_token_t pstd_string_commit(pstd_string_t* str)
 		.open_func = _open,
 		.close_func = _close,
 		.read_func = _read,
-		.eos_func = _eos
+		.eos_func = _eos,
+		.hash_func = _hash
 	};
 
 	return pstd_scope_add(&ent);

lib/pstd/types/trans.c

@@ -1,5 +1,6 @@
 /**
  * Copyright (C) 2018, Hao Hou
+ * Copyright (C) 2018, Feng Liu
  **/
 #include <stdlib.h>
 #include <stdint.h>
@@ -13,6 +14,7 @@
 #include <pstd/types/trans.h>
 #include <pstd/mempool.h>
 #include <pstd/scope.h>
+#include <utils/hash/murmurhash3.h>
 
 struct _pstd_trans_t {
 	uint32_t             commited:1;      /*!< Indicates if the token is committed */
@@ -284,6 +286,20 @@ static int _event(void* __restrict trans_mem, runtime_api_scope_ready_event_t* e
 	return 0;
 }
 
+static int _hash(const void* ptr, uint64_t out[2])
+{
+	const pstd_trans_t* trans = (const pstd_trans_t*)ptr;
+
+	uint64_t tk_hash[2];
+	int tk_res = pstd_scope_get_hash(trans->src_token, tk_hash);
+	if(ERROR_CODE(int) == tk_res || 0 == tk_res)
+		return tk_res;
+
+	murmurhash3_128(tk_hash, sizeof(tk_hash), trans->ctx.hash, out);
+
+	return 1;
+}
+
 scope_token_t pstd_trans_commit(pstd_trans_t* trans)
 {
 	if(NULL == trans || trans->commited)
@@ -297,7 +313,8 @@ scope_token_t pstd_trans_commit(pstd_trans_t* trans)
 		.close_func = _close,
 		.eos_func = _eos,
 		.read_func = _read,
-		.event_func = _event
+		.event_func = _event,
+		.hash_func = _hash
 	};
 
 	return pstd_scope_add(&ent);

servlets/network/http/render/zlib_token.c

@@ -1,5 +1,6 @@
 /**
  * Copyright (C) 2018, Hao Hou
+ * Copyright (C) 2018, Feng Liu
  **/
 #if HAS_ZLIB
 #include <stdint.h>
@@ -158,6 +159,7 @@ scope_token_t zlib_token_encode(scope_token_t data_token, zlib_token_format_t fo
 
 	pstd_trans_desc_t desc = {
 		.data = zs,
+		.hash = 211840590,
 		.init_func = _init,
 		.feed_func = _feed,
 		.fetch_func = _fetch,

src/module/pssm/module.c

@@ -1,5 +1,6 @@
 /**
  * Copyright (C) 2017-2018, Hao Hou
+ * Copyright (C) 2018, Feng Liu
  **/
 /**
  * @todo split this file to smaller files
@@ -563,6 +564,16 @@ static inline int _rscope_stream_ready_event(sched_rscope_stream_t* stream, runt
 	return sched_rscope_stream_get_event(stream, buf);
 }
 
+static inline int _rscope_get_hash(uint32_t token, uint64_t out[2])
+{
+	if(ERROR_CODE(runtime_api_scope_token_t) == token)
+	    ERROR_RETURN_LOG(int, "Invalid arguments");
+
+	/* Be careful */
+	runtime_api_scope_token_t internal_token = token - 1;
+	return sched_rscope_get_hash(internal_token, out);
+}
+
 static int _invoke(void* __restrict ctx, uint32_t opcode, va_list args)
 {
 	(void)ctx;
@@ -690,6 +701,17 @@ static int _invoke(void* __restrict ctx, uint32_t opcode, va_list args)
 			    return ERROR_CODE(int);
 			return 0;
 		}
+		case MODULE_PSSM_MODULE_OPCODE_SCOPE_GET_HASH:
+		{
+			uint32_t token = va_arg(args, uint32_t);
+			uint64_t* out = va_arg(args, uint64_t*);
+			int* ret = va_arg(args, int*);
+			if(NULL == ret || NULL == out)
+				ERROR_RETURN_LOG(int, "Invalid arguments");
+			if(ERROR_CODE(int) == (*ret = _rscope_get_hash(token, out)))
+				return ERROR_CODE(int);
+			return 0;
+		}
 		default:
 		    ERROR_RETURN_LOG(int, "Invalid opcode 0x%x", opcode);
 	}
@@ -716,6 +738,7 @@ static uint32_t _get_opcode(void* __restrict ctx, const char* name)
 	if(strcmp(name, "scope_stream_eof") == 0) return MODULE_PSSM_MODULE_OPCODE_SCOPE_STREAM_EOF;
 	if(strcmp(name, "scope_stream_read") == 0) return MODULE_PSSM_MODULE_OPCODE_SCOPE_STREAM_READ;
 	if(strcmp(name, "scope_stream_ready_event") == 0) return MODULE_PSSM_MODULE_OPCODE_SCOPE_STREAM_READY_EVENT;
+	if(strcmp(name, "scope_get_hash") == 0) return MODULE_PSSM_MODULE_OPCODE_SCOPE_GET_HASH;
 
 	ERROR_RETURN_LOG(uint32_t, "Invalid method name %s", name);
 }

src/sched/rscope.c

@@ -1,5 +1,6 @@
 /**
  * Copyright (C) 2017-2018, Hao Hou
+ * Copyright (C) 2018, Feng Liu
  **/
 #include <inttypes.h>
 #include <stdint.h>
@@ -456,3 +457,15 @@ int sched_rscope_stream_get_event(sched_rscope_stream_t* stream, runtime_api_sco
 	return ent->entity.event_func(stream->handle, buf);
 }
 
+int sched_rscope_get_hash(runtime_api_scope_token_t token, uint64_t out[2])
+{
+	if(_NULL_ENTRY == token || token >= _entry_table.capacity || _entry_table.data[token].data == NULL)
+	    ERROR_RETURN_LOG(int, "Invalid arguments");
+
+	const _entry_t* target = _entry_table.data + token;
+
+	if(target->data->entity.hash_func == NULL)
+		return 0;
+
+	return target->data->entity.hash_func(target->data->entity.data, out);
+}