Fixing some compile issues for alpine and centos, also fix endian issue for FLASH hashslot prefix (#722)

*fix for rocksdb centos compile fail

* fix alpine compile

* fix hashslot enumerate

* update alpine docker to latest version

* fix mac build, use big endian for expire prefix

* use endian.h for expires too

Author: msotheeswaran-sc

pkg/docker/Dockerfile_Alpine

@@ -1,4 +1,4 @@
-FROM alpine:3.12
+FROM alpine:3.18
 # add our user and group first to make sure their IDs get assigned consistently, regardless of whatever dependencies get added
 RUN addgroup -S -g 1000 keydb && adduser -S -G keydb -u 999 keydb
 RUN mkdir -p /etc/keydb

src/Makefile

@@ -358,6 +358,7 @@ endif
 OS := $(shell cat /etc/os-release | grep ID= | head -n 1 | cut -d'=' -f2)
 ifeq ($(OS),alpine)
     FINAL_CXXFLAGS+=-DUNW_LOCAL_ONLY
+	FINAL_CXXFLAGS+=-DALPINE
     FINAL_LIBS += -lunwind
 endif
 

src/readwritelock.h

@@ -42,9 +42,9 @@ class readWriteLock {
             while (m_readCount > 0)
                 m_cv.wait(rm);
             if (exclusive) {
-                /* Another thread might have the write lock while we have the read lock
-                but won't be able to release it until they can acquire the read lock
-                so release the read lock and try again instead of waiting to avoid deadlock */
+                /* Another thread might have the write lock while we have the internal lock
+                but won't be able to release it until they can acquire the internal lock
+                so release the internal lock and try again instead of waiting to avoid deadlock */
                 while(!m_writeLock.try_lock())
                     m_cv.wait(rm);
             }

src/server.h

@@ -3923,7 +3923,7 @@ void updateActiveReplicaMastersFromRsi(rdbSaveInfo *rsi);
 uint64_t getMvccTstamp();
 void incrementMvccTstamp();
 
-#if __GNUC__ >= 7 && !defined(NO_DEPRECATE_FREE)
+#if __GNUC__ >= 7 && !defined(NO_DEPRECATE_FREE) && !defined(ALPINE)
  [[deprecated]]
 void *calloc(size_t count, size_t size) noexcept;
  [[deprecated]]

src/storage/rocksdb.cpp

@@ -7,6 +7,8 @@
 #include "../cluster.h"
 #include "rocksdbfactor_internal.h"
 
+template class std::basic_string<char>;
+
 static const char keyprefix[] = INTERNAL_KEY_PREFIX;
 
 rocksdb::Options DefaultRocksDBOptions();
@@ -24,8 +26,9 @@ bool FInternalKey(const char *key, size_t cch)
 
 std::string getPrefix(unsigned int hashslot)
 {
-    char *hash_char = (char *)&hashslot;
-    return std::string(hash_char + (sizeof(unsigned int) - 2), 2);
+    HASHSLOT_PREFIX_TYPE slot = HASHSLOT_PREFIX_ENDIAN((HASHSLOT_PREFIX_TYPE)hashslot);
+    char *hash_char = (char *)&slot;
+    return std::string(hash_char, HASHSLOT_PREFIX_BYTES);
 }
 
 std::string prefixKey(const char *key, size_t cchKey)
@@ -184,7 +187,7 @@ bool RocksDBStorageProvider::enumerate(callback fn) const
         if (FInternalKey(it->key().data(), it->key().size()))
             continue;
         ++count;
-        bool fContinue = fn(it->key().data()+2, it->key().size()-2, it->value().data(), it->value().size());
+        bool fContinue = fn(it->key().data()+HASHSLOT_PREFIX_BYTES, it->key().size()-HASHSLOT_PREFIX_BYTES, it->value().data(), it->value().size());
         if (!fContinue)
             break;
     }
@@ -203,17 +206,17 @@ bool RocksDBStorageProvider::enumerate_hashslot(callback fn, unsigned int hashsl
     std::string prefix = getPrefix(hashslot);
     std::unique_ptr<rocksdb::Iterator> it = std::unique_ptr<rocksdb::Iterator>(m_spdb->NewIterator(ReadOptions(), m_spcolfamily.get()));
     size_t count = 0;
-    for (it->Seek(prefix.c_str()); it->Valid(); it->Next()) {
+    for (it->Seek(prefix); it->Valid(); it->Next()) {
         if (FInternalKey(it->key().data(), it->key().size()))
             continue;
-        if (strncmp(it->key().data(),prefix.c_str(),2) != 0)
+        if (HASHSLOT_PREFIX_RECOVER(*(HASHSLOT_PREFIX_TYPE *)it->key().data()) != hashslot)
             break;
         ++count;
-        bool fContinue = fn(it->key().data()+2, it->key().size()-2, it->value().data(), it->value().size());
+        bool fContinue = fn(it->key().data()+HASHSLOT_PREFIX_BYTES, it->key().size()-HASHSLOT_PREFIX_BYTES, it->value().data(), it->value().size());
         if (!fContinue)
             break;
     }
-    bool full_iter = !it->Valid() || (strncmp(it->key().data(),prefix.c_str(),2) != 0);
+    bool full_iter = !it->Valid() || (HASHSLOT_PREFIX_RECOVER(*(HASHSLOT_PREFIX_TYPE *)it->key().data()) != hashslot);
     if (full_iter && count != g_pserver->cluster->slots_keys_count[hashslot])
     {
         printf("WARNING: rocksdb hashslot count mismatch");
@@ -227,7 +230,8 @@ void RocksDBStorageProvider::setExpire(const char *key, size_t cchKey, long long
 {
     rocksdb::Status status;
     std::unique_lock<fastlock> l(m_lock);
-    std::string prefix((const char *)&expire,sizeof(long long));
+    long long beExpire = htobe64(expire);
+    std::string prefix((const char *)&beExpire,sizeof(long long));
     std::string strKey(key, cchKey);
     if (m_spbatch != nullptr)
         status = m_spbatch->Put(m_spexpirecolfamily.get(), rocksdb::Slice(prefix + strKey), rocksdb::Slice(strKey));
@@ -241,7 +245,8 @@ void RocksDBStorageProvider::removeExpire(const char *key, size_t cchKey, long l
 {
     rocksdb::Status status;
     std::unique_lock<fastlock> l(m_lock);
-    std::string prefix((const char *)&expire,sizeof(long long));
+    long long beExpire = htobe64(expire);
+    std::string prefix((const char *)&beExpire,sizeof(long long));
     std::string strKey(key, cchKey);
     std::string fullKey = prefix + strKey;
     if (!FExpireExists(fullKey))
@@ -278,7 +283,7 @@ std::vector<std::string> RocksDBStorageProvider::getEvictionCandidates(unsigned
         for (it->Seek(randomHashSlot()); it->Valid() && result.size() < count; it->Next()) {
             if (FInternalKey(it->key().data(), it->key().size()))
                 continue;
-            result.emplace_back(it->key().data() + 2, it->key().size() - 2);
+            result.emplace_back(it->key().data() + HASHSLOT_PREFIX_BYTES, it->key().size() - HASHSLOT_PREFIX_BYTES);
         }
     } else {
         std::unique_ptr<rocksdb::Iterator> it = std::unique_ptr<rocksdb::Iterator>(m_spdb->NewIterator(ReadOptions(), m_spexpirecolfamily.get()));

src/storage/rocksdb.h

@@ -4,9 +4,21 @@
 #include "../IStorage.h"
 #include <rocksdb/db.h>
 #include <rocksdb/utilities/write_batch_with_index.h>
+#ifdef __APPLE__
+#include <libkern/OSByteOrder.h>
+#define htole16 OSSwapHostToLittleInt16
+#define le16toh OSSwapLittleToHostInt16
+#define htobe64 OSSwapHostToBigInt64
+#else
+#include <endian.h>
+#endif
 #include "../fastlock.h"
 
 #define INTERNAL_KEY_PREFIX "\x00\x04\x03\x00\x05\x02\x04"
+#define HASHSLOT_PREFIX_TYPE uint16_t
+#define HASHSLOT_PREFIX_BYTES sizeof(HASHSLOT_PREFIX_TYPE)
+#define HASHSLOT_PREFIX_ENDIAN htole16
+#define HASHSLOT_PREFIX_RECOVER le16toh
 static const char count_key[] = INTERNAL_KEY_PREFIX "__keydb__count\1";
 static const char version_key[] = INTERNAL_KEY_PREFIX "__keydb__version\1";
 static const char meta_key[] = INTERNAL_KEY_PREFIX "__keydb__metadata\1";

src/storage/rocksdbfactory.cpp

@@ -72,7 +72,7 @@ RocksDBStorageFactory::RocksDBStorageFactory(const char *dbfile, int dbnum, cons
     rocksdb::DB *db = nullptr;
 
     auto options = RocksDbOptions();
-    options.prefix_extractor.reset(rocksdb::NewFixedPrefixTransform(2));
+    options.prefix_extractor.reset(rocksdb::NewFixedPrefixTransform(HASHSLOT_PREFIX_BYTES));
 
     for (int idb = 0; idb < dbnum; ++idb)
     {
@@ -196,7 +196,7 @@ IStorage *RocksDBStorageFactory::create(int db, key_load_iterator iter, void *pr
                 printf("\tDatabase %d was not shutdown cleanly, recomputing metrics\n", db);
             fFirstRealKey = false;
             if (iter != nullptr)
-                iter(it->key().data()+2, it->key().size()-2, privdata);
+                iter(it->key().data()+HASHSLOT_PREFIX_BYTES, it->key().size()-HASHSLOT_PREFIX_BYTES, privdata);
             ++count;
         }
     }