Build: release shotit-worker v0.9.28

.github/workflows/node.js.yml

@@ -15,7 +15,7 @@ jobs:
 
     strategy:
       matrix:
-        node-version: [16.x, 18.x, 20.x]
+        node-version: [22.x, 24.x]
         # See supported Node.js release schedule at https://nodejs.org/en/about/releases/
 
     steps:

README.md

@@ -4,13 +4,13 @@
 [![GitHub Workflow Status](https://img.shields.io/github/actions/workflow/status/shotit/shotit-worker/docker-image.yml?branch=main&style=flat-square)](https://github.com/shotit/shotit-worker/actions)
 [![GitHub release](https://img.shields.io/github/release/shotit/shotit-worker.svg)](https://github.com/shotit/shotit-worker/releases/latest)
 [![Watcher Docker](https://img.shields.io/docker/pulls/shotit/shotit-worker-watcher?style=flat-square)](https://hub.docker.com/r/shotit/shotit-worker-watcher)
-[![Watcher Docker Image Size](https://img.shields.io/docker/image-size/shotit/shotit-worker-watcher/v0.9.27?style=flat-square)](https://hub.docker.com/r/shotit/shotit-worker-watcher)
+[![Watcher Docker Image Size](https://img.shields.io/docker/image-size/shotit/shotit-worker-watcher/v0.9.28?style=flat-square)](https://hub.docker.com/r/shotit/shotit-worker-watcher)
 [![Hasher Docker](https://img.shields.io/docker/pulls/shotit/shotit-worker-hasher?style=flat-square)](https://hub.docker.com/r/shotit/shotit-worker-hasher)
-[![Hasher Docker Image Size](https://img.shields.io/docker/image-size/shotit/shotit-worker-hasher/v0.9.27?style=flat-square)](https://hub.docker.com/r/shotit/shotit-worker-hasher)
+[![Hasher Docker Image Size](https://img.shields.io/docker/image-size/shotit/shotit-worker-hasher/v0.9.28?style=flat-square)](https://hub.docker.com/r/shotit/shotit-worker-hasher)
 [![Loader Docker](https://img.shields.io/docker/pulls/shotit/shotit-worker-loader?style=flat-square)](https://hub.docker.com/r/shotit/shotit-worker-loader)
-[![Loader Docker Image Size](https://img.shields.io/docker/image-size/shotit/shotit-worker-loader/v0.9.27?style=flat-square)](https://hub.docker.com/r/shotit/shotit-worker-loader)
+[![Loader Docker Image Size](https://img.shields.io/docker/image-size/shotit/shotit-worker-loader/v0.9.28?style=flat-square)](https://hub.docker.com/r/shotit/shotit-worker-loader)
 [![Searcher Docker](https://img.shields.io/docker/pulls/shotit/shotit-worker-searcher?style=flat-square)](https://hub.docker.com/r/shotit/shotit-worker-searcher)
-[![Searcher Docker Image Size](https://img.shields.io/docker/image-size/shotit/shotit-worker-searcher/v0.9.27?style=flat-square)](https://hub.docker.com/r/shotit/shotit-worker-searcher)
+[![Searcher Docker Image Size](https://img.shields.io/docker/image-size/shotit/shotit-worker-searcher/v0.9.28?style=flat-square)](https://hub.docker.com/r/shotit/shotit-worker-searcher)
 
 Backend workers for [shotit](https://github.com/shotit/shotit). Four core workers of shotit: watcher, hasher, loader and searcher.
 

hasher.Dockerfile

@@ -5,7 +5,7 @@ ENV TINI_VERSION v0.19.0
 ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini /tini
 RUN chmod +x /tini
 ENTRYPOINT ["/tini", "--"]
-RUN apt-get update && apt-get install -y ffmpeg python3 openjdk-17-jre
+RUN apt-get update && apt-get install -y ffmpeg openjdk-17-jre
 ENV NODE_ENV=production
 WORKDIR /app
 COPY ["package.json", "yarn.lock*", "./"]

hasher.js

@@ -4,10 +4,11 @@ import path from "path";
 import os from "os";
 import fs from "fs-extra";
 import child_process from "child_process";
+import fetch from "node-fetch";
 // import lzma from "lzma-native";
 // import { compress } from "@napi-rs/lzma/xz"; // original motivation: extend linux/arm64 support
-import { Compressor } from "@napi-rs/lzma/xz";
-import fetch from "node-fetch";
+import lzma from "@napi-rs/lzma/xz";
+const { Compressor } = lzma;
 
 const { TRACE_API_URL, TRACE_API_SECRET, TRACE_MEDIA_URL } = process.env;
 

loader.js

@@ -1,6 +1,6 @@
 import "dotenv/config.js";
 import WebSocket from "ws";
-import xmldoc from "xmldoc";
+// import xmldoc from "xmldoc";
 // import lzma from "lzma-native";
 import { decompress } from "@napi-rs/lzma/xz";
 import fetch from "node-fetch";
@@ -185,17 +185,84 @@ const messageHandle = async (data) => {
     const xmlData = await decompress(Buffer.from(await res.arrayBuffer()));
 
     console.log("Parsing xml");
-    const hashList = new xmldoc.XmlDocument(xmlData).children
-      .filter((child) => child.name === "doc")
-      .map((doc) => {
-        const fields = doc.children.filter((child) => child.name === "field");
-        return {
-          time: parseFloat(fields.filter((field) => field.attr.name === "id")[0].val),
-          [ALGO_hi]: fields.filter((field) => field.attr.name === `${ALGO_hi}`)[0].val,
-          [ALGO_ha]: fields.filter((field) => field.attr.name === `${ALGO_ha}`)[0].val,
-        };
-      })
-      .sort((a, b) => a.time - b.time);
+    // const hashList = new xmldoc.XmlDocument(xmlData).children
+    //   .filter((child) => child.name === "doc")
+    //   .map((doc) => {
+    //     const fields = doc.children.filter((child) => child.name === "field");
+    //     return {
+    //       time: parseFloat(fields.filter((field) => field.attr.name === "id")[0].val),
+    //       [ALGO_hi]: fields.filter((field) => field.attr.name === `${ALGO_hi}`)[0].val,
+    //       [ALGO_ha]: fields.filter((field) => field.attr.name === `${ALGO_ha}`)[0].val,
+    //     };
+    //   })
+    //   .sort((a, b) => a.time - b.time);
+
+    /**
+     * Turn to adopt a buffer flow jsonify approach
+     */
+    const parseDocBuffer = (docBuffer) => {
+      const docStr = docBuffer.toString("utf8");
+
+      const idMatch = docStr.match(/<field name="id">(.*?)<\/field>/);
+      const ALGO_hiMatch = docStr.match(new RegExp(`<field name="${ALGO_hi}">(.*?)</field>`));
+      const ALGO_haMatch = docStr.match(new RegExp(`<field name="${ALGO_ha}">(.*?)</field>`));
+
+      if (!idMatch) return null;
+
+      return {
+        time: idMatch ? idMatch[1] : null,
+        [ALGO_hi]: ALGO_hiMatch ? ALGO_hiMatch[1] : null,
+        [ALGO_ha]: ALGO_haMatch ? ALGO_haMatch[1] : null,
+      };
+    };
+
+    const xmlBufferFlowJsonify = (xmlBuffer) => {
+      const delimiter = Buffer.from("</doc>\n", "utf8");
+      const delimiterLen = delimiter.length;
+
+      let startPos = 0;
+      let nextPos = 0;
+      let count = 0;
+
+      const results = [];
+
+      while (true) {
+        nextPos = xmlBuffer.indexOf(delimiter, startPos);
+
+        if (nextPos === -1) {
+          if (startPos < xmlBuffer.length) {
+            console.log(`Processed ${count} items... End`);
+          }
+          break;
+        }
+
+        const docBuffer = xmlBuffer.slice(startPos, nextPos + delimiterLen);
+
+        try {
+          const jsonObj = parseDocBuffer(docBuffer);
+          if (jsonObj) {
+            results.push(jsonObj);
+            count++;
+
+            if (count % 1000 === 0) {
+              console.log(`Processed ${count} items...`);
+            }
+          }
+        } catch (e) {
+          console.error(`Error parsing item at offset ${startPos}:`, e.message);
+        }
+
+        startPos = nextPos + delimiterLen;
+      }
+
+      console.log(`Total processed: ${count}`);
+      return results;
+    };
+
+    const hashList = xmlBufferFlowJsonify(xmlData).sort((a, b) => a.time - b.time);
+    /**
+     * End: buffer flow jsonify approach
+     */
 
     const duration = hashList.at(-1)["time"];
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "shotit-worker",
-  "version": "0.9.27",
+  "version": "0.9.28",
   "description": "Four core workers of shotit: watcher, hasher, loader and searcher",
   "main": "",
   "type": "module",
@@ -27,7 +27,7 @@
   },
   "homepage": "https://github.com/shotit/shotit-worker#readme",
   "dependencies": {
-    "@napi-rs/lzma": "^1.4.5",
+    "@napi-rs/lzma": "1.5.1",
     "@zilliz/milvus2-sdk-node": "2.6.1",
     "body-parser": "^1.20.0",
     "chokidar": "^3.5.3",
@@ -39,8 +39,7 @@
     "lodash": "^4.17.21",
     "node-cron": "^3.0.2",
     "node-fetch": "^3.3.2",
-    "ws": "^8.14.2",
-    "xmldoc": "^1.2.0"
+    "ws": "^8.14.2"
   },
   "devDependencies": {
     "prettier": "^2.8.8"