Merge pull request #4 from steve-krisjanovs/master

assafmo · web-flow · commit 9227a181c863 · 2020-09-14T14:13:35.000+03:00
Improved BLOB processing!
diff --git a/README.md b/README.md
@@ -26,6 +26,8 @@ Options:
     (a number)
   --cors: CORS URLs to allow requests from
     (default: [])
+  --requestlimit: request body limit for HTTP POSTs
+    (default: "1mb")
 ```
 
 # Usage examples
@@ -117,6 +119,48 @@ $ curl -H "Content-Type: application/json" -d '{"sql":"select DATETIME(?) AS UTC
 [{"UTC_ISO":"2020-09-10 02:06:02"}]
 ```
 
+## BLOB Handling
+
+Blobs via http POST/GET can be treated as byte arrays or base64-encoded text. This is handled via the blobtype variable in the HTTP GET request, or a blobtype object member in the request body for an HTTP POST. Allowable values for blobtype are "base64" and "array". If blobtype is omitted, "base64" is the default. 
+
+This also affects how your parameterized SQLite statements are sent to the server. BLOB query parameters must be structured as {"data": value}, whereas other parameter types (e.g. text, numberic) are treated as primitives in the params array (see exables below).
+
+```
+# EXAMPLES
+###############################################################################
+
+GET http://localhost:2048?sql=select BLOB_FIELD from BLOB_TABLE&blobtype=base64
+>> returns BLOB fields as base64 strings
+
+###############################################################################
+
+GET http://localhost:2048?sql=select BLOB_FIELD from BLOB_TABLE&blobtype=array
+>> returns BLOB fields as Buffer objects
+
+###############################################################################
+
+POST (application/json) http://localhost:2048
+BODY: 
+{
+    "blobtype": "base64",
+    "sql": "insert into BLOB_TABLE(KEY,BLOB_FIELD) values (?,?)",
+    "params": [1,{"data": "base64string"}]
+}
+>> base64 data is automatically converted to buffers before inserting the record
+
+###############################################################################
+
+POST (application/json) http://localhost:2048
+BODY: 
+{
+    "blobtype": "array",
+    "sql": "insert into BLOB_TABLE(KEY,BLOB_FIELD) values (?,?)",
+    "params": [1,{"data": bytearray[]}]
+}
+>> byte data is converted to buffer before inserting the record
+
+```
+
 ## CORS
 
 ```console
diff --git a/main.js b/main.js
@@ -5,12 +5,14 @@ flags.defineString("db", "", "DB File path");
 flags.defineBoolean("readonly", false, "Open the database for readonly");
 flags.defineNumber("port", 2048, "TCP Port to listen on");
 flags.defineMultiString("cors", [], "CORS URLs to allow requests from");
+flags.defineString("requestlimit", "1mb", "request body limit for HTTP POSTs");
 flags.parse();
 
 console.log("db", "=", flags.get("db"));
 console.log("readonly", "=", flags.get("readonly"));
 console.log("port", "=", flags.get("port"));
 console.log("cors", "=", flags.get("cors").join(", ") || "false");
+console.log("requestlimit", "=", flags.get("requestlimit"))
 
 const Database = require("better-sqlite3");
 
@@ -19,7 +21,7 @@ const bodyParser = require("body-parser");
 
 const app = express();
 app.use(require("compression")());
-app.use(bodyParser.urlencoded({ extended: false, limit: "1mb" }));
+app.use(bodyParser.urlencoded({ extended: false, limit: flags.get("requestlimit") }));
 app.use(bodyParser.json({ limit: "1mb" }));
 app.use(function (req, res, next) {
   req.connection.setTimeout(2 * 60 * 1000); // 2 minutes
@@ -45,35 +47,123 @@ if (flags.get("cors").length > 0) {
 
 function getSqlExecutor(httpRequestFieldName) {
   return function (req, res) {
+
+    let blobtype = req[httpRequestFieldName].blobtype;
+    if (blobtype == undefined || blobtype == null)
+    {
+      blobtype = "base64"; //default http blob handling to base64 if blobtype queryparam/bodyparam is missing
+    }
+    if ((typeof blobtype) != "string")
+    {
+      res.status(400);
+      res.send(
+        `${res.statusCode}: 'blobtype' element mandatory in http request ${httpRequestFieldName} must be string value 'base64' or 'array'!\n`
+      );
+      return;
+    }
+    blobtype = blobtype.toLowerCase().trim();
+    if (blobtype != "base64" && blobtype != "array")
+    {
+      res.status(400);
+      res.send(
+        `${res.statusCode}: 'blobtype' element mandatory in http request ${httpRequestFieldName} must be string value 'base64' or 'array'!\n`
+      );
+      return;
+    }
+
     const sql = req[httpRequestFieldName].sql;
     let params = [];
     if (httpRequestFieldName === "body" && req.is("application/json")) {
       params = req[httpRequestFieldName].params;
       if (params == undefined || params == null) {
         params = [];
       }
-    }
-    if (!sql) {
-      return res.send([]);
-    }
 
-    let db;
-    try {
       if (!Array.isArray(params)) {
         res.status(400);
         res.send(
-          `${err.code}: 'params' element in http request body must be an array!\n`
+          `${res.statusCode}: 'params' element in http request body must be an array!\n`
         );
         return;
       }
+      
+      if (blobtype === "base64") {
+        /********************************************************************************** 
+        Enumerate through sqlite parameters and if of them is a blob param
+        then decode+convert that param to a buffer.
+        - Base64 Blob parameter is structured as follows: {data: "base64data"},
+        - Non-blob parameters are primitives. not objects (numeric,string,bool)
+        ***********************************************************************************/
+        for (let i = 0; i < params.length; i++) {
+          let param = params[i];
+          //if the parameter is an object, assume it's a blob parameter
+          if (typeof param === 'object' && param !== null) {
+
+            if (param.hasOwnProperty("data"))
+            {
+              var data = param.data;
+              let buff = null;
+              if (typeof Buffer.from === "function") {
+                // Node 5.10+
+                buff = Buffer.from(data, 'base64');
+              }
+              else {
+                // older Node versions, now deprecated
+                buff = new Buffer(data, 'base64');
+              }
+              params[i] = buff;
+            }
+          }
+        }
+      }
+      else if (blobtype === "array") {
+        /********************************************************************************** 
+        Enumerate through sqlite parameters and if of them is a blob param
+        then convert the param to a buffer.
+        - Array Blob parameter is structured as follows: {data: bytearray[]},
+        - Non-blob parameters are primitives. not objects (numeric,string,bool)
+        ***********************************************************************************/
+        for (let i = 0; i < params.length; i++) {
+          let param = params[i];
+          //if the parameter is an object, assume it's a blob parameter
+          if (typeof param === 'object' && param !== null) {
+            if (param.hasOwnProperty("data"))
+            {
+              var data = param.data;
+              let buff = null;
+              if (typeof Buffer.from === "function") {
+                // Node 5.10+
+                buff = Buffer.from(data);
+              }
+              else {
+                // older Node versions, now deprecated
+                buff = new Buffer(data);
+              }
+              params[i] = buff;
+            }
+          }
+        }
+      }
+    }
+    if (!sql) {
+      return res.send([]);
+    }
+
+    let db;
+    try {
       const readonly = flags.get("readonly");
       db = new Database(flags.get("db"), { readonly });
       if (!readonly) {
         db.pragma("journal_mode = WAL");
       }
     } catch (err) {
       res.status(400);
-      res.send(`${err.code}: ${err.message}\n`);
+      //precautionary check if err doesn't have a code member
+      let errcode = res.statusCode;
+      if (err.code) {
+        errcode = err.code;
+      }
+      res.send(`${errcode}: ${err.message}\n`);
       db.close && db.close();
       return;
     }
@@ -95,6 +185,34 @@ function getSqlExecutor(httpRequestFieldName) {
     }
 
     db.close();
+
+    //if blobtype = base64, enumerate through the rows/fields and convert any buffer fields found to base64 string
+    if (blobtype == "base64")
+    {
+      for (let i = 0; i < rows.length; i++) {
+        let row = rows[i];
+
+        //get field count for row
+        let fieldcount = 0;
+        for (var prop in row) {
+          if (Object.prototype.hasOwnProperty.call(row, prop)) {
+              fieldcount ++;
+          }
+        }
+
+        //enumerate through fields
+        for (let j = 0; j < fieldcount; j++)
+        {
+          let fieldname = Object.keys(row)[j];
+          let fielddata = row[fieldname];
+          if (Buffer.isBuffer(fielddata)) {
+            let base64data = fielddata.toString('base64');
+            rows[i][fieldname] = base64data;
+          }
+        }
+      }
+    }
+
     res.send(rows);
   };
 }
