Refactoring: PECO-219 followup, PECO-218 (get rid of HiveUtils and simplify API) (#47)

* Prepare DBSQLOperation for decomposition

* Extract checkIfOperationHasMoreRows and getResult

* Extract waitUntilReady

* Finally remove HiveUtils::WaitUntilReady; update tests; restore progress and callback support for DBSQLOperation::waitUntilReady

* Remove HiveUtils class; put all utility modules together

* Update examples

* Update docs

* Update changelog

* Apply suggestions from code review

Co-authored-by: Jesse <jesse.whitehouse@databricks.com>

Co-authored-by: Jesse <jesse.whitehouse@databricks.com>

Author: kravets-levko

CHANGELOG.md

@@ -2,6 +2,24 @@
 
 ## 0.1.x (Unreleased)
 
+- `DBSQLOperation` interface simplified: `HiveUtils` were removed and replaced with new methods
+  `DBSQLOperation.fetchChunk`/`DBSQLOperation.fetchAll`. New API implements all necessary waiting
+  and data conversion routines internally
+- Better TypeScript support
+- Thrift definitions updated to support additional Databricks features
+- User-agent string updated; a part of user-agent string is configurable through `DBSQLClient`'s `clientId` option
+- Connection now uses keep-alive (not configurable at this moment)
+- `DBSQLClient` now prepends slash to path when needed
+- `DBSQLOperation`: default chunk size for data fetching increased from 100 to 100.000
+
+### Upgrading
+
+`DBSQLClient.utils` was permanently removed. Code which used `utils.waitUntilReady`, `utils.fetchAll`
+and `utils.getResult` to get data should now be replaced with the single `DBSQLOperation.fetchAll` method.
+Progress reporting, previously supported by `utils.waitUntilReady`, is now configurable via
+`DBSQLOperation.fetchChunk`/`DBSQLOperation.fetchAll` options. `DBSQLOperation.setMaxRows` also became
+an option of methods mentioned above.
+
 ## 0.1.8-beta.1 (2022-06-24)
 
 - Initial release

README.md

@@ -33,7 +33,6 @@ npm i @databricks/sql
 const { DBSQLClient } = require('@databricks/sql');
 
 const client = new DBSQLClient();
-const utils = DBSQLClient.utils;
 
 client
   .connect({
@@ -45,11 +44,9 @@ client
     const session = await client.openSession();
 
     const queryOperation = await session.executeStatement('SELECT "Hello, World!"', { runAsync: true });
-    await utils.waitUntilReady(queryOperation, false, () => {});
-    await utils.fetchAll(queryOperation);
+    const result = await queryOperation.fetchAll();
     await queryOperation.close();
 
-    const result = utils.getResult(queryOperation).getValue();
     console.table(result);
 
     await session.close();

docs/readme.md

@@ -5,9 +5,8 @@
 1. [Foreword](#foreword)
 2. [Example](#example) \
    2.1. [Error handling](#error-handling)
-3. [HiveSession](#hivesession)
-4. [HiveOperation](#hiveoperation) \
-   4.1. [HiveUtils](#hiveutils)
+3. [DBSQLSession](#dbsqlsession)
+4. [DBSQLOperation](#dbsqloperation)
 5. [Status](#status)
 6. [Finalize](#finalize)
 
@@ -23,7 +22,6 @@ If you find any mistakes, misleading or some confusion feel free to create an is
 const { DBSQLClient } = require('@databricks/sql');
 
 const client = new DBSQLClient();
-const utils = DBSQLClient.utils;
 
 client
   .connect({
@@ -37,20 +35,17 @@ client
     const createTableOperation = await session.executeStatement(
       'CREATE TABLE IF NOT EXISTS pokes (foo INT, bar STRING)',
     );
-    await utils.waitUntilReady(createTableOperation, false, () => {});
+    await createTableOperation.fetchAll();
     await createTableOperation.close();
 
     const loadDataOperation = await session.executeStatement('INSERT INTO pokes VALUES(123, "Hello, world!"');
-    await utils.waitUntilReady(loadDataOperation, false, () => {});
+    await loadDataOperation.fetchAll();
     await loadDataOperation.close();
 
     const selectDataOperation = await session.executeStatement('SELECT * FROM pokes', { runAsync: true });
-    await utils.waitUntilReady(selectDataOperation, false, () => {});
-    await utils.fetchAll(selectDataOperation);
+    const result = await selectDataOperation.fetchAll(selectDataOperation);
     await selectDataOperation.close();
 
-    const result = utils.getResult(selectDataOperation).getValue();
-
     console.log(JSON.stringify(result, null, '\t'));
 
     await session.close();
@@ -71,9 +66,9 @@ client.on('error', (error) => {
 });
 ```
 
-## HiveSession
+## DBSQLSession
 
-After you connect to the server you should open session to start working with Hive server.
+After you connect to the server you should open session to start working with server.
 
 ```javascript
 ...
@@ -84,9 +79,9 @@ To open session you must provide [OpenSessionRequest](/lib/hive/Commands/OpenSes
 
 Into "configuration" you may set any of the configurations that required for the session of your Hive instance.
 
-After the session is opened you will have the [HiveSession](/lib/HiveSession.ts) instance.
+After the session is opened you will have the [DBSQLSession](/lib/DBSQLSession.ts) instance.
 
-Class [HiveSession](/lib/HiveSession.ts) is a facade for API that works with [SessionHandle](/lib/hive/Types/index.ts#L77).
+Class [DBSQLSession](/lib/DBSQLSession.ts) is a facade for API that works with [SessionHandle](/lib/hive/Types/index.ts#L77).
 
 The method you will use the most is `executeStatement`
 
@@ -100,81 +95,44 @@ const operation = await session.executeStatement(
 
 - "statement" is DDL/DML statement (CREATE TABLE, INSERT, UPDATE, SELECT, LOAD, etc.)
 
-- [options](/lib/contracts/IHiveSession.ts#L14)
+- [options](/lib/contracts/IDBSQLSession.ts#L14)
 
   - runAsync allows executing operation asynchronously.
 
   - confOverlay overrides session configuration properties.
 
   - timeout is the maximum time to execute an operation. It has Buffer type because timestamp in Hive has capacity 64. So for such value, you should use [node-int64](https://www.npmjs.com/package/node-int64) npm module.
 
-To know other methods see [IHiveSession](/lib/contracts/IHiveSession.ts) and [examples/session.js](/examples/session.js).
-
-## HiveOperation
+To know other methods see [IDBSQLSession](/lib/contracts/IDBSQLSession.ts) and [examples/session.js](/examples/session.js).
 
-In most cases, HiveSession methods return [HiveOperation](/lib/HiveOperation.ts), which helps you to retrieve requested data.
+## DBSQLOperation
 
-After you fetch the result, the operation will have [TableSchema](/lib/hive/Types/index.ts#L143) and data (Array<[RowSet](/lib/hive/Types/index.ts#L218)>).
+In most cases, DBSQLSession methods return [DBSQLOperation](/lib/DBSQLOperation.ts), which helps you to retrieve requested data.
 
-### HiveUtils
+After you fetch the result, the operation will have [TableSchema](/lib/hive/Types/index.ts#L143) and data.
 
-Operation is executed asynchronously, so before retrieving the result, you have to wait until it has finished state.
+Operation is executed asynchronously, but `fetchChunk`/`fetchAll` will wait until it has finished. You can
+get current status of operation any time using a dedicated method:
 
 ```javascript
 ...
 const response = await operation.status();
 const isReady = response.operationState === TCLIService_types.TOperationState.FINISHED_STATE;
 ```
 
-Also, the result is fetched by portions, the size of a portion you can set by method [setMaxRows()](/lib/HiveOperation.ts#L115).
+Also, the result is fetched by portions, the size of a portion you can pass as option to `fetchChunk`/`fetchAll`.
 
 ```javascript
 ...
-operation.setMaxRows(500);
-const status = await operation.fetch();
+const results = await operation.fetchChunk({ maxRows: 500 });
 ```
 
-After you fetch all data and you have schema and set of data, you can transfrom data in readable format.
+Schema becomes available after you start fetching data.
 
 ```javascript
 ...
+await operation.fetchChunk();
 const schema = operation.getSchema();
-const data = operation.getData();
-```
-
-To simplify this process, you may use [HiveUtils](/lib/utils/HiveUtils.ts).
-
-```typescript
-/**
- * Executes until operation has status finished or has one of the invalid states.
- *
- * @param operation operation to perform
- * @param progress flag for operation status command. If it sets true, response will include progressUpdateResponse with progress information
- * @param callback if callback specified it will be called each time the operation status response received and it will be passed as first parameter
- */
-waitUntilReady(
-    operation: IOperation,
-    progress?: boolean,
-    callback?: Function
-): Promise<IOperation>
-
-/**
- * Fetches data until operation hasMoreRows.
- *
- * @param operation
- */
-fetchAll(operation: IOperation): Promise<IOperation>
-
-/**
- * Transforms operation result
- *
- * @param operation operation to perform
- * @param resultHandler you may specify your own handler. If not specified the result is transformed to JSON
- */
-getResult(
-    operation: IOperation,
-    resultHandler?: IOperationResult
-): IOperationResult
 ```
 
 _NOTICE_
@@ -187,19 +145,13 @@ For more details see [IOperation](/lib/contracts/IOperation.ts).
 ### Example
 
 ```javascript
-const { DBSQLClient } = require('@databricks/sql');
-const utils = DBSQLClient.utils;
 ...
-await utils.waitUntilReady(
-    operation,
-    true,
-    (stateResponse) => {
-        console.log(stateResponse.taskStatus);
-    }
-);
-await utils.fetchAll(operation);
-
-const result = utils.getResult(operation).getValue();
+const result = await operation.fetchAll({
+  progress: true,
+  callback: (stateResponse) => {
+    console.log(stateResponse.taskStatus);
+  },
+});
 ```
 
 ## Status

examples/cancel_operation.js

@@ -7,14 +7,10 @@ const path = '/sql/1.0/endpoints/****';
 const token = 'dapi********************************';
 
 async function getQueryResult(operation) {
-  const utils = DBSQLClient.utils;
-
-  await utils.waitUntilReady(operation, false, () => {});
   console.log('Fetching data...');
-  await utils.fetchAll(operation);
+  const results = await operation.fetchAll();
   await operation.close();
-
-  return utils.getResult(operation).getValue();
+  return results;
 }
 
 async function cancelQuery(operation) {
@@ -41,9 +37,9 @@ client
     console.log('Running query...');
     const queryOperation = await session.executeStatement(
       `
-        SELECT id 
+        SELECT *
         FROM RANGE(100000000)
-        ORDER BY RANDOM() + 2 asc
+        ORDER BY RANDOM() ASC
       `,
       { runAsync: true },
     );

examples/data_types.js

@@ -4,7 +4,9 @@ const client = new DBSQLClient();
 
 const utils = DBSQLClient.utils;
 
-const [host, path, token] = process.argv.slice(2);
+const host = '****.databricks.com';
+const path = '/sql/1.0/endpoints/****';
+const token = 'dapi********************************';
 
 client.connect({ host, path, token }).then(async (client) => {
   try {
@@ -175,16 +177,19 @@ const testComplexTypes = async (session) => {
 const execute = async (session, statement) => {
   const operation = await session.executeStatement(statement, { runAsync: true });
 
-  await utils.waitUntilReady(operation, true, (stateResponse) => {
-    return;
-    if (stateResponse.taskStatus) {
-      console.log(stateResponse.taskStatus);
-    } else {
-      console.log(utils.formatProgress(stateResponse.progressUpdateResponse));
-    }
+  const result = await operation.fetchAll({
+    progress: true,
+    callback: (stateResponse) => {
+      return;
+      if (stateResponse.taskStatus) {
+        console.log(stateResponse.taskStatus);
+      } else {
+        console.log(utils.formatProgress(stateResponse.progressUpdateResponse));
+      }
+    },
   });
-  await utils.fetchAll(operation);
+
   await operation.close();
 
-  return utils.getResult(operation).getValue();
+  return result;
 };

examples/repl

@@ -20,14 +20,10 @@ async function initClient({ host, endpointId, token }) {
 }
 
 async function runQuery(session, query) {
-    const utils = DBSQLClient.utils;
-
     const queryOperation = await session.executeStatement(query, { runAsync: true });
-    await utils.waitUntilReady(queryOperation, false, () => {});
-    await utils.fetchAll(queryOperation);
+    const result = await queryOperation.fetchAll();
     await queryOperation.close();
-
-    return utils.getResult(queryOperation).getValue();
+    return result;
 }
 
 const format = {

examples/session.js

@@ -4,7 +4,9 @@ const client = new DBSQLClient();
 
 const utils = DBSQLClient.utils;
 
-const [host, path, token] = process.argv.slice(2);
+const host = '****.databricks.com';
+const path = '/sql/1.0/endpoints/****';
+const token = 'dapi********************************';
 
 client
   .connect({ host, path, token })
@@ -44,12 +46,14 @@ client
   });
 
 async function handleOperation(operation) {
-  await utils.waitUntilReady(operation, true, (stateResponse) => {
-    console.log(stateResponse.taskStatus);
+  const result = await operation.fetchAll({
+    progress: true,
+    callback: (stateResponse) => {
+      console.log(stateResponse.taskStatus);
+    },
   });
-  await utils.fetchAll(operation);
   await operation.close();
-  return utils.getResult(operation).getValue();
+  return result;
 }
 
 const createTables = async (session) => {

examples/usage.js

@@ -2,7 +2,9 @@ const { DBSQLClient, thrift } = require('../');
 
 const client = new DBSQLClient();
 
-const [host, path, token] = process.argv.slice(2);
+const host = '****.databricks.com';
+const path = '/sql/1.0/endpoints/****';
+const token = 'dapi********************************';
 
 client
   .connect({ host, path, token })

lib/DBSQLClient.ts

@@ -17,11 +17,8 @@ import StatusFactory from './factory/StatusFactory';
 import HiveDriverError from './errors/HiveDriverError';
 import { buildUserAgentString, definedOrError } from './utils';
 import PlainHttpAuthentication from './connection/auth/PlainHttpAuthentication';
-import HiveUtils from './utils/HiveUtils';
 
 export default class DBSQLClient extends EventEmitter implements IDBSQLClient {
-  static utils = new HiveUtils();
-
   private client: TCLIService.Client | null;
   private connection: IThriftConnection | null;
   private statusFactory: StatusFactory;