Skip to content

Commit 4186df2

Browse files
kravets-levkokravets-levko
authored
[PECO-348] Enable direct results by default (#70)
* [PECO-348] Enable direct results by default Signed-off-by: Levko Kravets <levko.ne@gmail.com> * Add tests Signed-off-by: Levko Kravets <levko.ne@gmail.com> Signed-off-by: Levko Kravets <levko.ne@gmail.com>
1 parent 36d1e8f commit 4186df2

7 files changed

Lines changed: 102 additions & 26 deletions

File tree

‎CHANGELOG.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@
66
- `DBSQLClient.openSession` now uses the latest protocol version by default
77
- Direct results feature is now available for all IOperation methods which support it. To enable direct results feature,
88
`maxRows` option should be used
9+
- Direct results became enabled by default. If `maxRows` is omitted - it will default to `100000`. To disable direct
10+
results, set `maxRows` to `null`
911
- `FunctionNameRequest` type renamed to `FunctionsRequest`
1012
- `IDBSQLConnectionOptions` type renamed to `ConnectionOptions`
1113
- `IFetchOptions` renamed to `FetchOptions`

‎lib/DBSQLOperation/index.ts‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
import { stringify, NIL, parse } from 'uuid';
2-
import IOperation, { FetchOptions, GetSchemaOptions, FinishedOptions, defaultMaxRows } from '../contracts/IOperation';
2+
import IOperation, { FetchOptions, GetSchemaOptions, FinishedOptions } from '../contracts/IOperation';
33
import HiveDriver from '../hive/HiveDriver';
44
import {
55
TGetOperationStatusResp,
@@ -16,6 +16,8 @@ import FetchResultsHelper from './FetchResultsHelper';
1616
import CompleteOperationHelper from './CompleteOperationHelper';
1717
import IDBSQLLogger, { LogLevel } from '../contracts/IDBSQLLogger';
1818

19+
const defaultMaxRows = 100000;
20+
1921
export default class DBSQLOperation implements IOperation {
2022
private driver: HiveDriver;
2123

‎lib/DBSQLSession.ts‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,14 +22,16 @@ import InfoValue from './dto/InfoValue';
2222
import { definedOrError } from './utils';
2323
import IDBSQLLogger, { LogLevel } from './contracts/IDBSQLLogger';
2424

25+
const defaultMaxRows = 100000;
26+
2527
interface OperationResponseShape {
2628
status: TStatus;
2729
operationHandle?: TOperationHandle;
2830
directResults?: TSparkDirectResults;
2931
}
3032

31-
function getDirectResultsOptions(maxRows?: number) {
32-
if (!maxRows) {
33+
function getDirectResultsOptions(maxRows: number | null = defaultMaxRows) {
34+
if (maxRows === null) {
3335
return {};
3436
}
3537

‎lib/contracts/IDBSQLSession.ts‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -6,24 +6,24 @@ import { Int64 } from '../hive/Types';
66
export type ExecuteStatementOptions = {
77
queryTimeout?: Int64;
88
runAsync?: boolean;
9-
maxRows?: number;
9+
maxRows?: number | null;
1010
};
1111

1212
export type TypeInfoRequest = {
1313
runAsync?: boolean;
14-
maxRows?: number;
14+
maxRows?: number | null;
1515
};
1616

1717
export type CatalogsRequest = {
1818
runAsync?: boolean;
19-
maxRows?: number;
19+
maxRows?: number | null;
2020
};
2121

2222
export type SchemasRequest = {
2323
catalogName?: string;
2424
schemaName?: string;
2525
runAsync?: boolean;
26-
maxRows?: number;
26+
maxRows?: number | null;
2727
};
2828

2929
export type TablesRequest = {
@@ -32,12 +32,12 @@ export type TablesRequest = {
3232
tableName?: string;
3333
tableTypes?: Array<string>;
3434
runAsync?: boolean;
35-
maxRows?: number;
35+
maxRows?: number | null;
3636
};
3737

3838
export type TableTypesRequest = {
3939
runAsync?: boolean;
40-
maxRows?: number;
40+
maxRows?: number | null;
4141
};
4242

4343
export type ColumnsRequest = {
@@ -46,23 +46,23 @@ export type ColumnsRequest = {
4646
tableName?: string;
4747
columnName?: string;
4848
runAsync?: boolean;
49-
maxRows?: number;
49+
maxRows?: number | null;
5050
};
5151

5252
export type FunctionsRequest = {
5353
catalogName?: string;
5454
schemaName?: string;
5555
functionName: string;
5656
runAsync?: boolean;
57-
maxRows?: number;
57+
maxRows?: number | null;
5858
};
5959

6060
export type PrimaryKeysRequest = {
6161
catalogName?: string;
6262
schemaName: string;
6363
tableName: string;
6464
runAsync?: boolean;
65-
maxRows?: number;
65+
maxRows?: number | null;
6666
};
6767

6868
export type CrossReferenceRequest = {
@@ -73,7 +73,7 @@ export type CrossReferenceRequest = {
7373
foreignSchemaName: string;
7474
foreignTableName: string;
7575
runAsync?: boolean;
76-
maxRows?: number;
76+
maxRows?: number | null;
7777
};
7878

7979
export default interface IDBSQLSession {

‎lib/contracts/IOperation.ts‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,6 @@ export interface GetSchemaOptions extends WaitUntilReadyOptions {
2020
// no other options
2121
}
2222

23-
export const defaultMaxRows = 100000;
24-
2523
export default interface IOperation {
2624
/**
2725
* Fetch a portion of data

‎tests/e2e/batched_fetch.test.js‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,14 +27,14 @@ describe('Data fetching', () => {
2727

2828
it('fetch chunks should return a max row set of chunkSize', async () => {
2929
const session = await openSession();
30-
const operation = await session.executeStatement(query, { runAsync: true });
30+
const operation = await session.executeStatement(query, { runAsync: true, maxRows: null });
3131
let chunkedOp = await operation.fetchChunk({ maxRows: 10 }).catch((error) => logger(error));
3232
expect(chunkedOp.length).to.be.equal(10);
3333
});
3434

3535
it('fetch all should fetch all records', async () => {
3636
const session = await openSession();
37-
const operation = await session.executeStatement(query, { runAsync: true });
37+
const operation = await session.executeStatement(query, { runAsync: true, maxRows: null });
3838
let all = await operation.fetchAll();
3939
expect(all.length).to.be.equal(1000);
4040
});

‎tests/unit/DBSQLSession.test.js‎

Lines changed: 81 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,12 @@ describe('DBSQLSession', () => {
6363
const result = await session.executeStatement('SELECT * FROM table', { maxRows: 10 });
6464
expect(result).instanceOf(DBSQLOperation);
6565
});
66+
67+
it('should disable direct results', async () => {
68+
const session = createSession();
69+
const result = await session.executeStatement('SELECT * FROM table', { maxRows: null });
70+
expect(result).instanceOf(DBSQLOperation);
71+
});
6672
});
6773

6874
describe('getTypeInfo', () => {
@@ -77,6 +83,12 @@ describe('DBSQLSession', () => {
7783
const result = await session.getTypeInfo({ maxRows: 10 });
7884
expect(result).instanceOf(DBSQLOperation);
7985
});
86+
87+
it('should disable direct results', async () => {
88+
const session = createSession();
89+
const result = await session.getTypeInfo({ maxRows: null });
90+
expect(result).instanceOf(DBSQLOperation);
91+
});
8092
});
8193

8294
describe('getCatalogs', () => {
@@ -91,6 +103,12 @@ describe('DBSQLSession', () => {
91103
const result = await session.getCatalogs({ maxRows: 10 });
92104
expect(result).instanceOf(DBSQLOperation);
93105
});
106+
107+
it('should disable direct results', async () => {
108+
const session = createSession();
109+
const result = await session.getCatalogs({ maxRows: null });
110+
expect(result).instanceOf(DBSQLOperation);
111+
});
94112
});
95113

96114
describe('getSchemas', () => {
@@ -111,9 +129,13 @@ describe('DBSQLSession', () => {
111129

112130
it('should use direct results', async () => {
113131
const session = createSession();
114-
const result = await session.getSchemas({
115-
maxRows: 10,
116-
});
132+
const result = await session.getSchemas({ maxRows: 10 });
133+
expect(result).instanceOf(DBSQLOperation);
134+
});
135+
136+
it('should disable direct results', async () => {
137+
const session = createSession();
138+
const result = await session.getSchemas({ maxRows: null });
117139
expect(result).instanceOf(DBSQLOperation);
118140
});
119141
});
@@ -138,9 +160,13 @@ describe('DBSQLSession', () => {
138160

139161
it('should use direct results', async () => {
140162
const session = createSession();
141-
const result = await session.getTables({
142-
maxRows: 10,
143-
});
163+
const result = await session.getTables({ maxRows: 10 });
164+
expect(result).instanceOf(DBSQLOperation);
165+
});
166+
167+
it('should disable direct results', async () => {
168+
const session = createSession();
169+
const result = await session.getTables({ maxRows: null });
144170
expect(result).instanceOf(DBSQLOperation);
145171
});
146172
});
@@ -157,6 +183,12 @@ describe('DBSQLSession', () => {
157183
const result = await session.getTableTypes({ maxRows: 10 });
158184
expect(result).instanceOf(DBSQLOperation);
159185
});
186+
187+
it('should disable direct results', async () => {
188+
const session = createSession();
189+
const result = await session.getTableTypes({ maxRows: null });
190+
expect(result).instanceOf(DBSQLOperation);
191+
});
160192
});
161193

162194
describe('getColumns', () => {
@@ -179,9 +211,13 @@ describe('DBSQLSession', () => {
179211

180212
it('should use direct results', async () => {
181213
const session = createSession();
182-
const result = await session.getColumns({
183-
maxRows: 10,
184-
});
214+
const result = await session.getColumns({ maxRows: 10 });
215+
expect(result).instanceOf(DBSQLOperation);
216+
});
217+
218+
it('should disable direct results', async () => {
219+
const session = createSession();
220+
const result = await session.getColumns({ maxRows: null });
185221
expect(result).instanceOf(DBSQLOperation);
186222
});
187223
});
@@ -207,6 +243,17 @@ describe('DBSQLSession', () => {
207243
});
208244
expect(result).instanceOf(DBSQLOperation);
209245
});
246+
247+
it('should disable direct results', async () => {
248+
const session = createSession();
249+
const result = await session.getFunctions({
250+
catalogName: 'catalog',
251+
schemaName: 'schema',
252+
functionName: 'avg',
253+
maxRows: null,
254+
});
255+
expect(result).instanceOf(DBSQLOperation);
256+
});
210257
});
211258

212259
describe('getPrimaryKeys', () => {
@@ -230,6 +277,17 @@ describe('DBSQLSession', () => {
230277
});
231278
expect(result).instanceOf(DBSQLOperation);
232279
});
280+
281+
it('should disable direct results', async () => {
282+
const session = createSession();
283+
const result = await session.getPrimaryKeys({
284+
catalogName: 'catalog',
285+
schemaName: 'schema',
286+
tableName: 't1',
287+
maxRows: null,
288+
});
289+
expect(result).instanceOf(DBSQLOperation);
290+
});
233291
});
234292

235293
describe('getCrossReference', () => {
@@ -259,6 +317,20 @@ describe('DBSQLSession', () => {
259317
});
260318
expect(result).instanceOf(DBSQLOperation);
261319
});
320+
321+
it('should disable direct results', async () => {
322+
const session = createSession();
323+
const result = await session.getCrossReference({
324+
parentCatalogName: 'parentCatalogName',
325+
parentSchemaName: 'parentSchemaName',
326+
parentTableName: 'parentTableName',
327+
foreignCatalogName: 'foreignCatalogName',
328+
foreignSchemaName: 'foreignSchemaName',
329+
foreignTableName: 'foreignTableName',
330+
maxRows: null,
331+
});
332+
expect(result).instanceOf(DBSQLOperation);
333+
});
262334
});
263335

264336
describe('getDelegationToken', () => {

0 commit comments

Comments
 (0)