Skip to content

Commit ce9a42b

Browse files
Hirobxcodec
andauthored
feat: resolve test and concurency (#19)
* refactor: support multi-master connection * chore: add comments * chore: fix typo * chore: fix lint * refactor: increase module version * chore: add TODO * chore: fix data-struct * chore: fix linter * chore: remove todo * chore: fix struct's name * 🎨 just refactor * 🚸 Signed-off-by: Hiro <goferHiro@gmail.com> * 🚸 mostly refactoring Signed-off-by: Hiro <goferHiro@gmail.com> * ✅ mw tests + mocking Signed-off-by: Hiro <goferHiro@gmail.com> * 🐛 mandating 1 primary db connection Signed-off-by: Hiro <goferHiro@gmail.com> * ➖ sql loggers Signed-off-by: Hiro <goferHiro@gmail.com> * 🔀 main v2 Signed-off-by: Hiro <goferHiro@gmail.com> * 🚑 replicas count overlooked Signed-off-by: Hiro <goferHiro@gmail.com> * ✅ prepare Signed-off-by: Hiro <goferHiro@gmail.com> * 🚑 bugs being fixed Signed-off-by: Hiro <goferHiro@gmail.com> * ✅ ping Signed-off-by: Hiro <goferHiro@gmail.com> * ✅ ping context Signed-off-by: Hiro <goferHiro@gmail.com> * 🚧 prepare test Signed-off-by: Hiro <goferHiro@gmail.com> * 🚑 Prepare Context, Prepare Signed-off-by: Hiro <goferHiro@gmail.com> * 🚚 more tests, refactoring, Signed-off-by: Hiro <goferHiro@gmail.com> * ✅ table tests Signed-off-by: Hiro <goferHiro@gmail.com> * ✅ table tests Signed-off-by: Hiro <goferHiro@gmail.com> * 🎨 tests Signed-off-by: Hiro <goferHiro@gmail.com> * 🎨 tests Signed-off-by: Hiro <goferHiro@gmail.com> * 🎨 tests Signed-off-by: Hiro <goferHiro@gmail.com> * 🐛 tests Signed-off-by: Hiro <goferHiro@gmail.com> * 🐛 tests Signed-off-by: Hiro <goferHiro@gmail.com> * ⬆️ deps Signed-off-by: Hiro <goferHiro@gmail.com> * 🐛 concurrent safe .. Signed-off-by: Hiro <goferHiro@gmail.com> * 🐛 tests failing for random Signed-off-by: Hiro <goferHiro@gmail.com> * ⚗️ mutex Signed-off-by: Hiro <goferHiro@gmail.com> * Revert "⚗️ mutex" This reverts commit e4e0c89. * Revert "Revert ":alembic: mutex"" This reverts commit aeb7717. * Revert "Revert "Revert ":alembic: mutex""" This reverts commit f9b40e9. * 🚧 random land balancer Signed-off-by: Hiro <goferHiro@gmail.com> * 🎨 reusable code Signed-off-by: Hiro <goferHiro@gmail.com> * 🚧 concurrency for random Signed-off-by: Hiro <goferHiro@gmail.com> * 🚑 random lb fixed for concurrency Signed-off-by: Hiro <goferHiro@gmail.com> * ✅ for loadBalancerPolicy Signed-off-by: Hiro <goferHiro@gmail.com> * :beers for loadBalancerPolicy predict Signed-off-by: Hiro <goferHiro@gmail.com> * 🔀 main Signed-off-by: Hiro <goferHiro@gmail.com> * ✅ update test Signed-off-by: Hiro <goferHiro@gmail.com> * 🔊 undo my changes for help Signed-off-by: Hiro <goferHiro@gmail.com> * chore: remove unsupported func (#18) * chore: remove unsupported func * chore: fix typo * chore: fix lint * chore: fix linter and test * chore: ignore errcheck linter for test * chore: fix linter config Signed-off-by: Hiro <goferHiro@gmail.com> Co-authored-by: Iman Tumorang <iman.tumorang@gmail.com>
1 parent e855139 commit ce9a42b

10 files changed

Lines changed: 99 additions & 132 deletions

File tree

.golangci.yaml

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -132,11 +132,14 @@ issues:
132132
- path: _test\.go
133133
linters:
134134
- gomnd
135+
- path: db_test.go
136+
text: "deferInLoop: Possible resource leak, 'defer' is called in the 'for' loop"
137+
- path: db_test.go
138+
linters:
135139
- goconst
136140
- funlen
137141
- gocyclo
138-
- path: db_test.go
139-
text: "deferInLoop: Possible resource leak, 'defer' is called in the 'for' loop"
142+
- errcheck
140143
- path: loadbalancer.go
141144
text: "G404: Use of weak random number generator" #expected, just for randomLB policy
142145
run:

db.go

Lines changed: 5 additions & 73 deletions
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,6 @@ import (
44
"context"
55
"database/sql"
66
"database/sql/driver"
7-
"errors"
8-
"strings"
97
"time"
108

119
"go.uber.org/multierr"
@@ -55,45 +53,10 @@ type StmtLoadBalancer LoadBalancer[*sql.Stmt]
5553
type sqlDB struct {
5654
primaries []*sql.DB
5755
replicas []*sql.DB
58-
totalConnection int
5956
loadBalancer DBLoadBalancer
6057
stmtLoadBalancer StmtLoadBalancer
6158
}
6259

63-
// OpenMultiPrimary concurrently opens each underlying db connection
64-
// both primaryDataSourceNames and readOnlyDataSourceNames must be a semi-comma separated list of DSNs
65-
// primaryDataSourceNames will be used as the RW-database(primary)
66-
// and readOnlyDataSourceNames as RO databases (replicas).
67-
func OpenMultiPrimary(driverName, primaryDataSourceNames, readOnlyDataSourceNames string) (res DB, err error) {
68-
primaryConns := strings.Split(primaryDataSourceNames, ";")
69-
readOnlyConns := strings.Split(readOnlyDataSourceNames, ";")
70-
71-
if len(primaryConns) == 0 {
72-
return nil, errors.New("require primary data source name")
73-
}
74-
75-
opt := defaultOption()
76-
db := &sqlDB{
77-
replicas: make([]*sql.DB, len(readOnlyConns)),
78-
primaries: make([]*sql.DB, len(primaryConns)),
79-
loadBalancer: opt.DBLB,
80-
stmtLoadBalancer: opt.StmtLB,
81-
}
82-
83-
db.totalConnection = len(primaryConns) + len(readOnlyConns)
84-
err = doParallely(db.totalConnection, func(i int) (err error) {
85-
if i < len(primaryConns) {
86-
db.primaries[0], err = sql.Open(driverName, primaryConns[i])
87-
return err
88-
}
89-
roIndex := i - len(primaryConns)
90-
db.replicas[roIndex], err = sql.Open(driverName, readOnlyConns[roIndex])
91-
return err
92-
})
93-
94-
return db, err
95-
}
96-
9760
// PrimaryDBs return all the active primary DB
9861
func (db *sqlDB) PrimaryDBs() []*sql.DB {
9962
return db.primaries
@@ -138,7 +101,7 @@ func (db *sqlDB) BeginTx(ctx context.Context, opts *sql.TxOptions) (*sql.Tx, err
138101
// The args are for any placeholder parameters in the query.
139102
// Exec uses the RW-database as the underlying db connection
140103
func (db *sqlDB) Exec(query string, args ...interface{}) (sql.Result, error) {
141-
return db.ReadWrite().Exec(query, args...)
104+
return db.ExecContext(context.Background(), query, args...)
142105
}
143106

144107
// ExecContext executes a query without returning any rows.
@@ -151,13 +114,7 @@ func (db *sqlDB) ExecContext(ctx context.Context, query string, args ...interfac
151114
// Ping verifies if a connection to each physical database is still alive,
152115
// establishing a connection if necessary.
153116
func (db *sqlDB) Ping() error {
154-
errPrimaries := doParallely(len(db.primaries), func(i int) error {
155-
return db.primaries[i].Ping()
156-
})
157-
errReplicas := doParallely(len(db.replicas), func(i int) error {
158-
return db.replicas[i].Ping()
159-
})
160-
return multierr.Combine(errPrimaries, errReplicas)
117+
return db.PingContext(context.Background())
161118
}
162119

163120
// PingContext verifies if a connection to each physical database is still
@@ -175,32 +132,7 @@ func (db *sqlDB) PingContext(ctx context.Context) error {
175132
// Prepare creates a prepared statement for later queries or executions
176133
// on each physical database, concurrently.
177134
func (db *sqlDB) Prepare(query string) (_stmt Stmt, err error) {
178-
roStmts := make([]*sql.Stmt, len(db.replicas))
179-
primaryStmts := make([]*sql.Stmt, len(db.primaries))
180-
181-
errPrimaries := doParallely(len(db.primaries), func(i int) (err error) {
182-
primaryStmts[i], err = db.primaries[i].Prepare(query)
183-
return
184-
})
185-
errReplicas := doParallely(len(db.replicas), func(i int) (err error) {
186-
roStmts[i], err = db.replicas[i].Prepare(query)
187-
return err
188-
})
189-
190-
err = multierr.Combine(errPrimaries, errReplicas)
191-
192-
if err != nil {
193-
return
194-
}
195-
196-
_stmt = &stmt{
197-
db: db,
198-
loadBalancer: db.stmtLoadBalancer,
199-
primaryStmts: primaryStmts,
200-
replicaStmts: roStmts,
201-
}
202-
203-
return
135+
return db.PrepareContext(context.Background(), query)
204136
}
205137

206138
// PrepareContext creates a prepared statement for later queries or executions
@@ -240,7 +172,7 @@ func (db *sqlDB) PrepareContext(ctx context.Context, query string) (_stmt Stmt,
240172
// The args are for any placeholder parameters in the query.
241173
// Query uses a radonly db as the physical db.
242174
func (db *sqlDB) Query(query string, args ...interface{}) (*sql.Rows, error) {
243-
return db.ReadOnly().Query(query, args...)
175+
return db.QueryContext(context.Background(), query, args...)
244176
}
245177

246178
// QueryContext executes a query that returns rows, typically a SELECT.
@@ -255,7 +187,7 @@ func (db *sqlDB) QueryContext(ctx context.Context, query string, args ...interfa
255187
// Errors are deferred until Row's Scan method is called.
256188
// QueryRow uses a radonly db as the physical db.
257189
func (db *sqlDB) QueryRow(query string, args ...interface{}) *sql.Row {
258-
return db.ReadOnly().QueryRow(query, args...)
190+
return db.QueryRowContext(context.Background(), query, args...)
259191
}
260192

261193
// QueryRowContext executes a query that is expected to return at most one row.

db_test.go

Lines changed: 52 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,22 @@ import (
99
)
1010

1111
func TestMultiWrite(t *testing.T) {
12+
loadBalancerPolices := []LoadBalancerPolicy{
13+
RoundRobinLB,
14+
RandomLB,
15+
}
16+
17+
retrieveLoadBalancer := func() (loadBalancerPolicy LoadBalancerPolicy) {
18+
loadBalancerPolicy = loadBalancerPolices[0]
19+
loadBalancerPolices = loadBalancerPolices[1:]
20+
return
21+
}
22+
23+
BEGIN_TEST:
24+
loadBalancerPolicy := retrieveLoadBalancer()
25+
26+
t.Logf("LoadBalancer-%s", loadBalancerPolicy)
27+
1228
testCases := [][2]uint{
1329
{1, 0},
1430
{1, 1},
@@ -33,10 +49,12 @@ func TestMultiWrite(t *testing.T) {
3349
return int(testCase[0]), int(testCase[1])
3450
}
3551

36-
BEGIN:
37-
52+
BEGIN_TEST_CASE:
3853
if len(testCases) == 0 {
39-
return
54+
if len(loadBalancerPolices) == 0 {
55+
return
56+
}
57+
goto BEGIN_TEST
4058
}
4159

4260
noOfPrimaries, noOfReplicas := retrieveTestCase()
@@ -63,7 +81,6 @@ BEGIN:
6381

6482
for i := 0; i < noOfReplicas; i++ {
6583
db, mock, err := createMock()
66-
6784
if err != nil {
6885
t.Fatal("creating of mock failed")
6986
}
@@ -75,37 +92,40 @@ BEGIN:
7592
mockReplicas[i] = mock
7693
}
7794

78-
resolver := New(WithPrimaryDBs(primaries...), WithReplicaDBs(replicas...)).(*sqlDB)
95+
resolver := New(WithPrimaryDBs(primaries...), WithReplicaDBs(replicas...), WithLoadBalancer(loadBalancerPolicy)).(*sqlDB)
7996

8097
t.Run("primary dbs", func(t *testing.T) {
8198
for i := 0; i < noOfPrimaries*5; i++ {
8299
robin := resolver.loadBalancer.predict(noOfPrimaries)
83100
mock := mockPimaries[robin]
84101

85-
switch i % 5 {
102+
t.Log("case - ", i%4)
103+
104+
switch i % 4 {
86105
case 0:
87106
query := "SET timezone TO 'Asia/Tokyo'"
88-
expected := mock.ExpectExec(query)
89-
_, _ = resolver.Exec(query)
90-
t.Log("exec", expected.String())
107+
mock.ExpectExec(query)
108+
resolver.Exec(query)
109+
t.Log("exec")
91110
case 1:
92111
query := "SET timezone TO 'Asia/Tokyo'"
93112
mock.ExpectExec(query)
94-
_, _ = resolver.ExecContext(context.TODO(), query)
113+
resolver.ExecContext(context.TODO(), query)
95114
t.Log("exec context")
96115
case 2:
97116
mock.ExpectBegin()
98-
_, _ = resolver.Begin()
117+
resolver.Begin()
99118
t.Log("begin")
100-
case 4:
119+
case 3:
101120
mock.ExpectBegin()
102-
_, _ = resolver.BeginTx(context.TODO(), &sql.TxOptions{
121+
resolver.BeginTx(context.TODO(), &sql.TxOptions{
103122
Isolation: sql.LevelDefault,
104123
ReadOnly: false,
105124
})
106125
t.Log("begin transaction")
126+
default:
127+
t.Fatal("developer needs to work on the tests")
107128
}
108-
109129
if err := mock.ExpectationsWereMet(); err != nil {
110130
t.Errorf("there were unfulfilled expectations: %s", err)
111131
}
@@ -117,32 +137,34 @@ BEGIN:
117137
robin := resolver.loadBalancer.predict(noOfReplicas)
118138
mock := mockReplicas[robin]
119139

120-
switch i % 5 {
140+
t.Log("case -", i%4)
141+
142+
switch i % 4 {
121143
case 0:
122144
query := "select 1'"
123145
mock.ExpectQuery(query)
124-
res, _ := resolver.Query(query)
125-
_ = res
146+
resolver.Query(query)
126147
t.Log("query")
127148
case 1:
128-
query := "select 1'"
149+
query := "select 'row'"
129150
mock.ExpectQuery(query)
130-
_ = resolver.QueryRow(query)
151+
resolver.QueryRow(query)
131152
t.Log("query row")
132153
case 2:
133-
query := "select 1'"
154+
query := "select 'query-ctx' "
134155
mock.ExpectQuery(query)
135-
res, _ := resolver.QueryContext(context.TODO(), query)
136-
_ = res
156+
resolver.QueryContext(context.TODO(), query)
137157
t.Log("query context")
138-
case 4:
139-
query := "select 1'"
158+
case 3:
159+
query := "select 'row'"
140160
mock.ExpectQuery(query)
141-
_ = resolver.QueryRowContext(context.TODO(), query)
161+
resolver.QueryRowContext(context.TODO(), query)
142162
t.Log("query row context")
163+
default:
164+
t.Fatal("developer needs to work on the tests")
143165
}
144166
if err := mock.ExpectationsWereMet(); err != nil {
145-
t.Errorf("there were unfulfilled expectations: %s", err)
167+
t.Errorf("expect failed %s", err)
146168
}
147169
}
148170
})
@@ -178,7 +200,7 @@ BEGIN:
178200

179201
mock.ExpectExec(query)
180202

181-
_, _ = stmt.Exec()
203+
stmt.Exec()
182204
})
183205

184206
t.Run("ping", func(t *testing.T) {
@@ -203,11 +225,11 @@ BEGIN:
203225

204226
err := resolver.Ping()
205227
if err != nil {
206-
t.Errorf("got %v, want %v", err, nil)
228+
t.Errorf("ping failed %s", err)
207229
}
208230
err = resolver.PingContext(context.TODO())
209231
if err != nil {
210-
t.Errorf("got %v, want %v", err, nil)
232+
t.Errorf("ping failed %s", err)
211233
}
212234
})
213235

@@ -233,7 +255,7 @@ BEGIN:
233255
t.Logf("%dP%dR", noOfPrimaries, noOfReplicas)
234256
})
235257

236-
goto BEGIN
258+
goto BEGIN_TEST_CASE
237259
}
238260

239261
func createMock() (db *sql.DB, mock sqlmock.Sqlmock, err error) {

examples/example_wrap_dbs_test.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,5 +54,5 @@ func ExampleNew() {
5454
log.Print("go error when executing the query to the DB", err)
5555
}
5656
_ = connectionDB.QueryRowContext(context.Background(), "SELECT * FROM book WHERE id=$1") // will use replicaReadOnlyDB
57-
// Output :
57+
// Output:
5858
}

go.mod

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,17 @@ module github.com/bxcodec/dbresolver/v2
22

33
go 1.19
44

5-
require github.com/lib/pq v1.10.6
5+
require (
6+
github.com/lib/pq v1.10.6
7+
github.com/mattn/go-sqlite3 v1.14.14
8+
)
69

710
require (
811
github.com/DATA-DOG/go-sqlmock v1.5.0
912
go.uber.org/multierr v1.8.0
1013
)
1114

12-
require go.uber.org/atomic v1.10.0 // indirect
15+
require (
16+
github.com/golang-jwt/jwt/v4 v4.4.3 // indirect
17+
go.uber.org/atomic v1.10.0 // indirect
18+
)

go.sum

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,12 @@ github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q
33
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
44
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
55
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
6+
github.com/golang-jwt/jwt/v4 v4.4.3 h1:Hxl6lhQFj4AnOX6MLrsCb/+7tCj7DxP7VA+2rDIq5AU=
7+
github.com/golang-jwt/jwt/v4 v4.4.3/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
68
github.com/lib/pq v1.10.6 h1:jbk+ZieJ0D7EVGJYpL9QTz7/YW6UHbmdnZWYyK5cdBs=
79
github.com/lib/pq v1.10.6/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
10+
github.com/mattn/go-sqlite3 v1.14.14 h1:qZgc/Rwetq+MtyE18WhzjokPD93dNqLGNT3QJuLvBGw=
11+
github.com/mattn/go-sqlite3 v1.14.14/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
812
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
913
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
1014
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=

helper.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ func doParallely(n int, fn func(i int) error) error {
2222
close(errors)
2323
}(wg)
2424

25-
arrErrs := []error{}
25+
var arrErrs []error
2626
for err := range errors {
2727
if err != nil {
2828
arrErrs = append(arrErrs, err)

0 commit comments

Comments
 (0)