diff --git a/.github/workflows/allcov.yml b/.github/workflows/allcov.yml new file mode 100644 index 00000000..5c5708f6 --- /dev/null +++ b/.github/workflows/allcov.yml @@ -0,0 +1,30 @@ +name: allCov +on: + workflow_dispatch: + push: + branches: [ main ] + pull_request: + branches: [ main ] +jobs: + odbTest: + runs-on: ubuntu-latest + services: + mysql: + image: mysql + env: + MYSQL_ROOT_PASSWORD: 1-testDb + MYSQL_DATABASE: heratestdb + ports: + - 3306:3306 + steps: + - uses: actions/checkout@v3 + - name: Baseline for later scripted manual setup of go1.20 + uses: actions/setup-go@v4 + with: + go-version: 1.19 + - name: allCovSh + run: tests/unittest/allcover.sh + - uses: actions/upload-artifact@v3 + with: + name: coverage web page + path: /home/runner/go/allcover.htm diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index bbf044b9..75165e64 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -2,26 +2,29 @@ name: Go Build on: push: - branches: [ master ] + branches: [ main ] pull_request: - branches: [ master ] + branches: [ main ] jobs: build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - - name: Set up Go - uses: actions/setup-go@v2 - with: - go-version: 1.15 - - - name: Build - run: go build -v ./... - - name: Test - run: go test -v github.com/paypal/hera/worker/mysqlworker github.com/paypal/hera/utility/encoding/netstring github.com/paypal/hera/lib github.com/paypal/hera/common github.com/paypal/hera/cal + - name: Set up Go + uses: actions/setup-go@v4 + with: + go-version: 1.18.2 + - name: Install GO Modules + run: | + go mod tidy + go mod download github.com/godror/godror + - name: Build + run: go build -v ./... + - name: Test + run: go test -v github.com/paypal/hera/worker/mysqlworker github.com/paypal/hera/utility/encoding/netstring github.com/paypal/hera/lib github.com/paypal/hera/common github.com/paypal/hera/cal container-job: runs-on: ubuntu-latest services: @@ -33,13 +36,20 @@ jobs: ports: - 3306:3306 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - - name: Set up Go - uses: actions/setup-go@v2 - with: - go-version: 1.15 - - name: Build worker - run: go install github.com/paypal/hera/worker/mysqlworker - - name: System Test - run: tests/unittest/testall.sh + - name: Set up Go + uses: actions/setup-go@v4 + with: + go-version: 1.18.2 + - name: Install GO Modules + run: | + go mod tidy + go mod download github.com/godror/godror + - name: Build worker + run: go install github.com/paypal/hera/worker/mysqlworker + - name: System Test + run: | + export GOPATH="/home/runner/go" + echo $GOPATH + tests/unittest/testall.sh diff --git a/.github/workflows/gv-oracle.yml b/.github/workflows/gv-oracle.yml new file mode 100644 index 00000000..c387cf2b --- /dev/null +++ b/.github/workflows/gv-oracle.yml @@ -0,0 +1,26 @@ +name: gvenzl oracle +on: + workflow_dispatch: + push: + branches: [ main ] + pull_request: + branches: [ main ] +jobs: + odbTest: + runs-on: ubuntu-latest + + services: + odb3: + image: gvenzl/oracle-xe:21-slim-faststart + env: + ORACLE_PASSWORD: 1.2.8MomOfferExpand + ports: + - 1521:1521 + steps: + - uses: actions/checkout@v3 + - name: Set up Go + uses: actions/setup-go@v3 + with: + go-version: 1.15 + - name: System Test + run: tests/unittest3/testall.sh diff --git a/.github/workflows/jdbc-ci-maven.yml b/.github/workflows/jdbc-ci-maven.yml index f94bc5e3..9ac8aad9 100644 --- a/.github/workflows/jdbc-ci-maven.yml +++ b/.github/workflows/jdbc-ci-maven.yml @@ -5,9 +5,9 @@ name: Java CI with Maven on: push: - branches: [ master ] + branches: [ main ] pull_request: - branches: [ master ] + branches: [ main ] jobs: build: @@ -15,18 +15,18 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Go - uses: actions/setup-go@v2 + uses: actions/setup-go@v4 with: - go-version: 1.15 + go-version: 1.19 - name: whereami run: pwd - name: Set up JDK 8 - uses: actions/setup-java@v2 + uses: actions/setup-java@v3 with: java-version: '8' distribution: 'adopt' diff --git a/.github/workflows/manual-ft.yml b/.github/workflows/manual-ft.yml index e0a82dda..d9c941dc 100644 --- a/.github/workflows/manual-ft.yml +++ b/.github/workflows/manual-ft.yml @@ -1,7 +1,10 @@ name: Manual functionaltest on: workflow_dispatch: - + push: + branches: [ main ] + pull_request: + branches: [ main ] jobs: funcTest: runs-on: ubuntu-latest @@ -15,10 +18,10 @@ jobs: ports: - 3306:3306 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Go - uses: actions/setup-go@v2 + uses: actions/setup-go@v4 with: - go-version: 1.15 + go-version: 1.19 - name: System Test run: tests/functionaltest/runall.sh diff --git a/README.md b/README.md index 4b87afea..4e9b8ef3 100644 --- a/README.md +++ b/README.md @@ -77,6 +77,7 @@ The following sections explain the process for manually building mux without Doc 2. Install [MySQL](http://dev.mysql.com/downloads/mysql), [Oracle](https://www.oracle.com/index.html) or [PostgresSQL](https://www.postgresql.org/download/). 3. Install the [MySQL driver](https://github.com/go-sql-driver/mysql), [Oracle driver](https://github.com/go-goracle/goracle) and the [PostgreSQL driver](https://github.com/lib/pq) 4. Install Oracle instant client. +5. Install GCC. ### Build Binaries @@ -89,7 +90,7 @@ The following sections explain the process for manually building mux without Doc Clone the source code from [github](https://github.com/paypal/hera) ``` - git clone git@github.com:paypal/hera src/github.com/paypal/hera + git clone https://github.com/paypal/hera src/github.com/paypal/hera ``` Option 2 @@ -101,6 +102,7 @@ The following sections explain the process for manually building mux without Doc ``` 3. Install the dependencies: MySQL, Oracle and PostgreSQL driver ``` + cd src/github.com/paypal/hera go get github.com/go-sql-driver/mysql go get github.com/godror/godror go get github.com/lib/pq diff --git a/client/gosqldriver/connection.go b/client/gosqldriver/connection.go index 963e1923..e2186b22 100644 --- a/client/gosqldriver/connection.go +++ b/client/gosqldriver/connection.go @@ -208,3 +208,35 @@ func (c *heraConnection) SetClientInfo(poolName string, host string)(error){ } return nil } + +func (c *heraConnection) SetClientInfoWithPoolStack(poolName string, host string, poolStack string)(error){ + if len(poolName) <= 0 && len(host) <= 0 && len(poolStack) <= 0 { + return nil + } + + pid := os.Getpid() + data := fmt.Sprintf("PID: %d, HOST: %s, Poolname: %s, PoolStack: %s, Command: SetClientInfo,", pid, host, poolName, poolStack) + c.clientinfo = netstring.NewNetstringFrom(common.CmdClientInfo, []byte(string(data))) + if logger.GetLogger().V(logger.Verbose) { + logger.GetLogger().Log(logger.Verbose, "SetClientInfo", c.clientinfo.Serialized) + } + + _, err := c.conn.Write(c.clientinfo.Serialized) + if err != nil { + if logger.GetLogger().V(logger.Warning) { + logger.GetLogger().Log(logger.Warning, "Failed to send client info") + } + return errors.New("Failed custom auth, failed to send client info") + } + ns, err := c.reader.ReadNext() + if err != nil { + if logger.GetLogger().V(logger.Warning) { + logger.GetLogger().Log(logger.Warning, "Failed to read server info") + } + return errors.New("Failed to read server info") + } + if logger.GetLogger().V(logger.Debug) { + logger.GetLogger().Log(logger.Debug, "Server info:", string(ns.Payload)) + } + return nil +} \ No newline at end of file diff --git a/client/gosqldriver/ext.go b/client/gosqldriver/ext.go index f4c22dc9..96b26e5c 100644 --- a/client/gosqldriver/ext.go +++ b/client/gosqldriver/ext.go @@ -45,6 +45,8 @@ type HeraConn interface { SetCalCorrID(corrID string) SetClientInfo(poolname string, host string) error + + SetClientInfoWithPoolStack(poolName string, host string, poolStack string) error } // HeraStmt is an API extension for *sql.Stmt diff --git a/docker_build_and_run/HeraDockerfile b/docker_build_and_run/HeraDockerfile index 4f7046df..bb597889 100644 --- a/docker_build_and_run/HeraDockerfile +++ b/docker_build_and_run/HeraDockerfile @@ -15,7 +15,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM golang:1.14.3 +FROM golang:1.21.5 # Before doing a docker build, copy over the repo into this directory # tests/devdocker$ mkdir -p src/github.com/paypal/hera @@ -25,8 +25,8 @@ ADD src /go/src ADD srv /srv RUN git config --global http.sslverify false -RUN go get github.com/go-sql-driver/mysql -RUN go install github.com/paypal/hera/mux github.com/paypal/hera/worker/mysqlworker +RUN ( cd src/github.com/paypal/hera ; go get github.com/go-sql-driver/mysql ) +RUN ( cd src/github.com/paypal/hera ; go install github.com/paypal/hera/mux github.com/paypal/hera/worker/mysqlworker ) CMD ["/srv/start.sh"] diff --git a/go.mod b/go.mod index 14b12226..bd21a1f8 100644 --- a/go.mod +++ b/go.mod @@ -1,10 +1,15 @@ module github.com/paypal/hera -go 1.12 +go 1.18 require ( - github.com/go-sql-driver/mysql v1.4.1 + github.com/go-sql-driver/mysql v1.7.1 github.com/godror/godror v0.26.3 github.com/lib/pq v1.10.3 +) + +require ( + github.com/go-logfmt/logfmt v0.5.0 // indirect google.golang.org/appengine v1.6.7 // indirect + google.golang.org/protobuf v1.27.1 // indirect ) diff --git a/go.sum b/go.sum index 697ca1ba..158d55b8 100644 --- a/go.sum +++ b/go.sum @@ -1,15 +1,29 @@ -github.com/go-goracle/goracle v2.1.14+incompatible h1:uw4p4j6b6/ustNb/Yqd2rXIrBEJCGPh9ZCNFl00I9E0= -github.com/go-goracle/goracle v2.1.14+incompatible/go.mod h1:+EZ+XI0XYe/t96DSQbwEA8wvYxlyJftRDsjUhw/EVI8= -github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-logfmt/logfmt v0.5.0 h1:TrB8swr/68K7m9CcGut2g3UOihhbcbiMAYiuTXdEih4= +github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA= github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= -github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= -github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -gopkg.in/goracle.v2 v2.12.3 h1:LnNrOqNF9xj6suiuenVcOTd00S9/4aneyHarj84EaT4= -gopkg.in/goracle.v2 v2.12.3/go.mod h1:QhfGFGWSfZKBnBWnAkjd5GreYK1tan9ikkA7BMirttE= +github.com/go-sql-driver/mysql v1.7.1 h1:lUIinVbN1DY0xBg0eMOzmmtGoHwWBbvnWubQUrtU8EI= +github.com/go-sql-driver/mysql v1.7.1/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= +github.com/godror/godror v0.26.3 h1:V+z+Q/OBGgmmYzuAwyJzpcn4LsPF4Ev0xHAea68V00c= +github.com/godror/godror v0.26.3/go.mod h1:1QCn6oXh3r+IlB3DLE8V6qkHXLSHd18a3Hw7szQ9/3Y= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/lib/pq v1.10.3 h1:v9QZf2Sn6AmjXtQeFpdoq/eaNtYP6IN+7lcrygsIAtg= +github.com/lib/pq v1.10.3/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= diff --git a/lib/config.go b/lib/config.go index a0ffad0f..f0283cfa 100644 --- a/lib/config.go +++ b/lib/config.go @@ -120,6 +120,9 @@ type Config struct { // StateLogInterval int + // flag to enable CLIENT_INFO to worker + EnableCmdClientInfoToWorker bool + // if TAF is enabled EnableTAF bool // Timeout for a query to run on the primary, before fallback to secondary @@ -341,6 +344,8 @@ func InitConfig() error { gAppConfig.HostnamePrefix = parseMapStrStr(cdb.GetOrDefaultString("hostname_prefix", "")) + gAppConfig.EnableCmdClientInfoToWorker = cdb.GetOrDefaultBool("enable_client_info_to_worker", false) + gAppConfig.CfgFromTns = cdb.GetOrDefaultBool("cfg_from_tns", true) gAppConfig.CfgFromTnsOverrideNumShards = cdb.GetOrDefaultInt("cfg_from_tns_override_num_shards", -1) gAppConfig.CfgFromTnsOverrideTaf = cdb.GetOrDefaultInt("cfg_from_tns_override_taf", -1) diff --git a/lib/coordinator.go b/lib/coordinator.go index 7c20b326..3b36745a 100644 --- a/lib/coordinator.go +++ b/lib/coordinator.go @@ -53,6 +53,7 @@ type Coordinator struct { clientHostPrefix string clientHostName string poolName string + clientPoolStack string // tells if the current request is SELECT isRead bool // for debugging @@ -131,7 +132,7 @@ func (crd *Coordinator) Run() { } if crd.worker != nil { GetStateLog().PublishStateEvent(StateEvent{eType: ConnStateEvt, shardID: crd.worker.shardID, wType: crd.worker.Type, instID: crd.worker.instID, oldCState: Assign, newCState: Idle}) - go crd.worker.Recover(crd.workerpool, crd.ticket, &strandedCalInfo{raddr: crd.conn.RemoteAddr().String(), laddr: crd.conn.LocalAddr().String()}) + go crd.worker.Recover(crd.workerpool, crd.ticket, WorkerClientRecoverParam{allowSkipOciBreak: true}, &strandedCalInfo{raddr: crd.conn.RemoteAddr().String(), laddr: crd.conn.LocalAddr().String()}) crd.resetWorkerInfo() } return @@ -153,7 +154,7 @@ func (crd *Coordinator) Run() { // if (wk != nil) && !(crd.inTransaction) && (ns.IsComposite()) { GetStateLog().PublishStateEvent(StateEvent{eType: ConnStateEvt, shardID: crd.worker.shardID, wType: crd.worker.Type, instID: crd.worker.instID, oldCState: Assign, newCState: Idle}) - go crd.worker.Recover(crd.workerpool, crd.ticket, &strandedCalInfo{raddr: crd.conn.RemoteAddr().String(), laddr: crd.conn.LocalAddr().String(), nameSuffix: "_SWITCH_RECOVER"}, common.StrandedSwitch) + go crd.worker.Recover(crd.workerpool, crd.ticket, WorkerClientRecoverParam{allowSkipOciBreak: true}, &strandedCalInfo{raddr: crd.conn.RemoteAddr().String(), laddr: crd.conn.LocalAddr().String(), nameSuffix: "_SWITCH_RECOVER"}, common.StrandedSwitch) crd.resetWorkerInfo() // // ignore messages from recovering worker @@ -219,7 +220,7 @@ func (crd *Coordinator) Run() { // // not a worker failure. recover worker if failed to write to client // - go crd.worker.Recover(crd.workerpool, crd.ticket, &strandedCalInfo{raddr: crd.conn.RemoteAddr().String(), laddr: crd.conn.LocalAddr().String()}) + go crd.worker.Recover(crd.workerpool, crd.ticket, WorkerClientRecoverParam{allowSkipOciBreak: true}, &strandedCalInfo{raddr: crd.conn.RemoteAddr().String(), laddr: crd.conn.LocalAddr().String()}) } return } @@ -274,7 +275,7 @@ func (crd *Coordinator) Run() { logger.GetLogger().Log(logger.Debug, crd.id, "Run: worker ctrlchan abort", crd.worker.pid) } GetStateLog().PublishStateEvent(StateEvent{eType: ConnStateEvt, shardID: crd.worker.shardID, wType: crd.worker.Type, instID: crd.worker.instID, oldCState: Assign, newCState: Idle}) - go crd.worker.Recover(crd.workerpool, crd.ticket, &strandedCalInfo{raddr: crd.conn.RemoteAddr().String(), laddr: crd.conn.LocalAddr().String(), nameSuffix: "_SATURATION_RECOVERED"}, common.StrandedSaturationRecover) + go crd.worker.Recover(crd.workerpool, crd.ticket, WorkerClientRecoverParam{allowSkipOciBreak: !msg.bindEvict}, &strandedCalInfo{raddr: crd.conn.RemoteAddr().String(), laddr: crd.conn.LocalAddr().String(), nameSuffix: "_SATURATION_RECOVERED"}, common.StrandedSaturationRecover) crd.resetWorkerInfo() } else { // this should not happen, log in case it happens @@ -306,7 +307,7 @@ func (crd *Coordinator) Run() { et.Completed() GetStateLog().PublishStateEvent(StateEvent{eType: ConnStateEvt, shardID: crd.worker.shardID, wType: crd.worker.Type, instID: crd.worker.instID, oldCState: Assign, newCState: Idle}) - go crd.worker.Recover(crd.workerpool, crd.ticket, &strandedCalInfo{raddr: crd.conn.RemoteAddr().String(), laddr: crd.conn.LocalAddr().String()}) + go crd.worker.Recover(crd.workerpool, crd.ticket, WorkerClientRecoverParam{allowSkipOciBreak: true}, &strandedCalInfo{raddr: crd.conn.RemoteAddr().String(), laddr: crd.conn.LocalAddr().String()}) crd.resetWorkerInfo() } if logger.GetLogger().V(logger.Debug) { @@ -585,8 +586,12 @@ func (crd *Coordinator) processClientInfoMuxCommand(clientInfo string) { if pserr != nil { logger.GetLogger().Log(logger.Alert, pserr) evt := cal.NewCalEvent(cal.EventTypeClientInfo, crd.poolName, "1", pserr.Error()) + if GetConfig().EnableCmdClientInfoToWorker { + evt.SetType("CLIENT_INFO_MUX") + } evt.Completed() } + crd.clientPoolStack = parentPoolStack } } et.AddPoolStack() @@ -607,6 +612,10 @@ func (crd *Coordinator) processClientInfoMuxCommand(clientInfo string) { } } et.AddDataStr("corr_id", corrID) + // Rename after Adding PoolStack + if GetConfig().EnableCmdClientInfoToWorker { + et.SetType("CLIENT_INFO_MUX") + } et.Completed() if logger.GetLogger().V(logger.Debug) { logger.GetLogger().Log(logger.Debug, crd.id, "client info:", clientInfo, "| server info:", serverInfo, "| corr_id:", corrID) @@ -789,7 +798,7 @@ func (crd *Coordinator) dispatchRequest(request *netstring.Netstring) error { // this can happen when Oracle returns inTransaction for read SQLs if wait { GetStateLog().PublishStateEvent(StateEvent{eType: ConnStateEvt, shardID: worker.shardID, wType: worker.Type, instID: worker.instID, oldCState: Assign, newCState: Idle}) - go worker.Recover(workerpool, ticket, &strandedCalInfo{raddr: crd.conn.RemoteAddr().String(), laddr: crd.conn.LocalAddr().String()}) + go worker.Recover(workerpool, ticket, WorkerClientRecoverParam{allowSkipOciBreak: true}, &strandedCalInfo{raddr: crd.conn.RemoteAddr().String(), laddr: crd.conn.LocalAddr().String()}) return nil } } @@ -822,9 +831,9 @@ func (crd *Coordinator) dispatchRequest(request *netstring.Netstring) error { // donot return a stranded worker. recover inserts a good worker back to pool. // if err == ErrSaturationKill { - go worker.Recover(workerpool, ticket, &strandedCalInfo{raddr: crd.conn.RemoteAddr().String(), laddr: crd.conn.LocalAddr().String(), nameSuffix: "_SATURATION_RECOVERED"}, common.StrandedSaturationRecover) + go worker.Recover(workerpool, ticket, WorkerClientRecoverParam{allowSkipOciBreak: true}, &strandedCalInfo{raddr: crd.conn.RemoteAddr().String(), laddr: crd.conn.LocalAddr().String(), nameSuffix: "_SATURATION_RECOVERED"}, common.StrandedSaturationRecover) } else { - go worker.Recover(workerpool, ticket, &strandedCalInfo{raddr: crd.conn.RemoteAddr().String(), laddr: crd.conn.LocalAddr().String()}) + go worker.Recover(workerpool, ticket, WorkerClientRecoverParam{allowSkipOciBreak: true}, &strandedCalInfo{raddr: crd.conn.RemoteAddr().String(), laddr: crd.conn.LocalAddr().String()}) } } else { // @@ -886,7 +895,7 @@ func parseBinds(request *netstring.Netstring) map[string]string { */ func (crd *Coordinator) doRequest(ctx context.Context, worker *WorkerClient, request *netstring.Netstring, clientWriter io.Writer, rqTimer *time.Timer) (bool, error) { if logger.GetLogger().V(logger.Verbose) { - logger.GetLogger().Log(logger.Verbose, crd.id, "coordinator dorequeset: starting") + logger.GetLogger().Log(logger.Verbose, crd.id, "coordinator dorequest: starting") } defer func() { // @@ -932,21 +941,51 @@ func (crd *Coordinator) doRequest(ctx context.Context, worker *WorkerClient, req if corrID == nil { corrID = netstring.NewNetstringFrom(common.CmdClientCalCorrelationID, []byte("CorrId=NotSet")) } + var ns []*netstring.Netstring - if !request.IsComposite() { - ns = make([]*netstring.Netstring, 2) - ns[0] = corrID - ns[1] = request - } else { // composite - rnss, _ := netstring.SubNetstrings(request) - ns = make([]*netstring.Netstring, len(rnss)+1) - ns[0] = corrID - for i := 0; i < len(rnss); i++ { - ns[i+1] = rnss[i] + if GetConfig().EnableCmdClientInfoToWorker { + // logger.GetLogger().Log(logger.Verbose, len(crd.poolName), len(crd.clientPoolStack)) + logger.GetLogger().Log(logger.Verbose, len(crd.poolName)) + if crd.poolName == "null" || len(crd.poolName) == 0 { + crd.poolName = "unset" + } + // clientInfoMessage := fmt.Sprintf("%s|%s", crd.poolName, crd.clientPoolStack) + clientInfoMessage := crd.poolName + logger.GetLogger().Log(logger.Verbose, "GetConfig().EnableCmdClientInfoToWorker:", GetConfig().EnableCmdClientInfoToWorker) + logger.GetLogger().Log(logger.Verbose, "clientInfoMessage:", clientInfoMessage) + clientInfo := netstring.NewNetstringFrom(common.CmdClientInfo, []byte(clientInfoMessage)) + if !request.IsComposite() { + ns = make([]*netstring.Netstring, 3) + ns[0] = corrID + ns[1] = clientInfo + ns[2] = request + } else { // composite + rnss, _ := netstring.SubNetstrings(request) + ns = make([]*netstring.Netstring, len(rnss)+2) + ns[0] = corrID + ns[1] = clientInfo + for i := 0; i < len(rnss); i++ { + ns[i+2] = rnss[i] + } } + cnt += 2 + } else { + if !request.IsComposite() { + ns = make([]*netstring.Netstring, 2) + ns[0] = corrID + ns[1] = request + } else { // composite + rnss, _ := netstring.SubNetstrings(request) + ns = make([]*netstring.Netstring, len(rnss)+1) + ns[0] = corrID + for i := 0; i < len(rnss); i++ { + ns[i+1] = rnss[i] + } + } + cnt++ } plusAnyCorrId = netstring.NewNetstringEmbedded(ns) - cnt++ + } err := worker.Write(plusAnyCorrId, uint16(cnt)) if err != nil { diff --git a/lib/coordinatorsharding.go b/lib/coordinatorsharding.go index f38c3a34..60ba2114 100644 --- a/lib/coordinatorsharding.go +++ b/lib/coordinatorsharding.go @@ -398,6 +398,7 @@ func (crd *Coordinator) PreprocessSharding(requests []*netstring.Netstring) (boo } evt := cal.NewCalEvent(EvtTypeSharding, EvtNameBadShardKey, cal.TransOK, "") evt.AddDataInt("sql", int64(uint32(crd.sqlhash))) + evt.AddDataStr("shard_key", GetConfig().ShardKeyName) evt.Completed() ns := netstring.NewNetstringFrom(common.RcError, []byte(ErrNoShardValue.Error())) crd.respond(ns.Serialized) @@ -430,8 +431,9 @@ func (crd *Coordinator) PreprocessSharding(requests []*netstring.Netstring) (boo } evt := cal.NewCalEvent(EvtTypeSharding, EvtNameNoShardKey, cal.TransOK, "") evt.AddDataInt("sql", int64(uint32(crd.sqlhash))) + evt.AddDataStr("shard_key", GetConfig().ShardKeyName) evt.Completed() - ns := netstring.NewNetstringFrom(common.RcError, []byte(ErrNoShardKey.Error())) + ns := netstring.NewNetstringFrom(common.RcError, []byte(fmt.Sprintf("%s, shard_key=%s", ErrNoShardKey.Error(), GetConfig().ShardKeyName))) crd.respond(ns.Serialized) return false /*don't hangup*/, ErrNoShardKey } @@ -469,15 +471,30 @@ func (crd *Coordinator) PreprocessSharding(requests []*netstring.Netstring) (boo if len(crd.shard.shardValues) > 0 { // shard_key_auto_discovery - evt := cal.NewCalEvent(EvtTypeSharding, EvtNameShardKeyAutodisc, cal.TransOK, "") - evt.AddDataStr("shardkey", GetConfig().ShardKeyName+"|"+crd.shard.shardValues[0]) - evt.AddDataInt("shardid", int64(crd.shard.shardID)) - if len(crd.shard.shardRecs) > 0 { - evt.AddDataInt("scuttleid", int64(crd.shard.shardRecs[0].bin)) - evt.AddDataInt("flags", int64(crd.shard.shardRecs[0].flags)) + shardkey := GetConfig().ShardKeyName + "|" + crd.shard.shardValues[0] + shardid := int64(crd.shard.shardID) + shardRecs := crd.shard.shardRecs + sqlhash := int64(uint32(crd.sqlhash)) + if logger.GetLogger().V(logger.Verbose) { + logmsg := fmt.Sprintf("shard key auto discovery: shardkey=%s&shardid=%d", shardkey, shardid) + if len(shardRecs) > 0 { + logmsg += fmt.Sprintf("&scuttleid=%d&flags=%d", int64(shardRecs[0].bin), int64(shardRecs[0].flags)) + } + logmsg += fmt.Sprintf("&sqlhash=%d", sqlhash) + logger.GetLogger().Log(logger.Verbose, logmsg) + } + // restricting cal event to only String type Shard Keys + if GetConfig().ShardKeyValueTypeIsString { + evt := cal.NewCalEvent(EvtTypeSharding, EvtNameShardKeyAutodisc, cal.TransOK, "") + evt.AddDataStr("shardkey", shardkey) + evt.AddDataInt("shardid", shardid) + if len(shardRecs) > 0 { + evt.AddDataInt("scuttleid", int64(shardRecs[0].bin)) + evt.AddDataInt("flags", int64(shardRecs[0].flags)) + } + evt.AddDataInt("sqlhash", sqlhash) + evt.Completed() } - evt.AddDataInt("sqlhash", int64(uint32(crd.sqlhash))) - evt.Completed() } //This will handle scuttleID verification as part of this it compares scuttleID value with bucket value in shardRec @@ -535,7 +552,7 @@ func (crd *Coordinator) verifyValidShard() (bool, error) { crd.shard.shardRecs[0] = &ShardMapRecord{logical: 0} crd.shard.shardID = 0 } else { - ns := netstring.NewNetstringFrom(common.RcError, []byte(ErrNoShardKey.Error())) + ns := netstring.NewNetstringFrom(common.RcError, []byte(fmt.Sprintf("%s, shard_key=%s", ErrNoShardKey.Error(), GetConfig().ShardKeyName))) crd.respond(ns.Serialized) hangup := ((len(crd.shard.shardValues) > 0) && ((crd.shard.shardRecs[0].flags & ShardMapRecordFlagsBadLogical) != 0)) return hangup, ErrNoShardKey @@ -549,9 +566,10 @@ func (crd *Coordinator) verifyValidShard() (bool, error) { } evt := cal.NewCalEvent(EvtTypeSharding, EvtNameNoShardKey, cal.TransOK, "") evt.AddDataInt("sql", int64(uint32(crd.sqlhash))) + evt.AddDataStr("shard_key", GetConfig().ShardKeyName) evt.Completed() - ns := netstring.NewNetstringFrom(common.RcError, []byte(ErrNoShardKey.Error())) + ns := netstring.NewNetstringFrom(common.RcError, []byte(fmt.Sprintf("%s, shard_key=%s", ErrNoShardKey.Error(), GetConfig().ShardKeyName))) crd.respond(ns.Serialized) hangup := ((len(crd.shard.shardValues) > 0) && ((crd.shard.shardRecs[0].flags & ShardMapRecordFlagsBadLogical) != 0)) return hangup, ErrNoShardKey diff --git a/lib/coordinatortaf.go b/lib/coordinatortaf.go index 45dce177..e2ae8243 100644 --- a/lib/coordinatortaf.go +++ b/lib/coordinatortaf.go @@ -223,7 +223,7 @@ func (crd *Coordinator) DispatchTAFSession(request *netstring.Netstring) error { return nil } GetStateLog().PublishStateEvent(StateEvent{eType: ConnStateEvt, shardID: worker.shardID, wType: worker.Type, instID: worker.instID, oldCState: Assign, newCState: Idle}) - go worker.Recover(primaryPool, ticket, &strandedCalInfo{raddr: crd.conn.RemoteAddr().String(), laddr: crd.conn.LocalAddr().String()}) + go worker.Recover(primaryPool, ticket, WorkerClientRecoverParam{allowSkipOciBreak: true}, &strandedCalInfo{raddr: crd.conn.RemoteAddr().String(), laddr: crd.conn.LocalAddr().String()}) return ErrDML } GetStateLog().PublishStateEvent(StateEvent{eType: ConnStateEvt, shardID: worker.shardID, wType: worker.Type, instID: worker.instID, oldCState: Assign, newCState: Idle}) @@ -269,9 +269,10 @@ func (crd *Coordinator) DispatchTAFSession(request *netstring.Netstring) error { primaryPool.ReturnWorker(worker, ticket) // return err } else if err == ErrSaturationKill { - go worker.Recover(primaryPool, ticket, &strandedCalInfo{raddr: crd.conn.RemoteAddr().String(), laddr: crd.conn.LocalAddr().String(), nameSuffix: "_SATURATION_RECOVERED"}, common.StrandedSaturationRecover) + //go worker.Recover(primaryPool, ticket, &strandedCalInfo{raddr: crd.conn.RemoteAddr().String(), laddr: crd.conn.LocalAddr().String(), nameSuffix: "_SATURATION_RECOVERED"}, common.StrandedSaturationRecover) + go worker.Recover(primaryPool, ticket, WorkerClientRecoverParam{allowSkipOciBreak: true}, &strandedCalInfo{raddr: crd.conn.RemoteAddr().String(), laddr: crd.conn.LocalAddr().String(), nameSuffix: "_SATURATION_RECOVERED"}, common.StrandedSaturationRecover) } else { - go worker.Recover(primaryPool, ticket, &strandedCalInfo{raddr: crd.conn.RemoteAddr().String(), laddr: crd.conn.LocalAddr().String()}) + go worker.Recover(primaryPool, ticket, WorkerClientRecoverParam{allowSkipOciBreak: true}, &strandedCalInfo{raddr: crd.conn.RemoteAddr().String(), laddr: crd.conn.LocalAddr().String()}) } } // if worker fail, it is terminated so no need to recover if respProcessor.dataSent { @@ -341,7 +342,7 @@ func (crd *Coordinator) DispatchTAFSession(request *netstring.Netstring) error { return nil } GetStateLog().PublishStateEvent(StateEvent{eType: ConnStateEvt, shardID: worker.shardID, wType: worker.Type, instID: worker.instID, oldCState: Assign, newCState: Idle}) - go worker.Recover(primaryPool, ticket, &strandedCalInfo{raddr: crd.conn.RemoteAddr().String(), laddr: crd.conn.LocalAddr().String()}) + go worker.Recover(primaryPool, ticket, WorkerClientRecoverParam{allowSkipOciBreak: true}, &strandedCalInfo{raddr: crd.conn.RemoteAddr().String(), laddr: crd.conn.LocalAddr().String()}) return ErrDML } GetStateLog().PublishStateEvent(StateEvent{eType: ConnStateEvt, shardID: worker.shardID, wType: worker.Type, instID: worker.instID, oldCState: Assign, newCState: Idle}) @@ -368,9 +369,9 @@ func (crd *Coordinator) DispatchTAFSession(request *netstring.Netstring) error { fallbackPool.ReturnWorker(worker, fbticket) // return err } else if err == ErrSaturationKill { - go worker.Recover(fallbackPool, fbticket, &strandedCalInfo{raddr: crd.conn.RemoteAddr().String(), laddr: crd.conn.LocalAddr().String(), nameSuffix: "_SATURATION_RECOVERED"}, common.StrandedSaturationRecover) + go worker.Recover(fallbackPool, fbticket, WorkerClientRecoverParam{allowSkipOciBreak: true}, &strandedCalInfo{raddr: crd.conn.RemoteAddr().String(), laddr: crd.conn.LocalAddr().String(), nameSuffix: "_SATURATION_RECOVERED"}, common.StrandedSaturationRecover) } else { - go worker.Recover(fallbackPool, fbticket, &strandedCalInfo{raddr: crd.conn.RemoteAddr().String(), laddr: crd.conn.LocalAddr().String()}) + go worker.Recover(fallbackPool, fbticket, WorkerClientRecoverParam{allowSkipOciBreak: true}, &strandedCalInfo{raddr: crd.conn.RemoteAddr().String(), laddr: crd.conn.LocalAddr().String()}) } } } diff --git a/lib/racmaint.go b/lib/racmaint.go index f6b12f37..02870f09 100644 --- a/lib/racmaint.go +++ b/lib/racmaint.go @@ -22,9 +22,9 @@ import ( "database/sql" "fmt" "os" + "strconv" "strings" "time" - "strconv" "github.com/paypal/hera/cal" "github.com/paypal/hera/utility/logger" @@ -43,14 +43,14 @@ type racAct struct { delay bool } - type racCfgKey struct { - inst int + inst int module string } // MaxRacID is the maximum number of racs supported const MaxRacID = 16 + var curTime int64 var hostName string @@ -185,7 +185,7 @@ func racMaint(ctx context.Context, shard int, db *sql.DB, racSQL string, cmdLine cfgKey.inst = row.inst cfgKey.module = row.module _, ok := prev[cfgKey] - if (false == ok) { + if false == ok { racRow := racCfg{} racRow.inst = row.inst racRow.status = "U" @@ -195,17 +195,28 @@ func racMaint(ctx context.Context, shard int, db *sql.DB, racSQL string, cmdLine } if row.tm != prev[cfgKey].tm || (row.status != prev[cfgKey].status) { racReq := racAct{instID: row.inst, tm: row.tm, delay: true} - if row.status == "R" { + switch row.status { + case "R": racReq.delay = true - } else if row.status == "F" { + case "F": racReq.delay = false - } else { + case "U": + racReq.tm = 0 //This avoid accidental recycle of worker + default: // any invalid command void the action racReq.tm = 0 evt := cal.NewCalEvent("RACMAINT", "invalid_status", cal.TransOK, "") + evt.AddDataInt("inst", int64(row.inst)) + evt.AddDataStr("module", row.module) + evt.AddDataStr("status", row.status) + evt.AddDataInt("tm", int64(row.tm)) + evt.AddDataStr("priv_status", prev[cfgKey].status) evt.Completed() } + // Code flow will gets executed irrespective of valid or in-valid status but actual recycle of workers + //controlled by `racReq.tm`. + //`racReq.tm` value we are setting to "0" in-case of status "U" or "unknown" status. var workerpool *WorkerPool if strings.HasSuffix(row.module, "_TAF") { workerpool, err = GetWorkerBrokerInstance().GetWorkerPool(wtypeStdBy, 0, shard) diff --git a/lib/shardingcfg.go b/lib/shardingcfg.go index 96b508dd..c6dac50c 100644 --- a/lib/shardingcfg.go +++ b/lib/shardingcfg.go @@ -323,7 +323,11 @@ func InitShardingCfg() error { } go func() { for { - time.Sleep(time.Second * time.Duration(GetConfig().ShardingCfgReloadInterval)) + reloadInterval := time.Second * time.Duration(GetConfig().ShardingCfgReloadInterval) + if reloadInterval < 100 * time.Millisecond { + reloadInterval = 100 * time.Millisecond + } + time.Sleep(reloadInterval) for shard := 0; shard < GetConfig().NumOfShards; shard++ { if db != nil { db.Close() diff --git a/lib/workerclient.go b/lib/workerclient.go index e26ead60..1a86d619 100644 --- a/lib/workerclient.go +++ b/lib/workerclient.go @@ -125,9 +125,9 @@ type WorkerClient struct { // for bind eviction sqlBindNs atomic.Value // *netstring.Netstring - // for SQL eviction and throttle by host prefix - clientHostPrefix atomic.Value // string - clientApp atomic.Value // string + // for SQL eviction and throttle by host prefix + clientHostPrefix atomic.Value // string + clientApp atomic.Value // string // // time since hera_start in ms when the current prepare statement is sent to worker. // reset to 0 after eor meaning no sql running (same as start_time_offset_ms in c++). @@ -386,7 +386,6 @@ func (worker *WorkerClient) StartWorker() (err error) { } } - socketPair, err := syscall.Socketpair(syscall.AF_LOCAL, syscall.SOCK_STREAM, 0) if err != nil { return err @@ -571,13 +570,13 @@ func (worker *WorkerClient) Close() { /** * Sends the recover signal to the worker */ -func (worker *WorkerClient) initiateRecover(param int, p *WorkerPool, prior HeraWorkerStatus) <-chan time.Time { +func (worker *WorkerClient) initiateRecover(param int, p *WorkerPool, prior HeraWorkerStatus, recovParam WorkerClientRecoverParam) <-chan time.Time { dice := rand.Intn(100) - freePct := 100*p.activeQ.Len()/p.desiredSize + freePct := 100 * p.activeQ.Len() / p.desiredSize var rv <-chan time.Time // only skip and slow when on db-side (state==busy) - if 100-freePct > GetConfig().HighLoadPct && prior == wsBusy { + if recovParam.allowSkipOciBreak && 100-freePct > GetConfig().HighLoadPct && prior == wsBusy { timeMs := GetConfig().HighLoadStrandedWorkerTimeoutMs/2 + rand.Intn(GetConfig().HighLoadStrandedWorkerTimeoutMs) rv = time.After(time.Millisecond * time.Duration(timeMs)) if dice < GetConfig().HighLoadSkipInitiateRecoverPct { @@ -615,10 +614,23 @@ func (worker *WorkerClient) callogStranded(evtName string, info *strandedCalInfo et.Completed() } +type WorkerClientRecoverParam struct { + // adaptivequemgr.go writes a message to worker's ctrlCh + // coordinator.go reads and calls Recover() + + allowSkipOciBreak bool // for client disconn & taf + // could add param[0] which carries flags common.StrandedSkipBreakHiLoad.. + // could add *WorkerPool, ticket, calInfo + + // could add priorWorkerStatus that Recover() adds for downstream use in initiateRecover() + + // initiateRecover() later writes to workerOOBConn that goes to worker process +} + // Recover interrupts a worker busy executing a request, usually because a client went away. // It sends a break to the worker and expect the worker to respond with EOR free. If worker is not // free-ing in two seconds, the worker is stopped with SIGKILL -func (worker *WorkerClient) Recover(p *WorkerPool, ticket string, info *strandedCalInfo, param ...int) { +func (worker *WorkerClient) Recover(p *WorkerPool, ticket string, recovParam WorkerClientRecoverParam, info *strandedCalInfo, param ...int) { if atomic.CompareAndSwapInt32(&worker.isUnderRecovery, 0, 1) { if logger.GetLogger().V(logger.Debug) { logger.GetLogger().Log(logger.Debug, "begin recover worker: ", worker.pid) @@ -659,7 +671,7 @@ func (worker *WorkerClient) Recover(p *WorkerPool, ticket string, info *stranded killparam = param[0] } worker.callogStranded("RECOVERING", info) // TODO: should we have this? - workerRecoverTimeout := worker.initiateRecover(killparam, p, priorWorkerStatus) + workerRecoverTimeout := worker.initiateRecover(killparam, p, priorWorkerStatus, recovParam) for { select { case <-workerRecoverTimeout: diff --git a/mock/mockClient/java/pom.xml b/mock/mockClient/java/pom.xml index d7ef62d6..558e4888 100644 --- a/mock/mockClient/java/pom.xml +++ b/mock/mockClient/java/pom.xml @@ -199,7 +199,7 @@ org.json json - 20230227 + 20231013 com.github.jsqlparser diff --git a/tests/devdocker/Dockerfile b/tests/devdocker/Dockerfile index 4830f72c..8b1df471 100644 --- a/tests/devdocker/Dockerfile +++ b/tests/devdocker/Dockerfile @@ -15,7 +15,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM golang:1.14.3 +FROM golang:1.21.5 # Before doing a docker build, copy over the repo into this directory # tests/devdocker$ mkdir -p src/github.com/paypal/hera @@ -25,8 +25,8 @@ ADD src /go/src ADD srv /srv RUN echo $(echo -n | openssl s_client -showcerts -connect www.github.com:443 2>/dev/null | sed -ne '/-BEGIN CERTIFICATE-/,/-END CERTIFICATE-/p') >> /etc/ssl/certs/ca-certificates.crt -RUN go get github.com/go-sql-driver/mysql -RUN go install github.com/paypal/hera/mux github.com/paypal/hera/worker/mysqlworker +RUN ( cd src/github.com/paypal/hera ; go get github.com/go-sql-driver/mysql ) +RUN ( cd src/github.com/paypal/hera ; go install github.com/paypal/hera/mux github.com/paypal/hera/worker/mysqlworker ) CMD ["/srv/start.sh"] diff --git a/tests/devdocker/srv/start.sh b/tests/devdocker/srv/start.sh index b029943a..aed1bc61 100755 --- a/tests/devdocker/srv/start.sh +++ b/tests/devdocker/srv/start.sh @@ -20,22 +20,22 @@ cd /srv ln -sf $GOPATH/bin/mux . ln -sf $GOPATH/bin/mysqlworker . -export TWO_TASK="tcp(127.0.0.1:3306)/testschema" +export TWO_TASK="tcp(mysql-11:3306)/testschema" # consider using dns so DBAs can move db nodes around # for read-write split -#export TWO_TASK_READ=tcp(127.0.0.1:3306)/testschema +#export TWO_TASK_READ=tcp(mysql-11:3306)/testschema # for read-replica retry -#export TWO_TASK_STANDBY0=tcp(127.0.0.1:3306)/testschema +#export TWO_TASK_STANDBY0=tcp(mysql-11:3306)/testschema # for sharding -#export TWO_TASK_0=tcp(127.0.0.1:3306)/testschema -#export TWO_TASK_READ_0=tcp(127.0.0.1:3306)/testschema -#export TWO_TASK_STANDBY0_0=tcp(127.0.0.1:3306)/testschema -#export TWO_TASK_1=tcp(127.0.0.1:3306)/testschema -#export TWO_TASK_READ_1=tcp(127.0.0.1:3306)/testschema -#export TWO_TASK_STANDBY0_1=tcp(127.0.0.1:3306)/testschema +#export TWO_TASK_0=tcp(mysql-11:3306)/testschema +#export TWO_TASK_READ_0=tcp(mysql-11:3306)/testschema +#export TWO_TASK_STANDBY0_0=tcp(mysql-11:3306)/testschema +#export TWO_TASK_1=tcp(mysql-11:3306)/testschema +#export TWO_TASK_READ_1=tcp(mysql-11:3306)/testschema +#export TWO_TASK_STANDBY0_1=tcp(mysql-11:3306)/testschema export username=root # docker command should pass in db password diff --git a/tests/functionaltest/adaptive_queue_tests/backlog_longq_shortq/main_test.go b/tests/functionaltest/adaptive_queue_tests/backlog_longq_shortq/main_test_disable.go similarity index 100% rename from tests/functionaltest/adaptive_queue_tests/backlog_longq_shortq/main_test.go rename to tests/functionaltest/adaptive_queue_tests/backlog_longq_shortq/main_test_disable.go diff --git a/tests/functionaltest/adaptive_queue_tests/backlog_longq_shortq/stub.go b/tests/functionaltest/adaptive_queue_tests/backlog_longq_shortq/stub.go new file mode 100644 index 00000000..fcf0bd74 --- /dev/null +++ b/tests/functionaltest/adaptive_queue_tests/backlog_longq_shortq/stub.go @@ -0,0 +1,2 @@ +package main +func main(){} diff --git a/tests/functionaltest/adaptive_queue_tests/backlog_longq_timeout/main_test.go b/tests/functionaltest/adaptive_queue_tests/backlog_longq_timeout/main_test_disable.go similarity index 100% rename from tests/functionaltest/adaptive_queue_tests/backlog_longq_timeout/main_test.go rename to tests/functionaltest/adaptive_queue_tests/backlog_longq_timeout/main_test_disable.go diff --git a/tests/functionaltest/adaptive_queue_tests/backlog_longq_timeout/stub.go b/tests/functionaltest/adaptive_queue_tests/backlog_longq_timeout/stub.go new file mode 100644 index 00000000..fcf0bd74 --- /dev/null +++ b/tests/functionaltest/adaptive_queue_tests/backlog_longq_timeout/stub.go @@ -0,0 +1,2 @@ +package main +func main(){} diff --git a/tests/functionaltest/backoff_recycle/small_lifespan/main_test.go b/tests/functionaltest/backoff_recycle/small_lifespan/main_test_disable.go similarity index 98% rename from tests/functionaltest/backoff_recycle/small_lifespan/main_test.go rename to tests/functionaltest/backoff_recycle/small_lifespan/main_test_disable.go index 77824bba..52cb3f93 100644 --- a/tests/functionaltest/backoff_recycle/small_lifespan/main_test.go +++ b/tests/functionaltest/backoff_recycle/small_lifespan/main_test_disable.go @@ -20,7 +20,6 @@ func cfg() (map[string]string, map[string]string, testutil.WorkerType) { appcfg["bind_port"] = "31002" appcfg["log_level"] = "5" appcfg["log_file"] = "hera.log" - appcfg["sharding_cfg_reload_interval"] = "0" appcfg["rac_sql_interval"] = "2" appcfg["db_heartbeat_interval"] = "3" diff --git a/tests/functionaltest/backoff_recycle/small_lifespan/stub.go b/tests/functionaltest/backoff_recycle/small_lifespan/stub.go new file mode 100644 index 00000000..fcf0bd74 --- /dev/null +++ b/tests/functionaltest/backoff_recycle/small_lifespan/stub.go @@ -0,0 +1,2 @@ +package main +func main(){} diff --git a/tests/functionaltest/bind_eviction_tests/bind_throttle/main_test.go b/tests/functionaltest/bind_eviction_tests/bind_throttle/main_test_disable.go similarity index 100% rename from tests/functionaltest/bind_eviction_tests/bind_throttle/main_test.go rename to tests/functionaltest/bind_eviction_tests/bind_throttle/main_test_disable.go diff --git a/tests/functionaltest/bind_eviction_tests/bind_throttle/stub.go b/tests/functionaltest/bind_eviction_tests/bind_throttle/stub.go new file mode 100644 index 00000000..fcf0bd74 --- /dev/null +++ b/tests/functionaltest/bind_eviction_tests/bind_throttle/stub.go @@ -0,0 +1,2 @@ +package main +func main(){} diff --git a/tests/functionaltest/bind_eviction_tests/evict_source/main_test.go b/tests/functionaltest/bind_eviction_tests/evict_source/main_test_disable.go similarity index 100% rename from tests/functionaltest/bind_eviction_tests/evict_source/main_test.go rename to tests/functionaltest/bind_eviction_tests/evict_source/main_test_disable.go diff --git a/tests/functionaltest/bind_eviction_tests/evict_source/stub.go b/tests/functionaltest/bind_eviction_tests/evict_source/stub.go new file mode 100644 index 00000000..fcf0bd74 --- /dev/null +++ b/tests/functionaltest/bind_eviction_tests/evict_source/stub.go @@ -0,0 +1,2 @@ +package main +func main(){} diff --git a/tests/functionaltest/bind_eviction_tests/inclause_evict/main_test.go b/tests/functionaltest/bind_eviction_tests/inclause_evict/main_test_disable.go similarity index 100% rename from tests/functionaltest/bind_eviction_tests/inclause_evict/main_test.go rename to tests/functionaltest/bind_eviction_tests/inclause_evict/main_test_disable.go diff --git a/tests/functionaltest/bind_eviction_tests/inclause_evict/stub.go b/tests/functionaltest/bind_eviction_tests/inclause_evict/stub.go new file mode 100644 index 00000000..fcf0bd74 --- /dev/null +++ b/tests/functionaltest/bind_eviction_tests/inclause_evict/stub.go @@ -0,0 +1,2 @@ +package main +func main(){} diff --git a/tests/functionaltest/bind_eviction_tests/simple_evict/main_test.go b/tests/functionaltest/bind_eviction_tests/simple_evict/main_test_disable.go similarity index 100% rename from tests/functionaltest/bind_eviction_tests/simple_evict/main_test.go rename to tests/functionaltest/bind_eviction_tests/simple_evict/main_test_disable.go diff --git a/tests/functionaltest/bind_eviction_tests/simple_evict/stub.go b/tests/functionaltest/bind_eviction_tests/simple_evict/stub.go new file mode 100644 index 00000000..fcf0bd74 --- /dev/null +++ b/tests/functionaltest/bind_eviction_tests/simple_evict/stub.go @@ -0,0 +1,2 @@ +package main +func main(){} diff --git a/tests/functionaltest/coordinator_tests/fifo/main_test.go b/tests/functionaltest/coordinator_tests/fifo/main_test.go index 66702f4f..cad54cdb 100644 --- a/tests/functionaltest/coordinator_tests/fifo/main_test.go +++ b/tests/functionaltest/coordinator_tests/fifo/main_test.go @@ -27,7 +27,6 @@ func cfg() (map[string]string, map[string]string, testutil.WorkerType) { appcfg["bind_port"] = "31002" appcfg["log_level"] = "5" appcfg["log_file"] = "hera.log" - appcfg["sharding_cfg_reload_interval"] = "0" appcfg["rac_sql_interval"] = "0" appcfg["lifo_scheduler_enabled"] = "false" appcfg["child.executable"] = "mysqlworker" diff --git a/tests/functionaltest/coordinator_tests/idle_timeout_workerassigned/main_test.go b/tests/functionaltest/coordinator_tests/idle_timeout_workerassigned/main_test.go index ddbe46d0..20baa4d1 100644 --- a/tests/functionaltest/coordinator_tests/idle_timeout_workerassigned/main_test.go +++ b/tests/functionaltest/coordinator_tests/idle_timeout_workerassigned/main_test.go @@ -28,7 +28,6 @@ func cfg() (map[string]string, map[string]string, testutil.WorkerType) { appcfg["bind_port"] = "31002" appcfg["log_level"] = "5" appcfg["log_file"] = "hera.log" - appcfg["sharding_cfg_reload_interval"] = "0" appcfg["rac_sql_interval"] = "0" appcfg["opscfg.default.server.idle_timeout_ms"] = "3000" appcfg["database_type"] = "mysql" diff --git a/tests/functionaltest/coordinator_tests/idle_timeout_workernotassigned/main_test.go b/tests/functionaltest/coordinator_tests/idle_timeout_workernotassigned/main_test.go index e708538e..3afe625e 100644 --- a/tests/functionaltest/coordinator_tests/idle_timeout_workernotassigned/main_test.go +++ b/tests/functionaltest/coordinator_tests/idle_timeout_workernotassigned/main_test.go @@ -26,7 +26,6 @@ func cfg() (map[string]string, map[string]string, testutil.WorkerType) { appcfg["bind_port"] = "31002" appcfg["log_level"] = "5" appcfg["log_file"] = "hera.log" - appcfg["sharding_cfg_reload_interval"] = "0" appcfg["rac_sql_interval"] = "0" appcfg["opscfg.default.server.idle_timeout_ms"] = "4000" appcfg["child.executable"] = "mysqlworker" diff --git a/tests/functionaltest/coordinator_tests/lifo/main_test.go b/tests/functionaltest/coordinator_tests/lifo/main_test.go index 90033358..efcb4aa1 100644 --- a/tests/functionaltest/coordinator_tests/lifo/main_test.go +++ b/tests/functionaltest/coordinator_tests/lifo/main_test.go @@ -26,7 +26,6 @@ func cfg() (map[string]string, map[string]string, testutil.WorkerType) { appcfg["bind_port"] = "31002" appcfg["log_level"] = "5" appcfg["log_file"] = "hera.log" - appcfg["sharding_cfg_reload_interval"] = "0" appcfg["rac_sql_interval"] = "0" appcfg["child.executable"] = "mysqlworker" appcfg["database_type"] = "mysql" diff --git a/tests/functionaltest/coordinator_tests/max_lifespan_dml/main_test.go b/tests/functionaltest/coordinator_tests/max_lifespan_dml/main_test.go index ac4fc177..cc739671 100644 --- a/tests/functionaltest/coordinator_tests/max_lifespan_dml/main_test.go +++ b/tests/functionaltest/coordinator_tests/max_lifespan_dml/main_test.go @@ -29,7 +29,6 @@ func cfg() (map[string]string, map[string]string, testutil.WorkerType) { appcfg["bind_port"] = "31002" appcfg["log_level"] = "5" appcfg["log_file"] = "hera.log" - appcfg["sharding_cfg_reload_interval"] = "0" appcfg["rac_sql_interval"] = "0" appcfg["opscfg.default.server.max_lifespan_per_child"] = "5" appcfg["child.executable"] = "mysqlworker" diff --git a/tests/functionaltest/coordinator_tests/max_request_per_child_dml/main_test.go b/tests/functionaltest/coordinator_tests/max_request_per_child_dml/main_test.go index 949f7722..3e537c0c 100644 --- a/tests/functionaltest/coordinator_tests/max_request_per_child_dml/main_test.go +++ b/tests/functionaltest/coordinator_tests/max_request_per_child_dml/main_test.go @@ -28,7 +28,6 @@ func cfg() (map[string]string, map[string]string, testutil.WorkerType) { appcfg["bind_port"] = "31002" appcfg["log_level"] = "5" appcfg["log_file"] = "hera.log" - appcfg["sharding_cfg_reload_interval"] = "0" appcfg["rac_sql_interval"] = "0" appcfg["opscfg.default.server.max_requests_per_child"] = "10" appcfg["child.executable"] = "mysqlworker" diff --git a/tests/functionaltest/coordinator_tests/max_request_per_child_nondml/main_test.go b/tests/functionaltest/coordinator_tests/max_request_per_child_nondml/main_test_disable.go similarity index 99% rename from tests/functionaltest/coordinator_tests/max_request_per_child_nondml/main_test.go rename to tests/functionaltest/coordinator_tests/max_request_per_child_nondml/main_test_disable.go index bd631f45..cd3b9e05 100644 --- a/tests/functionaltest/coordinator_tests/max_request_per_child_nondml/main_test.go +++ b/tests/functionaltest/coordinator_tests/max_request_per_child_nondml/main_test_disable.go @@ -28,7 +28,6 @@ func cfg() (map[string]string, map[string]string, testutil.WorkerType) { appcfg["bind_port"] = "31002" appcfg["log_level"] = "5" appcfg["log_file"] = "hera.log" - appcfg["sharding_cfg_reload_interval"] = "0" appcfg["rac_sql_interval"] = "0" appcfg["opscfg.default.server.max_requests_per_child"] = "4" appcfg["child.executable"] = "mysqlworker" diff --git a/tests/functionaltest/coordinator_tests/max_request_per_child_nondml/stub.go b/tests/functionaltest/coordinator_tests/max_request_per_child_nondml/stub.go new file mode 100644 index 00000000..fcf0bd74 --- /dev/null +++ b/tests/functionaltest/coordinator_tests/max_request_per_child_nondml/stub.go @@ -0,0 +1,2 @@ +package main +func main(){} diff --git a/tests/functionaltest/coordinator_tests/max_request_per_child_opscfg_change/main_test.go b/tests/functionaltest/coordinator_tests/max_request_per_child_opscfg_change/main_test_disable.go similarity index 99% rename from tests/functionaltest/coordinator_tests/max_request_per_child_opscfg_change/main_test.go rename to tests/functionaltest/coordinator_tests/max_request_per_child_opscfg_change/main_test_disable.go index 7ae34cae..6977323e 100644 --- a/tests/functionaltest/coordinator_tests/max_request_per_child_opscfg_change/main_test.go +++ b/tests/functionaltest/coordinator_tests/max_request_per_child_opscfg_change/main_test_disable.go @@ -27,7 +27,6 @@ func cfg() (map[string]string, map[string]string, testutil.WorkerType) { appcfg["bind_port"] = "31002" appcfg["log_level"] = "5" appcfg["log_file"] = "hera.log" - appcfg["sharding_cfg_reload_interval"] = "0" appcfg["rac_sql_interval"] = "0" appcfg["opscfg.default.server.max_requests_per_child"] = "4" appcfg["child.executable"] = "mysqlworker" diff --git a/tests/functionaltest/coordinator_tests/max_request_per_child_opscfg_change/stub.go b/tests/functionaltest/coordinator_tests/max_request_per_child_opscfg_change/stub.go new file mode 100644 index 00000000..fcf0bd74 --- /dev/null +++ b/tests/functionaltest/coordinator_tests/max_request_per_child_opscfg_change/stub.go @@ -0,0 +1,2 @@ +package main +func main(){} diff --git a/tests/functionaltest/coordinator_tests/no_max_lifespan_per_child/main_test.go b/tests/functionaltest/coordinator_tests/no_max_lifespan_per_child/main_test.go index 4a6353f1..2409dbce 100644 --- a/tests/functionaltest/coordinator_tests/no_max_lifespan_per_child/main_test.go +++ b/tests/functionaltest/coordinator_tests/no_max_lifespan_per_child/main_test.go @@ -29,7 +29,6 @@ func cfg() (map[string]string, map[string]string, testutil.WorkerType) { appcfg["bind_port"] = "31002" appcfg["log_level"] = "5" appcfg["log_file"] = "hera.log" - appcfg["sharding_cfg_reload_interval"] = "0" appcfg["rac_sql_interval"] = "0" appcfg["child.executable"] = "mysqlworker" appcfg["database_type"] = "mysql" diff --git a/tests/functionaltest/coordinator_tests/no_max_request_per_child/main_test.go b/tests/functionaltest/coordinator_tests/no_max_request_per_child/main_test.go index bd92a4d7..747b6b4c 100644 --- a/tests/functionaltest/coordinator_tests/no_max_request_per_child/main_test.go +++ b/tests/functionaltest/coordinator_tests/no_max_request_per_child/main_test.go @@ -27,7 +27,6 @@ func cfg() (map[string]string, map[string]string, testutil.WorkerType) { appcfg["bind_port"] = "31002" appcfg["log_level"] = "5" appcfg["log_file"] = "hera.log" - appcfg["sharding_cfg_reload_interval"] = "0" appcfg["rac_sql_interval"] = "0" appcfg["enable_cache"] = "true" appcfg["child.executable"] = "mysqlworker" diff --git a/tests/functionaltest/rac_tests/disable_rac_feature/main_test.go b/tests/functionaltest/rac_tests/disable_rac_feature/main_test.go index 002e45bc..cec24eb5 100644 --- a/tests/functionaltest/rac_tests/disable_rac_feature/main_test.go +++ b/tests/functionaltest/rac_tests/disable_rac_feature/main_test.go @@ -27,7 +27,6 @@ func cfg() (map[string]string, map[string]string, testutil.WorkerType) { appcfg["bind_port"] = "31002" appcfg["log_level"] = "5" appcfg["log_file"] = "hera.log" - appcfg["sharding_cfg_reload_interval"] = "0" appcfg["rac_sql_interval"] = "0" appcfg["lifespan_check_interval"] = "1" appcfg["child.executable"] = "mysqlworker" diff --git a/tests/functionaltest/rac_tests/disable_rac_feature_2/main_test.go b/tests/functionaltest/rac_tests/disable_rac_feature_2/main_test.go index 777bf2b1..e7240133 100644 --- a/tests/functionaltest/rac_tests/disable_rac_feature_2/main_test.go +++ b/tests/functionaltest/rac_tests/disable_rac_feature_2/main_test.go @@ -27,7 +27,6 @@ func cfg() (map[string]string, map[string]string, testutil.WorkerType) { appcfg["bind_port"] = "31002" appcfg["log_level"] = "5" appcfg["log_file"] = "hera.log" - appcfg["sharding_cfg_reload_interval"] = "0" appcfg["rac_sql_interval"] = "-1" appcfg["lifespan_check_interval"] = "1" appcfg["child.executable"] = "mysqlworker" diff --git a/tests/functionaltest/rac_tests/empty_rac_sql_interval/main_test.go b/tests/functionaltest/rac_tests/empty_rac_sql_interval/main_test.go index 6b44cc89..96506855 100644 --- a/tests/functionaltest/rac_tests/empty_rac_sql_interval/main_test.go +++ b/tests/functionaltest/rac_tests/empty_rac_sql_interval/main_test.go @@ -25,7 +25,6 @@ func cfg() (map[string]string, map[string]string, testutil.WorkerType) { appcfg["bind_port"] = "31002" appcfg["log_level"] = "5" appcfg["log_file"] = "hera.log" - appcfg["sharding_cfg_reload_interval"] = "0" appcfg["rac_sql_interval"] = "" appcfg["child.executable"] = "mysqlworker" appcfg["database_type"] = "mysql" diff --git a/tests/functionaltest/rac_tests/empty_timefield/main_test.go b/tests/functionaltest/rac_tests/empty_timefield/main_test_disable.go similarity index 99% rename from tests/functionaltest/rac_tests/empty_timefield/main_test.go rename to tests/functionaltest/rac_tests/empty_timefield/main_test_disable.go index 4da3caab..0cb4c789 100644 --- a/tests/functionaltest/rac_tests/empty_timefield/main_test.go +++ b/tests/functionaltest/rac_tests/empty_timefield/main_test_disable.go @@ -28,7 +28,6 @@ func cfg() (map[string]string, map[string]string, testutil.WorkerType) { appcfg["bind_port"] = "31002" appcfg["log_level"] = "5" appcfg["log_file"] = "hera.log" - appcfg["sharding_cfg_reload_interval"] = "0" appcfg["rac_sql_interval"] = "2" appcfg["lifespan_check_interval"] = "1" appcfg["child.executable"] = "mysqlworker" diff --git a/tests/functionaltest/rac_tests/empty_timefield/stub.go b/tests/functionaltest/rac_tests/empty_timefield/stub.go new file mode 100644 index 00000000..fcf0bd74 --- /dev/null +++ b/tests/functionaltest/rac_tests/empty_timefield/stub.go @@ -0,0 +1,2 @@ +package main +func main(){} diff --git a/tests/functionaltest/rac_tests/empty_timefield2/main_test.go b/tests/functionaltest/rac_tests/empty_timefield2/main_test_disable.go similarity index 99% rename from tests/functionaltest/rac_tests/empty_timefield2/main_test.go rename to tests/functionaltest/rac_tests/empty_timefield2/main_test_disable.go index e5befa1c..69a0f9b0 100644 --- a/tests/functionaltest/rac_tests/empty_timefield2/main_test.go +++ b/tests/functionaltest/rac_tests/empty_timefield2/main_test_disable.go @@ -28,7 +28,6 @@ func cfg() (map[string]string, map[string]string, testutil.WorkerType) { appcfg["bind_port"] = "31002" appcfg["log_level"] = "5" appcfg["log_file"] = "hera.log" - appcfg["sharding_cfg_reload_interval"] = "0" appcfg["rac_sql_interval"] = "2" appcfg["lifespan_check_interval"] = "1" appcfg["child.executable"] = "mysqlworker" diff --git a/tests/functionaltest/rac_tests/empty_timefield2/stub.go b/tests/functionaltest/rac_tests/empty_timefield2/stub.go new file mode 100644 index 00000000..fcf0bd74 --- /dev/null +++ b/tests/functionaltest/rac_tests/empty_timefield2/stub.go @@ -0,0 +1,2 @@ +package main +func main(){} diff --git a/tests/functionaltest/rac_tests/rac_restart_window_R/main_test.go b/tests/functionaltest/rac_tests/rac_restart_window_R/main_test_disable.go similarity index 98% rename from tests/functionaltest/rac_tests/rac_restart_window_R/main_test.go rename to tests/functionaltest/rac_tests/rac_restart_window_R/main_test_disable.go index 7806ccd9..3f12d231 100644 --- a/tests/functionaltest/rac_tests/rac_restart_window_R/main_test.go +++ b/tests/functionaltest/rac_tests/rac_restart_window_R/main_test_disable.go @@ -26,7 +26,6 @@ func cfg() (map[string]string, map[string]string, testutil.WorkerType) { appcfg["bind_port"] = "31002" appcfg["log_level"] = "5" appcfg["log_file"] = "hera.log" - appcfg["sharding_cfg_reload_interval"] = "0" appcfg["rac_sql_interval"] = "10" appcfg["rac_restart_window"] = "120" appcfg["child.executable"] = "mysqlworker" diff --git a/tests/functionaltest/rac_tests/rac_restart_window_R/stub.go b/tests/functionaltest/rac_tests/rac_restart_window_R/stub.go new file mode 100644 index 00000000..fcf0bd74 --- /dev/null +++ b/tests/functionaltest/rac_tests/rac_restart_window_R/stub.go @@ -0,0 +1,2 @@ +package main +func main(){} diff --git a/tests/functionaltest/rac_tests/status_F_Tcurr_GT_Tsch/main_test.go b/tests/functionaltest/rac_tests/status_F_Tcurr_GT_Tsch/main_test_disable.go similarity index 100% rename from tests/functionaltest/rac_tests/status_F_Tcurr_GT_Tsch/main_test.go rename to tests/functionaltest/rac_tests/status_F_Tcurr_GT_Tsch/main_test_disable.go diff --git a/tests/functionaltest/rac_tests/status_F_Tcurr_GT_Tsch/stub.go b/tests/functionaltest/rac_tests/status_F_Tcurr_GT_Tsch/stub.go new file mode 100644 index 00000000..fcf0bd74 --- /dev/null +++ b/tests/functionaltest/rac_tests/status_F_Tcurr_GT_Tsch/stub.go @@ -0,0 +1,2 @@ +package main +func main(){} diff --git a/tests/functionaltest/rac_tests/status_F_dedicated_worker/main_test.go b/tests/functionaltest/rac_tests/status_F_dedicated_worker/main_test_disable.go similarity index 99% rename from tests/functionaltest/rac_tests/status_F_dedicated_worker/main_test.go rename to tests/functionaltest/rac_tests/status_F_dedicated_worker/main_test_disable.go index 3f3a984f..15fb61ce 100644 --- a/tests/functionaltest/rac_tests/status_F_dedicated_worker/main_test.go +++ b/tests/functionaltest/rac_tests/status_F_dedicated_worker/main_test_disable.go @@ -28,7 +28,6 @@ func cfg() (map[string]string, map[string]string, testutil.WorkerType) { appcfg["bind_port"] = "31002" appcfg["log_level"] = "5" appcfg["log_file"] = "hera.log" - appcfg["sharding_cfg_reload_interval"] = "0" appcfg["rac_sql_interval"] = "1" appcfg["lifespan_check_interval"] = "1" appcfg["child.executable"] = "mysqlworker" diff --git a/tests/functionaltest/rac_tests/status_F_dedicated_worker/stub.go b/tests/functionaltest/rac_tests/status_F_dedicated_worker/stub.go new file mode 100644 index 00000000..fcf0bd74 --- /dev/null +++ b/tests/functionaltest/rac_tests/status_F_dedicated_worker/stub.go @@ -0,0 +1,2 @@ +package main +func main(){} diff --git a/tests/functionaltest/rac_tests/status_R_Tcurr_GT_Tsch/main_test.go b/tests/functionaltest/rac_tests/status_R_Tcurr_GT_Tsch/main_test_disable.go similarity index 100% rename from tests/functionaltest/rac_tests/status_R_Tcurr_GT_Tsch/main_test.go rename to tests/functionaltest/rac_tests/status_R_Tcurr_GT_Tsch/main_test_disable.go diff --git a/tests/functionaltest/rac_tests/status_R_Tcurr_GT_Tsch/stub.go b/tests/functionaltest/rac_tests/status_R_Tcurr_GT_Tsch/stub.go new file mode 100644 index 00000000..fcf0bd74 --- /dev/null +++ b/tests/functionaltest/rac_tests/status_R_Tcurr_GT_Tsch/stub.go @@ -0,0 +1,2 @@ +package main +func main(){} diff --git a/tests/functionaltest/rac_tests/status_R_Tcurr_GT_Tsch_and_Tsch_LT_Tstart/main_test.go b/tests/functionaltest/rac_tests/status_R_Tcurr_GT_Tsch_and_Tsch_LT_Tstart/main_test_disable.go similarity index 100% rename from tests/functionaltest/rac_tests/status_R_Tcurr_GT_Tsch_and_Tsch_LT_Tstart/main_test.go rename to tests/functionaltest/rac_tests/status_R_Tcurr_GT_Tsch_and_Tsch_LT_Tstart/main_test_disable.go diff --git a/tests/functionaltest/rac_tests/status_R_Tcurr_GT_Tsch_and_Tsch_LT_Tstart/stub.go b/tests/functionaltest/rac_tests/status_R_Tcurr_GT_Tsch_and_Tsch_LT_Tstart/stub.go new file mode 100644 index 00000000..fcf0bd74 --- /dev/null +++ b/tests/functionaltest/rac_tests/status_R_Tcurr_GT_Tsch_and_Tsch_LT_Tstart/stub.go @@ -0,0 +1,2 @@ +package main +func main(){} diff --git a/tests/functionaltest/rac_tests/status_R_Tcurr_LT_Tsch/main_test.go b/tests/functionaltest/rac_tests/status_R_Tcurr_LT_Tsch/main_test_disable.go similarity index 100% rename from tests/functionaltest/rac_tests/status_R_Tcurr_LT_Tsch/main_test.go rename to tests/functionaltest/rac_tests/status_R_Tcurr_LT_Tsch/main_test_disable.go diff --git a/tests/functionaltest/rac_tests/status_R_Tcurr_LT_Tsch/stub.go b/tests/functionaltest/rac_tests/status_R_Tcurr_LT_Tsch/stub.go new file mode 100644 index 00000000..fcf0bd74 --- /dev/null +++ b/tests/functionaltest/rac_tests/status_R_Tcurr_LT_Tsch/stub.go @@ -0,0 +1,2 @@ +package main +func main(){} diff --git a/tests/functionaltest/rac_tests/status_R_dedicated_worker/main_test.go b/tests/functionaltest/rac_tests/status_R_dedicated_worker/main_test_disable.go similarity index 99% rename from tests/functionaltest/rac_tests/status_R_dedicated_worker/main_test.go rename to tests/functionaltest/rac_tests/status_R_dedicated_worker/main_test_disable.go index e4498ef8..9a985a8f 100644 --- a/tests/functionaltest/rac_tests/status_R_dedicated_worker/main_test.go +++ b/tests/functionaltest/rac_tests/status_R_dedicated_worker/main_test_disable.go @@ -28,7 +28,6 @@ func cfg() (map[string]string, map[string]string, testutil.WorkerType) { appcfg["bind_port"] = "31002" appcfg["log_level"] = "5" appcfg["log_file"] = "hera.log" - appcfg["sharding_cfg_reload_interval"] = "0" appcfg["rac_sql_interval"] = "1" appcfg["rac_restart_window"] = "10" appcfg["lifespan_check_interval"] = "1" diff --git a/tests/functionaltest/rac_tests/status_R_dedicated_worker/stub.go b/tests/functionaltest/rac_tests/status_R_dedicated_worker/stub.go new file mode 100644 index 00000000..fcf0bd74 --- /dev/null +++ b/tests/functionaltest/rac_tests/status_R_dedicated_worker/stub.go @@ -0,0 +1,2 @@ +package main +func main(){} diff --git a/tests/functionaltest/root_transaction_tests/InitDBError/main_test.go b/tests/functionaltest/root_transaction_tests/InitDBError/main_test_disable.go similarity index 98% rename from tests/functionaltest/root_transaction_tests/InitDBError/main_test.go rename to tests/functionaltest/root_transaction_tests/InitDBError/main_test_disable.go index 826d7cda..a83ab587 100644 --- a/tests/functionaltest/root_transaction_tests/InitDBError/main_test.go +++ b/tests/functionaltest/root_transaction_tests/InitDBError/main_test_disable.go @@ -20,7 +20,6 @@ func cfg() (map[string]string, map[string]string, testutil.WorkerType) { appcfg["bind_port"] = "31002" appcfg["log_level"] = "5" appcfg["log_file"] = "hera.log" - appcfg["sharding_cfg_reload_interval"] = "0" appcfg["rac_sql_interval"] = "2" appcfg["db_heartbeat_interval"] = "3" diff --git a/tests/functionaltest/root_transaction_tests/InitDBError/stub.go b/tests/functionaltest/root_transaction_tests/InitDBError/stub.go new file mode 100644 index 00000000..fcf0bd74 --- /dev/null +++ b/tests/functionaltest/root_transaction_tests/InitDBError/stub.go @@ -0,0 +1,2 @@ +package main +func main(){} diff --git a/tests/functionaltest/runall.sh b/tests/functionaltest/runall.sh index cbdddc00..1e0f2770 100755 --- a/tests/functionaltest/runall.sh +++ b/tests/functionaltest/runall.sh @@ -6,26 +6,56 @@ then ln -s $PWD testrun/src/github.com/paypal/hera fi $GOROOT/bin/go install github.com/paypal/hera/worker/mysqlworker -suites="bind_eviction_tests strandedchild_tests coordinator_tests saturation_tests adaptive_queue_tests rac_tests sharding_tests" +$GOROOT/bin/go install github.com/paypal/hera/watchdog +$GOROOT/bin/go install github.com/paypal/hera/mux +ls -l $GOPATH/bin + +cat << EOF > shortRun +sharding_tests/set_shard_id +sharding_whitelist_tests/no_shard_no_error +sharding_whitelist_tests/set_shard_id_wl +bind_eviction_tests/bind_eviction_disable +EOF + +basedir=$GOPATH/src/github.com/paypal/hera/tests/functionaltest/ +find $basedir -name main_test.go | sed -e "s,^$basedir,,;s,/main_test.go,," > toRun + +# suites="bind_eviction_tests strandedchild_tests coordinator_tests saturation_tests adaptive_queue_tests rac_tests sharding_tests" +# ls -F $GOPATH/src/github.com/paypal/hera/tests/functionaltest/$suite | grep /$ | sed -e 's,/,,' | egrep -v '(testutil| +# no_shard_no_error|set_shard_id_wl|reset_shard_id_wl)' | sed -e "s,^,$suite/," >> toRun + finalResult=0 -for suite in $suites +for pathD in `cat toRun` # shortRun do - for d in `ls -F $GOPATH/src/github.com/paypal/hera/tests/functionaltest/$suite | grep /$ | sed -e 's,/,,' | egrep -v '(testutil|no_shard_no_error|set_shard_id_wl|reset_shard_id_wl)'` - do - pushd $GOPATH/src/github.com/paypal/hera/tests/functionaltest/$suite/$d - cp $GOPATH/bin/mysqlworker . - $GOROOT/bin/go test -c . - ./$d.test -test.v - rv=$? - if [ 0 != $rv ] - then - echo failing $suite $d - grep ^ *.log - finalResult=$rv -# exit $rv - fi - popd - sleep 10 - done + pushd $GOPATH/src/github.com/paypal/hera/tests/functionaltest/$pathD + d=`basename $pathD` + ln $GOPATH/bin/mysqlworker . + + $GOROOT/bin/go test -c . + + retry=2 + while [ $retry -ge 0 ] + do + ./$d.test -test.v 2>&1 | tee std.log + pkill watchdog + pkill mux + pkill mysqlworker + if grep -q '^--- PASS:' std.log + then + break + else + echo failing $pathD with $retry retries left + sleep 11.1 + mv std.log{,$retry} + + if [ $retry -eq 0 ] + then + tail -n111 *.log + finalResult=1 + fi + fi + retry=$(($retry-1)) + done + popd done exit $finalResult diff --git a/tests/functionaltest/saturation_tests/backlog_nosaturation/main_test.go b/tests/functionaltest/saturation_tests/backlog_nosaturation/main_test_disable.go similarity index 100% rename from tests/functionaltest/saturation_tests/backlog_nosaturation/main_test.go rename to tests/functionaltest/saturation_tests/backlog_nosaturation/main_test_disable.go diff --git a/tests/functionaltest/saturation_tests/backlog_nosaturation/stub.go b/tests/functionaltest/saturation_tests/backlog_nosaturation/stub.go new file mode 100644 index 00000000..fcf0bd74 --- /dev/null +++ b/tests/functionaltest/saturation_tests/backlog_nosaturation/stub.go @@ -0,0 +1,2 @@ +package main +func main(){} diff --git a/tests/functionaltest/saturation_tests/change_throttle_rate_runtime/main_test.go b/tests/functionaltest/saturation_tests/change_throttle_rate_runtime/main_test_disable.go similarity index 100% rename from tests/functionaltest/saturation_tests/change_throttle_rate_runtime/main_test.go rename to tests/functionaltest/saturation_tests/change_throttle_rate_runtime/main_test_disable.go diff --git a/tests/functionaltest/saturation_tests/change_throttle_rate_runtime/stub.go b/tests/functionaltest/saturation_tests/change_throttle_rate_runtime/stub.go new file mode 100644 index 00000000..fcf0bd74 --- /dev/null +++ b/tests/functionaltest/saturation_tests/change_throttle_rate_runtime/stub.go @@ -0,0 +1,2 @@ +package main +func main(){} diff --git a/tests/functionaltest/saturation_tests/multi_saturation_zero_threshold/main_test.go b/tests/functionaltest/saturation_tests/multi_saturation_zero_threshold/main_test_disable.go similarity index 100% rename from tests/functionaltest/saturation_tests/multi_saturation_zero_threshold/main_test.go rename to tests/functionaltest/saturation_tests/multi_saturation_zero_threshold/main_test_disable.go diff --git a/tests/functionaltest/saturation_tests/multi_saturation_zero_threshold/stub.go b/tests/functionaltest/saturation_tests/multi_saturation_zero_threshold/stub.go new file mode 100644 index 00000000..fcf0bd74 --- /dev/null +++ b/tests/functionaltest/saturation_tests/multi_saturation_zero_threshold/stub.go @@ -0,0 +1,2 @@ +package main +func main(){} diff --git a/tests/functionaltest/saturation_tests/no_saturation_entry/main_test.go b/tests/functionaltest/saturation_tests/no_saturation_entry/main_test.go index ee03aad7..5767b490 100644 --- a/tests/functionaltest/saturation_tests/no_saturation_entry/main_test.go +++ b/tests/functionaltest/saturation_tests/no_saturation_entry/main_test.go @@ -66,7 +66,7 @@ func TestMain(m *testing.M) { */ -func NoSaturationEntry(t *testing.T) { +func TestNoSaturationEntry(t *testing.T) { fmt.Println ("NoSaturationEntry begin +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++") logger.GetLogger().Log(logger.Debug, "NoSaturationEntry begin +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n") diff --git a/tests/functionaltest/saturation_tests/saturation_enabled_runtime/main_test.go b/tests/functionaltest/saturation_tests/saturation_enabled_runtime/main_test_disable.go similarity index 100% rename from tests/functionaltest/saturation_tests/saturation_enabled_runtime/main_test.go rename to tests/functionaltest/saturation_tests/saturation_enabled_runtime/main_test_disable.go diff --git a/tests/functionaltest/saturation_tests/saturation_enabled_runtime/stub.go b/tests/functionaltest/saturation_tests/saturation_enabled_runtime/stub.go new file mode 100644 index 00000000..fcf0bd74 --- /dev/null +++ b/tests/functionaltest/saturation_tests/saturation_enabled_runtime/stub.go @@ -0,0 +1,2 @@ +package main +func main(){} diff --git a/tests/functionaltest/saturation_tests/saturation_recover/main_test.go b/tests/functionaltest/saturation_tests/saturation_recover/main_test_disable.go similarity index 100% rename from tests/functionaltest/saturation_tests/saturation_recover/main_test.go rename to tests/functionaltest/saturation_tests/saturation_recover/main_test_disable.go diff --git a/tests/functionaltest/saturation_tests/saturation_recover/stub.go b/tests/functionaltest/saturation_tests/saturation_recover/stub.go new file mode 100644 index 00000000..fcf0bd74 --- /dev/null +++ b/tests/functionaltest/saturation_tests/saturation_recover/stub.go @@ -0,0 +1,2 @@ +package main +func main(){} diff --git a/tests/functionaltest/saturation_tests/saturation_recycle/main_test.go b/tests/functionaltest/saturation_tests/saturation_recycle/main_test_disable.go similarity index 100% rename from tests/functionaltest/saturation_tests/saturation_recycle/main_test.go rename to tests/functionaltest/saturation_tests/saturation_recycle/main_test_disable.go diff --git a/tests/functionaltest/saturation_tests/saturation_recycle/stub.go b/tests/functionaltest/saturation_tests/saturation_recycle/stub.go new file mode 100644 index 00000000..fcf0bd74 --- /dev/null +++ b/tests/functionaltest/saturation_tests/saturation_recycle/stub.go @@ -0,0 +1,2 @@ +package main +func main(){} diff --git a/tests/functionaltest/saturation_tests/write_worker_saturation/main_test.go b/tests/functionaltest/saturation_tests/write_worker_saturation/main_test_disable.go similarity index 100% rename from tests/functionaltest/saturation_tests/write_worker_saturation/main_test.go rename to tests/functionaltest/saturation_tests/write_worker_saturation/main_test_disable.go diff --git a/tests/functionaltest/saturation_tests/write_worker_saturation/stub.go b/tests/functionaltest/saturation_tests/write_worker_saturation/stub.go new file mode 100644 index 00000000..fcf0bd74 --- /dev/null +++ b/tests/functionaltest/saturation_tests/write_worker_saturation/stub.go @@ -0,0 +1,2 @@ +package main +func main(){} diff --git a/tests/functionaltest/sharding_tests/reset_shard_id/main_test.go b/tests/functionaltest/sharding_tests/reset_shard_id/main_test.go index 62882a1c..b160f9ea 100644 --- a/tests/functionaltest/sharding_tests/reset_shard_id/main_test.go +++ b/tests/functionaltest/sharding_tests/reset_shard_id/main_test.go @@ -23,7 +23,6 @@ func cfg() (map[string]string, map[string]string, testutil.WorkerType) { appcfg["bind_port"] = "31002" appcfg["log_level"] = "5" appcfg["log_file"] = "hera.log" - appcfg["sharding_cfg_reload_interval"] = "0" appcfg["rac_sql_interval"] = "0" //For sharding diff --git a/tests/functionaltest/sharding_tests/set_shard_id/main_test.go b/tests/functionaltest/sharding_tests/set_shard_id/main_test.go index 82d5f36a..c63b4f0e 100644 --- a/tests/functionaltest/sharding_tests/set_shard_id/main_test.go +++ b/tests/functionaltest/sharding_tests/set_shard_id/main_test.go @@ -23,7 +23,6 @@ func cfg() (map[string]string, map[string]string, testutil.WorkerType) { appcfg["bind_port"] = "31002" appcfg["log_level"] = "5" appcfg["log_file"] = "hera.log" - appcfg["sharding_cfg_reload_interval"] = "0" appcfg["rac_sql_interval"] = "0" //For sharding diff --git a/tests/functionaltest/sharding_tests/shard_basic/main_test.go b/tests/functionaltest/sharding_tests/shard_basic/main_test.go index 52c82b6d..a85fd48e 100644 --- a/tests/functionaltest/sharding_tests/shard_basic/main_test.go +++ b/tests/functionaltest/sharding_tests/shard_basic/main_test.go @@ -24,7 +24,6 @@ func cfg() (map[string]string, map[string]string, testutil.WorkerType) { appcfg["bind_port"] = "31002" appcfg["log_level"] = "5" appcfg["log_file"] = "hera.log" - appcfg["sharding_cfg_reload_interval"] = "0" appcfg["rac_sql_interval"] = "0" //For sharding @@ -144,11 +143,13 @@ func TestShardBasic(t *testing.T) { t.Fatalf ("Error: No Shard key event for fetch request in CAL"); } - fmt.Println ("Check CAL log for correct events"); - cal_count = testutil.RegexCountFile ("SHARDING.*shard_key_auto_discovery.*0.*shardkey=accountid|12346&shardid=3&scuttleid=", "cal.log") - if (cal_count < 1) { - t.Fatalf ("Error: No shard_key_auto_discovery event seen in CAL"); + fmt.Println ("Check log for shard key auto discovery"); + count = testutil.RegexCount ("shard key auto discovery: shardkey=accountid|12346&shardid=3&scuttleid=") + if (count < 1) { + t.Fatalf ("Error: Did NOT get shard key auto discovery in log"); } + + fmt.Println ("Check CAL log for correct events") cal_count = testutil.RegexCountFile ("T.*API.*CLIENT_SESSION_3", "cal.log") if (cal_count < 1) { t.Fatalf ("Error: Request is not executed by shard 3 as expected"); diff --git a/tests/functionaltest/sharding_whitelist_tests/no_shard_no_error/main_test.go b/tests/functionaltest/sharding_whitelist_tests/no_shard_no_error/main_test.go index b41dae23..cc60e280 100644 --- a/tests/functionaltest/sharding_whitelist_tests/no_shard_no_error/main_test.go +++ b/tests/functionaltest/sharding_whitelist_tests/no_shard_no_error/main_test.go @@ -21,7 +21,6 @@ func cfg() (map[string]string, map[string]string, testutil.WorkerType) { appcfg["bind_port"] = "31002" appcfg["log_level"] = "5" appcfg["log_file"] = "hera.log" - appcfg["sharding_cfg_reload_interval"] = "0" appcfg["rac_sql_interval"] = "0" //For sharding @@ -126,24 +125,14 @@ func TestNoShardNoError(t *testing.T) { time.Sleep (time.Duration(2) * time.Second) fmt.Println ("Verify insert request is sent to shard 2") count := testutil.RegexCount ("WORKER shd2.*Preparing.*insert into test_simple_table_2") - if (count < 1) { - t.Fatalf ("Error: Insert query does NOT go to shd2"); + if (count >= 1) { + t.Fatalf ("Error: Insert query went to shd2"); } fmt.Println ("Verify there is no shard key error for fetch request") count = testutil.RegexCount ("Error preprocessing sharding, hangup: OCC-373: no shard key or more than one or bad logical db false") if (count > 0) { - t.Fatalf ("Error: should NOT get no shard key error"); - } - fmt.Println ("Check CAL log for correct events"); - cal_count := testutil.RegexCountFile ("SHARDING.*shard_key_auto_discovery.*0.*shardkey=accountid|12345&shardid=3&scuttleid=428", "cal.log") - if (cal_count < 1) { - t.Fatalf ("Error: Did NOT get shard_key_auto_discovery in CAL log"); - } - fmt.Println ("Verify Insert and Select event are sent to shard 3 "); - cal_count = testutil.RegexCountFile ("T.*API.*CLIENT_SESSION_3", "cal.log") - if (cal_count < 2) { - t.Fatalf ("Error: Did NOT get API.*CLIENT_SESSION_3 in CAL log"); + t.Fatalf ("Error: should NOT get no shard key error, default sh0"); } testutil.DoDefaultValidation(t) logger.GetLogger().Log(logger.Debug, "TestNoShardNoError done -------------------------------------------------------------") diff --git a/tests/functionaltest/sharding_whitelist_tests/reset_shard_id_wl/main_test.go b/tests/functionaltest/sharding_whitelist_tests/reset_shard_id_wl/main_test.go index 6fa3ec38..182276c9 100644 --- a/tests/functionaltest/sharding_whitelist_tests/reset_shard_id_wl/main_test.go +++ b/tests/functionaltest/sharding_whitelist_tests/reset_shard_id_wl/main_test.go @@ -21,7 +21,6 @@ func cfg() (map[string]string, map[string]string, testutil.WorkerType) { appcfg["bind_port"] = "31002" appcfg["log_level"] = "5" appcfg["log_file"] = "hera.log" - appcfg["sharding_cfg_reload_interval"] = "0" appcfg["rac_sql_interval"] = "0" //For sharding diff --git a/tests/functionaltest/sharding_whitelist_tests/set_shard_id_wl/main_test.go b/tests/functionaltest/sharding_whitelist_tests/set_shard_id_wl/main_test.go index 56d78ddc..d2699c38 100644 --- a/tests/functionaltest/sharding_whitelist_tests/set_shard_id_wl/main_test.go +++ b/tests/functionaltest/sharding_whitelist_tests/set_shard_id_wl/main_test.go @@ -21,7 +21,6 @@ func cfg() (map[string]string, map[string]string, testutil.WorkerType) { appcfg["bind_port"] = "31002" appcfg["log_level"] = "5" appcfg["log_file"] = "hera.log" - appcfg["sharding_cfg_reload_interval"] = "0" appcfg["rac_sql_interval"] = "0" //For sharding diff --git a/tests/functionaltest/strandedchild_tests/busyworkercrashes/main_test.go b/tests/functionaltest/strandedchild_tests/busyworkercrashes/main_test.go index f6dabfca..f9df4e01 100644 --- a/tests/functionaltest/strandedchild_tests/busyworkercrashes/main_test.go +++ b/tests/functionaltest/strandedchild_tests/busyworkercrashes/main_test.go @@ -29,7 +29,6 @@ func cfg() (map[string]string, map[string]string, testutil.WorkerType) { appcfg["bind_port"] = "31002" appcfg["log_level"] = "5" appcfg["log_file"] = "hera.log" - appcfg["sharding_cfg_reload_interval"] = "0" appcfg["rac_sql_interval"] = "0" appcfg["child.executable"] = "mysqlworker" appcfg["database_type"] = "mysql" diff --git a/tests/functionaltest/strandedchild_tests/dmldisconnect/main_test.go b/tests/functionaltest/strandedchild_tests/dmldisconnect/main_test.go index e3823fd9..e261b35a 100644 --- a/tests/functionaltest/strandedchild_tests/dmldisconnect/main_test.go +++ b/tests/functionaltest/strandedchild_tests/dmldisconnect/main_test.go @@ -28,7 +28,6 @@ func cfg() (map[string]string, map[string]string, testutil.WorkerType) { appcfg["bind_port"] = "31002" appcfg["log_level"] = "5" appcfg["log_file"] = "hera.log" - appcfg["sharding_cfg_reload_interval"] = "0" appcfg["rac_sql_interval"] = "0" appcfg["opscfg.default.server.idle_timeout_ms"] = "3000" appcfg["opscfg.default.server.transaction_idle_timeout_ms"] = "5000" diff --git a/tests/functionaltest/strandedchild_tests/dmlidle/main_test.go b/tests/functionaltest/strandedchild_tests/dmlidle/main_test.go index 155e6682..4369adb9 100644 --- a/tests/functionaltest/strandedchild_tests/dmlidle/main_test.go +++ b/tests/functionaltest/strandedchild_tests/dmlidle/main_test.go @@ -30,7 +30,6 @@ func cfg() (map[string]string, map[string]string, testutil.WorkerType) { appcfg["bind_port"] = "31002" appcfg["log_level"] = "5" appcfg["log_file"] = "hera.log" - appcfg["sharding_cfg_reload_interval"] = "0" appcfg["rac_sql_interval"] = "0" appcfg["opscfg.default.server.idle_timeout_ms"] = "5000" appcfg["child.executable"] = "mysqlworker" diff --git a/tests/functionaltest/strandedchild_tests/freeworkercrashes/main_test.go b/tests/functionaltest/strandedchild_tests/freeworkercrashes/main_test.go index 338549f6..350d1d1c 100644 --- a/tests/functionaltest/strandedchild_tests/freeworkercrashes/main_test.go +++ b/tests/functionaltest/strandedchild_tests/freeworkercrashes/main_test.go @@ -29,7 +29,6 @@ func cfg() (map[string]string, map[string]string, testutil.WorkerType) { appcfg["bind_port"] = "31002" appcfg["log_level"] = "5" appcfg["log_file"] = "hera.log" - appcfg["sharding_cfg_reload_interval"] = "0" appcfg["rac_sql_interval"] = "0" //appcfg["child.executable"] = "mysqlworker" appcfg["database_type"] = "mysql" diff --git a/tests/functionaltest/strandedchild_tests/no_free_worker3/main_test.go b/tests/functionaltest/strandedchild_tests/no_free_worker3/main_test_disable.go similarity index 100% rename from tests/functionaltest/strandedchild_tests/no_free_worker3/main_test.go rename to tests/functionaltest/strandedchild_tests/no_free_worker3/main_test_disable.go diff --git a/tests/functionaltest/strandedchild_tests/no_free_worker3/stub.go b/tests/functionaltest/strandedchild_tests/no_free_worker3/stub.go new file mode 100644 index 00000000..fcf0bd74 --- /dev/null +++ b/tests/functionaltest/strandedchild_tests/no_free_worker3/stub.go @@ -0,0 +1,2 @@ +package main +func main(){} diff --git a/tests/functionaltest/strandedchild_tests/nondmlidle/main_test.go b/tests/functionaltest/strandedchild_tests/nondmlidle/main_test.go index d409b5a3..e31d17ad 100644 --- a/tests/functionaltest/strandedchild_tests/nondmlidle/main_test.go +++ b/tests/functionaltest/strandedchild_tests/nondmlidle/main_test.go @@ -30,7 +30,6 @@ func cfg() (map[string]string, map[string]string, testutil.WorkerType) { appcfg["bind_port"] = "31002" appcfg["log_level"] = "5" appcfg["log_file"] = "hera.log" - appcfg["sharding_cfg_reload_interval"] = "0" appcfg["rac_sql_interval"] = "0" appcfg["opscfg.default.server.idle_timeout_ms"] = "5000" appcfg["child.executable"] = "mysqlworker" diff --git a/tests/functionaltest/strandedchild_tests/transaction_idle_timeout/main_test.go b/tests/functionaltest/strandedchild_tests/transaction_idle_timeout/main_test.go index 1677c464..6d3648a8 100644 --- a/tests/functionaltest/strandedchild_tests/transaction_idle_timeout/main_test.go +++ b/tests/functionaltest/strandedchild_tests/transaction_idle_timeout/main_test.go @@ -28,7 +28,6 @@ func cfg() (map[string]string, map[string]string, testutil.WorkerType) { appcfg["bind_port"] = "31002" appcfg["log_level"] = "5" appcfg["log_file"] = "hera.log" - appcfg["sharding_cfg_reload_interval"] = "0" appcfg["rac_sql_interval"] = "0" appcfg["opscfg.default.server.idle_timeout_ms"] = "3000" appcfg["opscfg.default.server.transaction_idle_timeout_ms"] = "5000" diff --git a/tests/functionaltest/testutil/setup.go b/tests/functionaltest/testutil/setup.go index 2102167b..cd617267 100755 --- a/tests/functionaltest/testutil/setup.go +++ b/tests/functionaltest/testutil/setup.go @@ -103,6 +103,7 @@ func (m *mux) setupConfig() error { } else if m.wType == PostgresWorker { m.appcfg["child.executable"] = "postgresworker" } + m.appcfg["random_start_ms"] = "33" err := createCfg(m.appcfg, "hera") if err != nil { return err @@ -199,8 +200,8 @@ func (m *mux) cleanupConfig() error { func MakeDB(dockerName string, dbName string, dbType DBType) (ip string) { CleanDB(dockerName) if dbType == MySQL { - //Commented out temporarily so we don't have to run docker all the time - cmd := exec.Command("docker", "run", "--name", dockerName, "-e", "MYSQL_ROOT_PASSWORD=1-testDb", "-e", "MYSQL_DATABASE="+dbName, "-d", "mysql:latest") + // mac must use port forward + cmd := exec.Command("docker", "run", "--name", dockerName, "-p3306:3306", "-e", "MYSQL_ROOT_PASSWORD=1-testDb", "-e", "MYSQL_DATABASE="+dbName, "-d", "mysql:latest") cmd.Run() // find its IP @@ -322,6 +323,7 @@ func DBDirect(query string, ip string, dbName string, dbType DBType) error { db0.SetMaxIdleConns(0) // defer db0.Close() dbs[ip+dbName] = db0 + dbs["127.0.0.1"+dbName] = db0 } } else if dbType == PostgreSQL { if !ok { @@ -393,6 +395,9 @@ func (m *mux) StartServer() error { os.Setenv("TWO_TASK_4", "tcp(127.0.0.1:2121)/heratestdb") } else if xMysql == "auto" { ip := MakeDB("mysql22", "heratestdb", MySQL) + if os.Getenv("SHELL") == "/bin/zsh" { + ip = "127.0.0.1" // for mac + } os.Setenv("TWO_TASK", "tcp("+ip+":3306)/heratestdb") os.Setenv("TWO_TASK_1", "tcp("+ip+":3306)/heratestdb") os.Setenv("TWO_TASK_2", "tcp("+ip+":3306)/heratestdb") diff --git a/tests/functionaltest/testutil/util.go b/tests/functionaltest/testutil/util.go index 02ac321c..bf2fabb9 100755 --- a/tests/functionaltest/testutil/util.go +++ b/tests/functionaltest/testutil/util.go @@ -15,7 +15,8 @@ import ( "strings" "testing" "github.com/paypal/hera/client/gosqldriver" - _"github.com/paypal/hera/client/gosqldriver/tcp" + _ "github.com/paypal/hera/client/gosqldriver/tcp" + _ "github.com/paypal/hera/lib" "github.com/paypal/hera/utility/logger" ) @@ -138,114 +139,22 @@ func RunMysql(sql string) (string, error) { func RunDML(dml string) error { - db, err := sql.Open("heraloop", fmt.Sprintf("%d:0:0", 0)) - if err != nil { - return err - } - db.SetMaxIdleConns(0) - defer db.Close() - - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - // cancel must be called before conn.Close() - defer cancel() - conn, err := db.Conn(ctx) - if err != nil { - return err - } - defer conn.Close() - mux := gosqldriver.InnerConn(conn) - err= mux.SetClientInfo(GetClientInfo().Appname, GetClientInfo().Host) - if err != nil { - fmt.Println("Error sending Client Info:", err) - } - tx, _ := conn.BeginTx(ctx, nil) - stmt, _ := tx.PrepareContext(ctx, dml) - defer stmt.Close() - _, err = stmt.Exec() - if err != nil { - return err - } - err = tx.Commit() - if err != nil { - return err - } - + DBDirect(dml, "127.0.0.1" /* os.Getenv("MYSQL_IP") */, "heratestdb", MySQL) return nil } func RunDML1(dml string) error { - hostname := GetHostname() - fmt.Println ("Hostname: ", hostname); - db, err := sql.Open("hera", hostname + ":31002") - if err != nil { - fmt.Println("Error connecting to OCC:", err) - return err - } - db.SetMaxIdleConns(0) - defer db.Close() - - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - // cleanup and insert one row in the table - conn, err := db.Conn(ctx) - if err != nil { - return err - } - defer conn.Close() - // cancel must be called before conn.Close() - defer cancel() - mux := gosqldriver.InnerConn(conn) - err= mux.SetClientInfo(GetClientInfo().Appname, GetClientInfo().Host) - if err != nil { - fmt.Println("Error sending Client Info:", err) - } - tx, _ := conn.BeginTx(ctx, nil) - stmt, _ := tx.PrepareContext(ctx, dml) - defer stmt.Close() - _, err = stmt.Exec() - if err != nil { - return err - } - err = tx.Commit() - if err != nil { - return err - } - - return nil + return RunDML(dml) } func PopulateShardMap(max_scuttle int) error { - db, err := sql.Open("heraloop", fmt.Sprintf("%d:0:0", 0)) - if err != nil { - return err - } - db.SetMaxIdleConns(0) - defer db.Close() - - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - // cancel must be called before conn.Close() - defer cancel() - // cleanup and insert one row in the table - conn, err := db.Conn(ctx) - if err != nil { - return err - } - defer conn.Close() - tx, _ := conn.BeginTx(ctx, nil) for x := 0; x < max_scuttle; x++ { dml := fmt.Sprint ("INSERT INTO hera_shard_map VALUES (", x, ", ", x % 5, ",'Y','Y','Y','Initial')") - stmt, _ := tx.PrepareContext(ctx, dml) - defer stmt.Close() - _, err = stmt.Exec() - } - if err != nil { - return err - } - err = tx.Commit() - if err != nil { + err := RunDML(dml) + if err != nil { return err - } - - fmt.Println ("=== Finished loading shard map ===") + } + } return nil } @@ -260,37 +169,13 @@ func PopulateWhilelistShardMap() error { query[6] = "INSERT INTO hera_whitelist ( enable, shard_key, shard_id, read_status, write_status ) VALUES ( 'Y', 444, 4, 'Y', 'Y' )" query[7] = "INSERT INTO hera_whitelist ( enable, shard_key, shard_id, read_status, write_status ) VALUES ( 'Y', 555, 5, 'Y', 'Y' )" query[8] = "INSERT INTO hera_whitelist ( enable, shard_key, shard_id, read_status, write_status ) VALUES ( 'Y', 1234, 4, 'Y', 'Y' )" - db, err := sql.Open("heraloop", fmt.Sprintf("%d:0:0", 0)) - if err != nil { - return err - } - db.SetMaxIdleConns(0) - defer db.Close() - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - // cancel must be called before conn.Close() - defer cancel() - // cleanup and insert one row in the table - conn, err := db.Conn(ctx) - if err != nil { - return err - } - defer conn.Close() - tx, _ := conn.BeginTx(ctx, nil) for x := 0; x < len (query) ; x++ { - stmt, _ := tx.PrepareContext(ctx, query[x]) - defer stmt.Close() - _, err = stmt.Exec() - } - if err != nil { - return err - } - err = tx.Commit() - if err != nil { + err := RunDML(query[x]) + if err != nil { return err + } } - - fmt.Println ("***Done loading shard map") return nil } @@ -606,10 +491,5 @@ func ModifyOpscfgParam (t *testing.T, logfile string, opscfg_param string, opscf } func GetHostname() string { - hostname,err := os.Hostname() - - if err != nil { - hostname = "127.0.0.1" - } - return hostname -} \ No newline at end of file + return "127.0.0.1" +} diff --git a/tests/unittest/allcover.sh b/tests/unittest/allcover.sh new file mode 100755 index 00000000..74020f55 --- /dev/null +++ b/tests/unittest/allcover.sh @@ -0,0 +1,48 @@ +tarball=go1.20.7.linux-amd64.tar.gz +wget -nv https://go.dev/dl/$tarball +echo f0a87f1bcae91c4b69f8dc2bc6d7e6bfcd7524fceec130af525058c0c17b1b44 $tarball | sha256sum -c +if [ x$? != x0 ] +then + echo bad sha256sum + exit 1 +fi +tar zxf $tarball + +export GOROOT=$PWD/go +export GOPATH=/home/runner/go +mkdir -p $GOPATH/src/github.com/paypal +ln -s $PWD $GOPATH/src/github.com/paypal/hera + +rm -rf $GOPATH/allcover{,2} +mkdir $GOPATH/allcover + +$GOROOT/bin/go install -cover github.com/paypal/hera/worker/mysqlworker + +overall=0 +for d in `ls -F tests/unittest | grep /$ | sed -e "s,/,," | egrep -v '(mysql_recycle|log_checker_initdb|testutil|rac_maint|mysql_direct|failover)'` +do + echo ==== $d + cd tests/unittest/$d + cp $GOPATH/bin/mysqlworker . + rm -f *.log + + $GOROOT/bin/go run ../testutil/regen rewrite tests/unittest/$d + $GOROOT/bin/go build -cover github.com/paypal/hera/tests/unittest/$d + mkdir integcov + GOCOVERDIR=integcov ./$d + rv=$? + echo rv rv $rv for test $d under integration coverage run + $GOROOT/bin/go tool covdata percent -i=integcov + mkdir $GOPATH/allcover2 + $GOROOT/bin/go tool covdata merge -i=integcov,$GOPATH/allcover -o $GOPATH/allcover2 + rm -rf $GOPATH/allcover + mv $GOPATH/allcover{2,} + + rm -f *.log + cd ../../.. +done +$GOROOT/bin/go tool covdata func -i=$GOPATH/allcover +$GOROOT/bin/go tool covdata percent -i=$GOPATH/allcover +$GOROOT/bin/go tool covdata textfmt -i=$GOPATH/allcover -o $GOPATH/allcover.out +$GOROOT/bin/go tool cover -html=$GOPATH/allcover.out -o $GOPATH/allcover.htm +exit $overall diff --git a/tests/unittest/bindEvict/main_test.go b/tests/unittest/bindEvict/main_test.go new file mode 100644 index 00000000..3b8b87e0 --- /dev/null +++ b/tests/unittest/bindEvict/main_test.go @@ -0,0 +1,220 @@ +package main + +import ( + "context" + "database/sql" + "fmt" + "os" + "testing" + "time" + + //"github.com/paypal/hera/client/gosqldriver" + _ "github.com/paypal/hera/client/gosqldriver/tcp" /*to register the driver*/ + + "github.com/paypal/hera/lib" + "github.com/paypal/hera/tests/unittest/testutil" + "github.com/paypal/hera/utility/logger" +) + +var mx testutil.Mux +var tableName string +var max_conn float64 + +func cfg() (map[string]string, map[string]string, testutil.WorkerType) { + + appcfg := make(map[string]string) + // best to chose an "unique" port in case golang runs tests in paralel + appcfg["bind_port"] = "31002" + appcfg["log_level"] = "5" + appcfg["log_file"] = "hera.log" + appcfg["sharding_cfg_reload_interval"] = "0" + appcfg["rac_sql_interval"] = "0" + appcfg["child.executable"] = "mysqlworker" + appcfg["bind_eviction_names"] = "p" + appcfg["bind_eviction_threshold_pct"] = "50" + + appcfg["request_backlog_timeout"] = "1000" + appcfg["soft_eviction_probability"] = "100" + + opscfg := make(map[string]string) + max_conn = 25 + opscfg["opscfg.default.server.max_connections"] = fmt.Sprintf("%d", int(max_conn)) + opscfg["opscfg.default.server.log_level"] = "5" + + opscfg["opscfg.default.server.saturation_recover_threshold"] = "10" + //opscfg["opscfg.default.server.saturation_recover_throttle_rate"]= "100" + opscfg["opscfg.hera.server.saturation_recover_throttle_rate"] = "100" + // saturation_recover_throttle_rate + + return appcfg, opscfg, testutil.MySQLWorker +} + +func before() error { + fmt.Printf("before run mysql") + testutil.RunMysql("create table sleep_info (id bigint, seconds float);") + testutil.RunMysql("insert into sleep_info (id,seconds) values(10, 0.01);") + testutil.RunMysql("insert into sleep_info (id,seconds) values(100, 0.1);") + testutil.RunMysql("insert into sleep_info (id,seconds) values(1600, 2.6);") + testutil.RunMysql("insert into sleep_info (id,seconds) values(21001111, 0.1);") + testutil.RunMysql("insert into sleep_info (id,seconds) values(22001111, 0.1);") + testutil.RunMysql("insert into sleep_info (id,seconds) values(29001111, 2.9);") + out, err := testutil.RunMysql(`DELIMITER $$ +CREATE FUNCTION sleep_option (id bigint) +RETURNS float +DETERMINISTIC +BEGIN + declare dur float; + declare rv bigint; + select max(seconds) into dur from sleep_info where sleep_info.id=id; + select sleep(dur) into rv; + RETURN dur; +END$$ +DELIMITER ;`) + if err != nil { + fmt.Printf("err after run mysql " + err.Error()) + return nil + } + fmt.Printf("after run mysql " + out) // */ + return nil +} + +func TestMain(m *testing.M) { + logger.GetLogger().Log(logger.Debug, "begin 20230918kkang TestMain") + fmt.Printf("TestMain 20230918kkang\n") + os.Exit(testutil.UtilMain(m, cfg, before)) +} + +func sleepyQ(conn *sql.Conn, delayRow int) error { + stmt, err := conn.PrepareContext(context.Background(), "select * from sleep_info where ( seconds > sleep_option(?) or seconds > 0.0 )") + if err != nil { + fmt.Printf("Error preparing sleepyQ %s\n", err.Error()) + return err + } + defer stmt.Close() + rows, err := stmt.Query(delayRow) + if err != nil { + fmt.Printf("Error query sleepyQ %s\n", err.Error()) + return err + } + defer rows.Close() + return nil +} + +func fastAndSlowBinds() error { + db, err := sql.Open("hera", "127.0.0.1:31002") + if err != nil { + fmt.Printf("Error db %s\n", err.Error()) + return err + } + db.SetConnMaxLifetime(22 * time.Second) + db.SetMaxIdleConns(0) + db.SetMaxOpenConns(22111) + defer db.Close() + + // client threads of slow queries + var stop2 int + var badCliErr string + mkClients(1+int(max_conn*1.6), &stop2, 29001111, "badClient", &badCliErr, db) + time.Sleep(3100 * time.Millisecond) + /* if (testutil.RegexCountFile("BIND_THROTTLE", "cal.log") == 0) { + return fmt.Errorf("BIND_THROTTLE was not triggered") + } + if (testutil.RegexCountFile("BIND_EVICT", "cal.log") == 0) { + return fmt.Errorf("BIND_EVICT was not triggered") + } // */ + + // start normal clients after initial backlog timeouts + var normCliErrStr string + var stop int + mkClients(1, &stop, 21001111, "n client", &normCliErrStr, db) + time.Sleep(1100 * time.Millisecond) + + // if we throttle down or stop, it restores + stop2 = 1 // stop bad clients + lib.GetConfig().BindEvictionDecrPerSec = 11500.1 + defer func() { lib.GetConfig().BindEvictionDecrPerSec = 1.1 }() + time.Sleep(1 * time.Second) + conn, err := db.Conn(context.Background()) + if err != nil { + fmt.Printf("Error conn %s\n", err.Error()) + return err + } + defer conn.Close() + err = sleepyQ(conn, 29001111) + if err != nil { + msg := fmt.Sprintf("test failed, throttle down didn't restore") + fmt.Printf("%s", msg) + return fmt.Errorf("%s", msg) + } + + stop = 1 + if len(normCliErrStr) != 0 { + return NormCliErr() + } + return nil +} + +var normCliErr error + +func NormCliErr() error { + if normCliErr == nil { + normCliErr = fmt.Errorf("normal client got error") + } + return normCliErr +} + +func mkClients(num int, stop *int, bindV int, grpName string, outErr *string, db *sql.DB) { + for i := 0; i < num; i++ { + go func(clientId int) { + count := 0 + var conn *sql.Conn + var err error + var curErr string + for *stop == 0 { + nowStr := time.Now().Format("15:04:05.000000 ") + if conn == nil { + conn, err = db.Conn(context.Background()) + fmt.Printf(grpName+" connected %d\n", clientId) + if err != nil { + fmt.Printf(nowStr+grpName+" Error %d conn %s\n", clientId, err.Error()) + time.Sleep(7 * time.Millisecond) + continue + } + } + + fmt.Printf(nowStr+grpName+"%d loop%d %s\n", clientId, count, time.Now().Format("20060102j150405.000000")) + err := sleepyQ(conn, bindV) + if err != nil { + if err.Error() == curErr { + fmt.Printf(nowStr+grpName+"%d same err twice\n", clientId) + conn.Close() + conn = nil + } else { + curErr = err.Error() + *outErr = curErr + fmt.Printf(nowStr+grpName+"%d err %s\n", clientId, curErr) + } + } + count++ + time.Sleep(10 * time.Millisecond) + } + fmt.Printf(time.Now().Format("15:04:05.000000 ")+grpName+"%d END loop%d\n", clientId, count) + }(i) + } +} + +func TestBindEvict(t *testing.T) { + // we would like to clear hera.log, but even if we try, lots of messages still go there + logger.GetLogger().Log(logger.Debug, "TestBindEvict +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n") + err := fastAndSlowBinds() + if err != nil { + t.Fatalf("main step function returned err %s", err.Error()) + } + if testutil.RegexCountFile("BIND_THROTTLE", "cal.log") == 0 { + t.Fatalf("BIND_THROTTLE was not triggered") + } + if testutil.RegexCountFile("BIND_EVICT", "cal.log") == 0 { + t.Fatalf("BIND_EVICT was not triggered") + } + logger.GetLogger().Log(logger.Debug, "TestBindEvict stop +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n") +} // */ diff --git a/tests/unittest/bindLess/main_test.go b/tests/unittest/bindLess/main_test.go new file mode 100644 index 00000000..b59b1f07 --- /dev/null +++ b/tests/unittest/bindLess/main_test.go @@ -0,0 +1,242 @@ +package main + +import ( + "context" + "database/sql" + "fmt" + "os" + "testing" + "time" + + //"github.com/paypal/hera/client/gosqldriver" + _ "github.com/paypal/hera/client/gosqldriver/tcp" /*to register the driver*/ + + "github.com/paypal/hera/lib" + "github.com/paypal/hera/tests/unittest/testutil" + "github.com/paypal/hera/utility/logger" +) + +var mx testutil.Mux +var tableName string +var max_conn float64 + +func cfg() (map[string]string, map[string]string, testutil.WorkerType) { + + appcfg := make(map[string]string) + // best to chose an "unique" port in case golang runs tests in paralel + appcfg["bind_port"] = "31002" + appcfg["log_level"] = "5" + appcfg["log_file"] = "hera.log" + appcfg["sharding_cfg_reload_interval"] = "0" + appcfg["rac_sql_interval"] = "0" + appcfg["child.executable"] = "mysqlworker" + appcfg["bind_eviction_names"] = "p" + appcfg["bind_eviction_threshold_pct"] = "50" + + appcfg["request_backlog_timeout"] = "1000" + appcfg["soft_eviction_probability"] = "100" + + opscfg := make(map[string]string) + max_conn = 25 + opscfg["opscfg.default.server.max_connections"] = fmt.Sprintf("%d", int(max_conn)) + opscfg["opscfg.default.server.log_level"] = "5" + + opscfg["opscfg.default.server.saturation_recover_threshold"] = "10" + //opscfg["opscfg.default.server.saturation_recover_throttle_rate"]= "100" + opscfg["opscfg.hera.server.saturation_recover_throttle_rate"] = "100" + // saturation_recover_throttle_rate + + return appcfg, opscfg, testutil.MySQLWorker +} + +func before() error { + fmt.Printf("before run mysql") + testutil.RunMysql("create table sleep_info (id bigint, seconds float);") + testutil.RunMysql("insert into sleep_info (id,seconds) values(10, 0.01);") + testutil.RunMysql("insert into sleep_info (id,seconds) values(100, 0.1);") + testutil.RunMysql("insert into sleep_info (id,seconds) values(1600, 2.6);") + testutil.RunMysql("insert into sleep_info (id,seconds) values(21001111, 0.1);") + testutil.RunMysql("insert into sleep_info (id,seconds) values(22001111, 0.1);") + testutil.RunMysql("insert into sleep_info (id,seconds) values(29001111, 2.9);") + out, err := testutil.RunMysql(`DELIMITER $$ +CREATE FUNCTION sleep_option (id bigint) +RETURNS float +DETERMINISTIC +BEGIN + declare dur float; + declare rv bigint; + select max(seconds) into dur from sleep_info where sleep_info.id=id; + select sleep(dur) into rv; + RETURN dur; +END$$ +DELIMITER ;`) + if err != nil { + fmt.Printf("err after run mysql " + err.Error()) + return nil + } + fmt.Printf("after run mysql " + out) // */ + return nil +} + +func TestMain(m *testing.M) { + logger.GetLogger().Log(logger.Debug, "begin 20230918kkang TestMain") + fmt.Printf("TestMain 20230918kkang\n") + os.Exit(testutil.UtilMain(m, cfg, before)) +} + +func sleepyQ(conn *sql.Conn, delayRow int) error { + stmt, err := conn.PrepareContext(context.Background(), "select * from sleep_info where ( seconds > sleep_option(?) or seconds > 0.0 )") + if err != nil { + fmt.Printf("Error preparing sleepyQ %s\n", err.Error()) + return err + } + defer stmt.Close() + rows, err := stmt.Query(delayRow) + if err != nil { + fmt.Printf("Error query sleepyQ %s\n", err.Error()) + return err + } + defer rows.Close() + return nil +} + +var normCliErr error + +func NormCliErr() error { + if normCliErr == nil { + normCliErr = fmt.Errorf("normal client got error") + } + return normCliErr +} + +func partialBadLoad(fracBad float64) error { + db, err := sql.Open("hera", "127.0.0.1:31002") + if err != nil { + fmt.Printf("Error db %s\n", err.Error()) + return err + } + db.SetConnMaxLifetime(111 * time.Second) + db.SetMaxIdleConns(0) + db.SetMaxOpenConns(22111) + defer db.Close() + + // client threads of slow queries + var stop2 int + var stop3 int + var badCliErr string + var cliErr string + numBad := int(max_conn * fracBad) + numNorm := int(max_conn*2.1) + 1 - numBad + fmt.Printf("spawning clients bad%d norm%d\n", numBad, numNorm) + mkClients(numBad, &stop2, 29001111, "badClient", &badCliErr, db) + mkClients(numNorm, &stop3, 100, "normClient", &cliErr, db) // bind value is short, so bindevict won't trigger + time.Sleep(3100 * time.Millisecond) + //time.Sleep(33100 * time.Millisecond) + + // start normal clients after initial backlog timeouts + var stop int + var normCliErrStr string + mkClients(1, &stop, 21001111, "n client", &normCliErrStr, db) + time.Sleep(1100 * time.Millisecond) + + // if we throttle down or stop, it restores + stop2 = 1 // stop bad clients + stop3 = 1 + lib.GetConfig().BindEvictionDecrPerSec = 11333.1 + defer func() { lib.GetConfig().BindEvictionDecrPerSec = 1.1 }() + time.Sleep(1 * time.Second) + conn, err := db.Conn(context.Background()) + if err != nil { + fmt.Printf("Error conn %s\n", err.Error()) + return err + } + defer conn.Close() + err = sleepyQ(conn, 29001111) + if err != nil { + msg := fmt.Sprintf("test failed, throttle down didn't restore") + fmt.Printf("%s", msg) + return fmt.Errorf("%s", msg) + } + + stop = 1 + // tolerate soft eviction on normal client when we did not use bind eviction + if len(normCliErrStr) != 0 { + return NormCliErr() + } // */ + return nil +} + +func mkClients(num int, stop *int, bindV int, grpName string, outErr *string, db *sql.DB) { + for i := 0; i < num; i++ { + go func(clientId int) { + count := 0 + var conn *sql.Conn + var err error + var curErr string + for *stop == 0 { + nowStr := time.Now().Format("15:04:05.000000 ") + if conn == nil { + conn, err = db.Conn(context.Background()) + fmt.Printf(grpName+" connected %d\n", clientId) + if err != nil { + fmt.Printf(nowStr+grpName+" Error %d conn %s\n", clientId, err.Error()) + time.Sleep(7 * time.Millisecond) + continue + } + } + + fmt.Printf(nowStr+grpName+"%d loop%d %s\n", clientId, count, time.Now().Format("20060102j150405.000000")) + err := sleepyQ(conn, bindV) + if err != nil { + if err.Error() == curErr { + fmt.Printf(nowStr+grpName+"%d same err twice\n", clientId) + conn.Close() + conn = nil + } else { + curErr = err.Error() + *outErr = curErr + fmt.Printf(nowStr+grpName+"%d err %s\n", clientId, curErr) + } + } + count++ + time.Sleep(10 * time.Millisecond) + } + fmt.Printf(time.Now().Format("15:04:05.000000 ")+grpName+"%d END loop%d\n", clientId, count) + }(i) + } +} + +func TestBindLess(t *testing.T) { + // we would like to clear hera.log, but even if we try, lots of messages still go there + logger.GetLogger().Log(logger.Debug, "TestBindLess +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n") + testutil.BackupAndClear("cal", "BindLess start") + testutil.BackupAndClear("hera", "BindLess start") + err := partialBadLoad(0.10) + if err != nil && err != NormCliErr() { + t.Fatalf("main step function returned err %s", err.Error()) + } + if testutil.RegexCountFile("BIND_THROTTLE", "cal.log") > 0 { + t.Fatalf("BIND_THROTTLE should not trigger") + } + if testutil.RegexCountFile("BIND_EVICT", "cal.log") > 0 { + t.Fatalf("BIND_EVICT should not trigger") + } + if testutil.RegexCountFile("HERA-10", "hera.log") == 0 { + t.Fatal("backlog timeout or saturation was not triggered") + } // */ + + if true { + logger.GetLogger().Log(logger.Debug, "TestBindLess midpt +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n") + err := partialBadLoad(0.7) + if err != nil { + // t.Fatalf("main step function returned err %s", err.Error()) // can be triggered since test only has one sql + } + if testutil.RegexCountFile("BIND_THROTTLE", "cal.log") == 0 { + t.Fatalf("BIND_THROTTLE should trigger") + } + if testutil.RegexCountFile("BIND_EVICT", "cal.log") == 0 { + t.Fatalf("BIND_EVICT should trigger") + } + } // endif + logger.GetLogger().Log(logger.Debug, "TestBindLess done +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n") +} // */ diff --git a/tests/unittest/client_info_to_worker_disabled/main_test.go b/tests/unittest/client_info_to_worker_disabled/main_test.go new file mode 100644 index 00000000..33ae3601 --- /dev/null +++ b/tests/unittest/client_info_to_worker_disabled/main_test.go @@ -0,0 +1,100 @@ +package main + +import ( + + "os" + "testing" + "database/sql" + "context" + "time" + "fmt" + "github.com/paypal/hera/client/gosqldriver" + "github.com/paypal/hera/tests/unittest/testutil" + "github.com/paypal/hera/utility/logger" +) + +var mx testutil.Mux +var tableName string + +func cfg() (map[string]string, map[string]string, testutil.WorkerType) { + + appcfg := make(map[string]string) + // appcfg["x-mysql"] = "manual" // disable test framework spawning mysql server + // best to chose an "unique" port in case golang runs tests in paralel + appcfg["bind_port"] = "31002" + appcfg["log_level"] = "5" + appcfg["log_file"] = "hera.log" + appcfg["sharding_cfg_reload_interval"] = "0" + appcfg["rac_sql_interval"] = "0" + appcfg["db_heartbeat_interval"] = "10" + + opscfg := make(map[string]string) + opscfg["opscfg.default.server.max_connections"] = "3" + opscfg["opscfg.default.server.log_level"] = "5" + opscfg["opscfg.default.server.max_lifespan_per_child"]="5" + + appcfg["child.executable"] = "mysqlworker" + + if os.Getenv("WORKER") == "postgres" { + return appcfg, opscfg, testutil.PostgresWorker + } + + return appcfg, opscfg, testutil.MySQLWorker +} + +var ip1 string +var dbName = "heratestdb" + +func TestMain(m *testing.M) { + // startup mysql DBs + // ip1 := testutil.MakeDB("mysql33", dbName, testutil.MySQL) + // os.Setenv("TWO_TASK", "tcp("+ip1+":3306)/"+dbName+"?timeout=11s") + os.Exit(testutil.UtilMain(m, cfg, nil)) +} + +func TestClientInfoToWorkerDisabled(t *testing.T) { + logger.GetLogger().Log(logger.Debug, "TestClientInfoToWorkerDisabled begin +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n") + time.Sleep(5*time.Second) + shard := 0 + db, err := sql.Open("heraloop", fmt.Sprintf("%d:0:0", shard)) + if err != nil { + t.Fatal("Error starting Mux:", err) + return + } + db.SetMaxIdleConns(0) + defer db.Close() + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + conn, err := db.Conn(ctx); + if err != nil { + t.Fatalf("Error getting connection %s\n", err.Error()) + } + mux := gosqldriver.InnerConn(conn) + mux.SetCalCorrID("583f5e4a2758e") + err = mux.SetClientInfoWithPoolStack("testApplication", "localhost", "testApplication:testURL*CalThreadId=0*TopLevelTxnStartTime=18840b76115*Host=localhost*pid=100") + if err != nil { + t.Fatalf("Unable to set CLIENT_INFO") + } + rows, _ := conn.QueryContext(ctx, "SELECT version()") + // rows, _ := stmt.Query(1) + if !rows.Next() { + t.Fatalf("Expected 1 row") + } + rows.Close() + + if testutil.RegexCountFile(".*GetConfig().EnableCmdClientInfoToWorker.*", "hera.log") > 0 { + t.Fatalf("Error: should not have entered this block in coordinator") + } + + if testutil.RegexCountFile("CmdClientInfo:.*", "hera.log") > 0 { + t.Fatalf("Error: worker should not have got this command") + } + + if testutil.RegexCountFile("CLIENT_INFO_MUX.*", "cal.log") > 0 { + t.Fatalf("Error: Mux should not rename the CLIENT_INFO event") + } + + cancel() + conn.Close() + + logger.GetLogger().Log(logger.Debug, "TestClientInfoToWorkerDisabled done +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n") +} \ No newline at end of file diff --git a/tests/unittest/client_info_to_worker_enabled/main_test.go b/tests/unittest/client_info_to_worker_enabled/main_test.go new file mode 100644 index 00000000..7bf0b68f --- /dev/null +++ b/tests/unittest/client_info_to_worker_enabled/main_test.go @@ -0,0 +1,195 @@ +package main + +import ( + + "os" + "testing" + "database/sql" + "context" + "time" + "fmt" + "github.com/paypal/hera/client/gosqldriver" + "github.com/paypal/hera/tests/unittest/testutil" + "github.com/paypal/hera/utility/logger" +) + +var mx testutil.Mux +var tableName string + +func cfg() (map[string]string, map[string]string, testutil.WorkerType) { + + appcfg := make(map[string]string) + // appcfg["x-mysql"] = "manual" // disable test framework spawning mysql server + // best to chose an "unique" port in case golang runs tests in paralel + appcfg["bind_port"] = "31002" + appcfg["log_level"] = "5" + appcfg["log_file"] = "hera.log" + appcfg["sharding_cfg_reload_interval"] = "0" + appcfg["rac_sql_interval"] = "0" + appcfg["db_heartbeat_interval"] = "10" + appcfg["enable_client_info_to_worker"] = "true" + + opscfg := make(map[string]string) + opscfg["opscfg.default.server.max_connections"] = "3" + opscfg["opscfg.default.server.log_level"] = "5" + opscfg["opscfg.default.server.max_lifespan_per_child"]="5" + + appcfg["child.executable"] = "mysqlworker" + + if os.Getenv("WORKER") == "postgres" { + return appcfg, opscfg, testutil.PostgresWorker + } + + return appcfg, opscfg, testutil.MySQLWorker +} + +var ip1 string +var dbName = "heratestdb" + +func TestMain(m *testing.M) { + // startup mysql DBs + // ip1 := testutil.MakeDB("mysql33", dbName, testutil.MySQL) + // os.Setenv("TWO_TASK", "tcp("+ip1+":3306)/"+dbName+"?timeout=11s") + os.Exit(testutil.UtilMain(m, cfg, nil)) +} + +func TestClientInfoToWorkerHappyPath(t *testing.T) { + logger.GetLogger().Log(logger.Debug, "TestClientInfoToWorkerHappyPath begin +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n") + time.Sleep(5*time.Second) + shard := 0 + db, err := sql.Open("heraloop", fmt.Sprintf("%d:0:0", shard)) + if err != nil { + t.Fatal("Error starting Mux:", err) + return + } + db.SetMaxIdleConns(0) + defer db.Close() + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + conn, err := db.Conn(ctx); + if err != nil { + t.Fatalf("Error getting connection %s\n", err.Error()) + } + mux := gosqldriver.InnerConn(conn) + mux.SetCalCorrID("583f5e4a2758e") + err = mux.SetClientInfoWithPoolStack("testApplication", "localhost", "testApplication:testURL*CalThreadId=0*TopLevelTxnStartTime=18840b76115*Host=localhost*pid=100") + if err != nil { + t.Fatalf("Unable to set CLIENT_INFO") + } + rows, _ := conn.QueryContext(ctx, "SELECT version()") + // rows, _ := stmt.Query(1) + if !rows.Next() { + t.Fatalf("Expected 1 row") + } + rows.Close() + + if testutil.RegexCountFile("clientInfoMessage: testApplication", "hera.log") < 1 { + t.Fatalf("Error: should have sent CmdClientInfoToWorker to worker") + } + + if testutil.RegexCountFile("clientApplication: testApplication", "hera.log") < 1 { + t.Fatalf("Error: CmdProcessor should have processed CmdClientInfo") + } + + if testutil.RegexCountFile("CLIENT_INFO_MUX.*testApplication.*", "cal.log") < 1 { + t.Fatalf("Error: Mux should rename the CLIENT_INFO event") + } + + cancel() + conn.Close() + + logger.GetLogger().Log(logger.Debug, "TestClientInfoToWorkerHappyPath done +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n") +} + +func TestClientInfoToWorkerMissingClientInfo(t *testing.T) { + logger.GetLogger().Log(logger.Debug, "TestClientInfoToWorkerMissingClientInfo begin +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n") + time.Sleep(5*time.Second) + shard := 0 + db, err := sql.Open("heraloop", fmt.Sprintf("%d:0:0", shard)) + if err != nil { + t.Fatal("Error starting Mux:", err) + return + } + db.SetMaxIdleConns(0) + defer db.Close() + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + conn, err := db.Conn(ctx); + if err != nil { + t.Fatalf("Error getting connection %s\n", err.Error()) + } + mux := gosqldriver.InnerConn(conn) + mux.SetCalCorrID("583f5e4a2758e") + // err = mux.SetClientInfoWithPoolStack("testApplication", "localhost", "testApplication:testURL*CalThreadId=0*TopLevelTxnStartTime=18840b76115*Host=localhost*pid=100") + // if err != nil { + // t.Fatalf("Unable to set CLIENT_INFO") + // } + rows, _ := conn.QueryContext(ctx, "SELECT version()") + // rows, _ := stmt.Query(1) + if !rows.Next() { + t.Fatalf("Expected 1 row") + } + rows.Close() + + if testutil.RegexCountFile("clientInfoMessage: unset", "hera.log") < 1 { + t.Fatalf("Error: mux should have set the poolName to unset") + } + + if testutil.RegexCountFile("clientApplication: unset", "hera.log") < 1 { + t.Fatalf("Error: should have got unset from mux") + } + + if testutil.RegexCountFile("CLIENT_INFO_MUX.*testApplication.*", "cal.log") < 1 { + t.Fatalf("Error: Mux should not have processed CmdClientInfo") + } + + cancel() + conn.Close() + + logger.GetLogger().Log(logger.Debug, "TestClientInfoToWorkerMissingClientInfo done +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n") +} + +func TestClientInfoToWorkerMissingPoolName(t *testing.T) { + logger.GetLogger().Log(logger.Debug, "TestClientInfoToWorkerMissingPoolName begin +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n") + time.Sleep(5*time.Second) + shard := 0 + db, err := sql.Open("heraloop", fmt.Sprintf("%d:0:0", shard)) + if err != nil { + t.Fatal("Error starting Mux:", err) + return + } + db.SetMaxIdleConns(0) + defer db.Close() + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + conn, err := db.Conn(ctx); + if err != nil { + t.Fatalf("Error getting connection %s\n", err.Error()) + } + mux := gosqldriver.InnerConn(conn) + mux.SetCalCorrID("583f5e4a2758e") + err = mux.SetClientInfoWithPoolStack("", "localhost", "testApplication:testURL*CalThreadId=0*TopLevelTxnStartTime=18840b76115*Host=localhost*pid=100") + if err != nil { + t.Fatalf("Unable to set CLIENT_INFO") + } + rows, _ := conn.QueryContext(ctx, "SELECT version()") + // rows, _ := stmt.Query(1) + if !rows.Next() { + t.Fatalf("Expected 1 row") + } + rows.Close() + + if testutil.RegexCountFile("clientInfoMessage: unset", "hera.log") < 1 { + t.Fatalf("Error: mux should have set the poolName to unset") + } + + if testutil.RegexCountFile("clientApplication: unset", "hera.log") < 1 { + t.Fatalf("Error: should have got unset from mux") + } + + if testutil.RegexCountFile("CLIENT_INFO_MUX.*", "cal.log") < 2 { + t.Fatalf("Error: Mux should rename the CLIENT_INFO event") + } + + cancel() + conn.Close() + + logger.GetLogger().Log(logger.Debug, "TestClientInfoToWorkerMissingPoolName done +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n") +} diff --git a/tests/unittest/coordinator_basic/main_test.go b/tests/unittest/coordinator_basic/main_test.go index 165106ef..e623549c 100644 --- a/tests/unittest/coordinator_basic/main_test.go +++ b/tests/unittest/coordinator_basic/main_test.go @@ -83,10 +83,11 @@ func TestCoordinatorBasic(t *testing.T) { t.Fatalf("Error getting connection %s\n", err.Error()) } tx, _ := conn.BeginTx(ctx, nil) - stmt, _ := tx.PrepareContext(ctx, "/*cmd*/delete from "+tableName) + sqlTxt := "/*cmd*/delete from "+tableName + stmt, _ := tx.PrepareContext(ctx, sqlTxt) _, err = stmt.Exec() if err != nil { - t.Fatalf("Error preparing test (delete table) %s\n", err.Error()) + t.Fatalf("Error preparing test (delete table) %s with %s ==== sql\n", err.Error(), sqlTxt) } stmt, _ = tx.PrepareContext(ctx, "/*cmd*/insert into "+tableName+" (id, int_val, str_val) VALUES(?, ?, ?)") _, err = stmt.Exec(1, time.Now().Unix(), "val 1") diff --git a/tests/unittest/coordinator_rqid/main_test.go b/tests/unittest/coordinator_rqid/main_test.go index 17c1dcbb..eedaea90 100644 --- a/tests/unittest/coordinator_rqid/main_test.go +++ b/tests/unittest/coordinator_rqid/main_test.go @@ -52,10 +52,10 @@ func before() error { if tableName == "" { tableName = "jdbc_hera_test" } - if strings.HasPrefix(os.Getenv("TWO_TASK"), "tcp") { // mysql + if strings.HasPrefix(os.Getenv("TWO_TASK"), "tcp") { // mysql // with testutil.RunDML, extra log line throws off test testutil.DBDirect("create table jdbc_hera_test ( ID BIGINT, INT_VAL BIGINT, STR_VAL VARCHAR(500))", os.Getenv("MYSQL_IP"), "heratestdb", testutil.MySQL) - } + } return nil } diff --git a/tests/unittest/coordinator_sharding/main_test.go b/tests/unittest/coordinator_sharding/main_test.go index e12eb9f6..713c05f2 100644 --- a/tests/unittest/coordinator_sharding/main_test.go +++ b/tests/unittest/coordinator_sharding/main_test.go @@ -44,37 +44,37 @@ func cfg() (map[string]string, map[string]string, testutil.WorkerType) { return appcfg, opscfg, testutil.MySQLWorker } -func setupShardMap(t *testing.T) { - twoTask := os.Getenv("TWO_TASK") - if !strings.HasPrefix(twoTask, "tcp") { - // not mysql - return - } - shard := 0 - db, err := sql.Open("heraloop", fmt.Sprintf("%d:0:0", shard)) - if err != nil { - t.Fatal("Error starting Mux:", err) - return - } - db.SetMaxIdleConns(0) - defer db.Close() - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - conn, err := db.Conn(ctx) - if err != nil { - t.Fatalf("Error getting connection %s\n", err.Error()) - } - defer conn.Close() - - testutil.RunDML("create table hera_shard_map ( scuttle_id smallint not null, shard_id tinyint not null, status char(1) , read_status char(1), write_status char(1), remarks varchar(500))") - - for i := 0; i < 1024; i++ { +func setupShardMap() { + twoTask := os.Getenv("TWO_TASK") + if !strings.HasPrefix(twoTask, "tcp") { + // not mysql + return + } + shard := 0 + db, err := sql.Open("heraloop", fmt.Sprintf("%d:0:0", shard)) + if err != nil { + testutil.Fatal("Error starting Mux:", err) + return + } + db.SetMaxIdleConns(0) + defer db.Close() + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + conn, err := db.Conn(ctx) + if err != nil { + testutil.Fatalf("Error getting connection %s\n", err.Error()) + } + defer conn.Close() + + testutil.RunDML("create table hera_shard_map ( scuttle_id smallint not null, shard_id tinyint not null, status char(1) , read_status char(1), write_status char(1), remarks varchar(500))") + + for i := 0; i < 1024; i++ { shard := 0 if i <= 8 { - shard = i%3 + shard = i % 3 } - testutil.RunDML(fmt.Sprintf("insert into hera_shard_map ( scuttle_id, shard_id, status, read_status, write_status ) values ( %d, %d, 'Y', 'Y', 'Y' )", i, shard ) ) - } + testutil.RunDML(fmt.Sprintf("insert into hera_shard_map ( scuttle_id, shard_id, status, read_status, write_status ) values ( %d, %d, 'Y', 'Y', 'Y' )", i, shard)) + } } func before() error { @@ -82,10 +82,10 @@ func before() error { if tableName == "" { tableName = "jdbc_hera_test" } - if strings.HasPrefix(os.Getenv("TWO_TASK"), "tcp") { - // mysql - testutil.RunDML("create table jdbc_hera_test ( ID BIGINT, INT_VAL BIGINT, STR_VAL VARCHAR(500))") - } + if strings.HasPrefix(os.Getenv("TWO_TASK"), "tcp") { + // mysql + testutil.RunDML("create table jdbc_hera_test ( ID BIGINT, INT_VAL BIGINT, STR_VAL VARCHAR(500))") + } return nil } @@ -106,7 +106,7 @@ func cleanup(ctx context.Context, conn *sql.Conn) error { func TestShardingBasic(t *testing.T) { logger.GetLogger().Log(logger.Debug, "TestShardingBasic setup") - setupShardMap(t) + setupShardMap() logger.GetLogger().Log(logger.Debug, "TestShardingBasic begin +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n") hostname, _ := os.Hostname() @@ -180,7 +180,7 @@ func TestShardingSetShard(t *testing.T) { db.SetMaxIdleConns(0) defer db.Close() - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second) conn, err := db.Conn(ctx) if err != nil { t.Fatalf("Error getting connection %s\n", err.Error()) @@ -230,7 +230,7 @@ func TestShardingSetShard(t *testing.T) { if err == nil { t.Fatal("Expected to fail because no shard key") } - if err.Error() != "Internal hera error: HERA-373: no shard key or more than one or bad logical db" { + if err.Error() != "Internal hera error: HERA-373: no shard key or more than one or bad logical db, shard_key=id" { t.Fatal("Expected error HERA-373") } @@ -545,3 +545,57 @@ func TestShardingSetShardKey(t *testing.T) { logger.GetLogger().Log(logger.Debug, "TestShardingSetShardKey done -------------------------------------------------------------") } + +func TestShardingWithNoShardKey(t *testing.T) { + logger.GetLogger().Log(logger.Debug, "TestShardingWithNoShardKey setup") + setupShardMap() + logger.GetLogger().Log(logger.Debug, "TestShardingWithNoShardKey begin +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n") + + hostname, _ := os.Hostname() + db, err := sql.Open("hera", hostname+":31003") + if err != nil { + t.Fatal("Error starting Mux:", err) + return + } + db.SetMaxIdleConns(0) + defer db.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second) + conn, err := db.Conn(ctx) + if err != nil { + t.Fatalf("Error getting connection %s\n", err.Error()) + } + cleanup(ctx, conn) + // insert one row in the table + tx, _ := conn.BeginTx(ctx, nil) + currentTime := time.Now().Unix() + stmt, err := tx.PrepareContext(ctx, "/*TestShardingWithNoShardKey*/insert into "+tableName+" (id, int_val, str_val) VALUES(:id, :int_val, :str_val)") + if err != nil { + t.Fatalf("Error creating statement(create row in table) %s\n", err.Error()) + } + _, err = stmt.Exec(sql.Named("id", 1), sql.Named("int_val", currentTime), sql.Named("str_val", "val 1")) + if err != nil { + t.Fatalf("Error preparing test (create row in table) %s\n", err.Error()) + } + err = tx.Commit() + if err != nil { + t.Fatalf("Error commit %s\n", err.Error()) + } + + stmt, _ = conn.PrepareContext(ctx, "/*TestShardingWithNoShardKey*/Select id, int_val, str_val from "+tableName+" where int_val=:int_val") + _, err = stmt.Query(sql.Named("int_val", currentTime)) + + if err == nil { + t.Fatalf("Expected no shard-key error for shrd key 'id'") + } + stmt.Close() + // check the logs that in fact shard 1 was used + out, err := testutil.BashCmd("grep 'shard_key=id' cal.log | wc -l") + if (err != nil) || (len(out) == 0) { + err = nil + t.Fatalf("Expected shard key details, shard_key=id in cal.log. err = %v, len(out) = %d", err, len(out)) + } + conn.Close() + cancel() + logger.GetLogger().Log(logger.Debug, "TestShardingWithNoShardKey done ----------------------------------------------------------") +} diff --git a/tests/unittest/coordinator_sharding_mod/main_test.go b/tests/unittest/coordinator_sharding_mod/main_test.go index 86dc7c63..4b19eeeb 100644 --- a/tests/unittest/coordinator_sharding_mod/main_test.go +++ b/tests/unittest/coordinator_sharding_mod/main_test.go @@ -44,7 +44,7 @@ func cfg() (map[string]string, map[string]string, testutil.WorkerType) { return appcfg, opscfg, testutil.MySQLWorker } -func setupShardMap(t *testing.T) { +func setupShardMap() { twoTask := os.Getenv("TWO_TASK") if !strings.HasPrefix(twoTask, "tcp") { // not mysql @@ -53,7 +53,7 @@ func setupShardMap(t *testing.T) { shard := 0 db, err := sql.Open("heraloop", fmt.Sprintf("%d:0:0", shard)) if err != nil { - t.Fatal("Error starting Mux:", err) + testutil.Fatal("Error starting Mux:", err) return } db.SetMaxIdleConns(0) @@ -62,7 +62,7 @@ func setupShardMap(t *testing.T) { defer cancel() conn, err := db.Conn(ctx) if err != nil { - t.Fatalf("Error getting connection %s\n", err.Error()) + testutil.Fatalf("Error getting connection %s\n", err.Error()) } defer conn.Close() @@ -106,7 +106,7 @@ func cleanup(ctx context.Context, conn *sql.Conn) error { func TestShardingMod(t *testing.T) { logger.GetLogger().Log(logger.Debug, "TestShardingMod setup") - setupShardMap(t) + setupShardMap() logger.GetLogger().Log(logger.Debug, "TestShardingMod begin +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n") hostname, _ := os.Hostname() diff --git a/tests/unittest/log_checker/main_test.go b/tests/unittest/log_checker/main_test.go index 08120346..54e7711f 100644 --- a/tests/unittest/log_checker/main_test.go +++ b/tests/unittest/log_checker/main_test.go @@ -14,6 +14,7 @@ import ( "github.com/paypal/hera/tests/unittest/testutil" "github.com/paypal/hera/utility/logger" + "github.com/paypal/hera/client/gosqldriver" ) /* @@ -42,8 +43,8 @@ func cfg() (map[string]string, map[string]string, testutil.WorkerType) { appcfg["log_level"] = "5" appcfg["log_file"] = "hera.log" appcfg["sharding_cfg_reload_interval"] = "0" - appcfg["rac_sql_interval"] = "2" - appcfg["db_heartbeat_interval"] = "3" + appcfg["rac_sql_interval"] = "0" + appcfg["db_heartbeat_interval"] = "20" opscfg := make(map[string]string) opscfg["opscfg.default.server.max_connections"] = "3" @@ -85,6 +86,9 @@ func TestCalClientSessionDur(t *testing.T) { t.Fatalf("Error getting connection %s\n", err.Error()) } + mux := gosqldriver.InnerConn(conn) + mux.SetCalCorrID("aaaf5e4a2758e") + stmt, err := conn.PrepareContext(ctx, "select 'foo' from dual") if err != nil { t.Fatalf("Error with the prepared statement") @@ -98,15 +102,149 @@ func TestCalClientSessionDur(t *testing.T) { cancel() conn.Close() - clientSessionDurLogScan(t) + err = clientSessionDurLogScan() + if err != nil { + t.Fatalf("clientSessionDurLogScan %v", err) + } logger.GetLogger().Log(logger.Debug, "TestCalClientSessionDur done -------------------------------------------------------------") } -func clientSessionDurLogScan(t *testing.T){ +func TestCalClientSessionCorrId(t *testing.T) { + logger.GetLogger().Log(logger.Debug, "TestCalClientSessionCorrId begin +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n") + time.Sleep(5*time.Second) + shard := 0 + db, err := sql.Open("heraloop", fmt.Sprintf("%d:0:0", shard)) + if err != nil { + t.Fatal("Error starting Mux:", err) + return + } + db.SetMaxIdleConns(0) + defer db.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + conn, err := db.Conn(ctx); + if err != nil { + t.Fatalf("Error getting connection %s\n", err.Error()) + } + mux := gosqldriver.InnerConn(conn) + // mux.SetCalCorrID("583f5e4a27aaa") + mux.SetCalCorrID("583f5e4a27aaa&PoolStack: testApplication:*CalThreadId=1572864*TopLevelTxnStartTime=18aa9970c9c*Host=testNode") + + rows, _ := conn.QueryContext(ctx, "SELECT version()") + // rows, _ := stmt.Query(1) + if !rows.Next() { + t.Fatalf("Expected 1 row") + } + rows.Close() + + if testutil.RegexCountFile("CmdClientCalCorrelationID: CorrId=583f5e4a27aaa", "hera.log") < 1 { + t.Fatalf("Error: should have handled CmdClientCalCorrelationID") + } + + if testutil.RegexCountFile("corr_id_= 583f5e4a27aaa", "hera.log") < 1 { + t.Fatalf("Error: should have parsed the input") + } + + if testutil.RegexCountFile("CLIENT_SESSION.*corrid=583f5e4a27aaa", "cal.log") != 1 { + t.Fatalf("Error: should have corrid in CLIENT_SESSION") + } + + cancel() + conn.Close() + + logger.GetLogger().Log(logger.Debug, "TestCalClientSessionCorrId done +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n") +} + +func TestCalClientSessionCorrIdInvalid(t *testing.T) { + logger.GetLogger().Log(logger.Debug, "TestCalClientSessionCorrIdInvalid begin +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n") + time.Sleep(5*time.Second) + shard := 0 + db, err := sql.Open("heraloop", fmt.Sprintf("%d:0:0", shard)) + if err != nil { + t.Fatal("Error starting Mux:", err) + return + } + db.SetMaxIdleConns(0) + defer db.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + conn, err := db.Conn(ctx); + if err != nil { + t.Fatalf("Error getting connection %s\n", err.Error()) + } + + mux := gosqldriver.InnerConn(conn) + mux.SetCalCorrID("PoolStack: testApplication:*CalThreadId=1572864*TopLevelTxnStartTime=18aa9970*Host=testNode") + + rows, _ := conn.QueryContext(ctx, "SELECT version()") + // rows, _ := stmt.Query(1) + if !rows.Next() { + t.Fatalf("Expected 1 row") + } + rows.Close() + + if testutil.RegexCountFile("corrid not in expected format.*PoolStack: testApplication", "hera.log") < 1 { + t.Fatalf("Error: should have thrown error due to corrId size") + } + + if testutil.RegexCountFile("CLIENT_SESSION.*corrid=unset", "cal.log") != 1 { + t.Fatalf("Error: should have corrid as unset") + } + + cancel() + conn.Close() + + logger.GetLogger().Log(logger.Debug, "TestCalClientSessionCorrIdInvalid done +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n") +} + +func TestCalClientSessionEmptyCorrId(t *testing.T) { + logger.GetLogger().Log(logger.Debug, "TestCalClientSessionEmptyCorrId begin +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n") + time.Sleep(5*time.Second) + shard := 0 + db, err := sql.Open("heraloop", fmt.Sprintf("%d:0:0", shard)) + if err != nil { + t.Fatal("Error starting Mux:", err) + return + } + db.SetMaxIdleConns(0) + defer db.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + conn, err := db.Conn(ctx); + if err != nil { + t.Fatalf("Error getting connection %s\n", err.Error()) + } + + mux := gosqldriver.InnerConn(conn) + mux.SetCalCorrID("") + + rows, _ := conn.QueryContext(ctx, "SELECT version()") + // rows, _ := stmt.Query(1) + if !rows.Next() { + t.Fatalf("Expected 1 row") + } + rows.Close() + + if testutil.RegexCountFile("corrid not in expected format: CorrId=", "hera.log") < 1 { + t.Fatalf("Error: should have thrown error due to corrId format") + } + + if testutil.RegexCountFile("CLIENT_SESSION.*corrid=unset", "cal.log") != 2 { + t.Fatalf("Error: should have corrid as unset") + } + + cancel() + conn.Close() + + logger.GetLogger().Log(logger.Debug, "TestCalClientSessionEmptyCorrId done +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n") +} + +func clientSessionDurLogScan() (error){ file, err := os.Open("cal.log") defer file.Close() if err != nil { - t.Fatalf("Error in opening cal.log") + fmt.Printf("Error in opening cal.log") + return err } re := regexp.MustCompile("[ |\t][0-9]+\\.[0-9]") cliSession_re := regexp.MustCompile("CLIENT_SESSION.*corr_id_") @@ -116,12 +254,15 @@ func clientSessionDurLogScan(t *testing.T){ if(cliSession_re.MatchString(line)){ _, err := strconv.ParseFloat(strings.TrimSpace(re.FindAllString(line, -1)[0]),32) if(err != nil){ - t.Fatalf("Num error for CLIENT_SESSION duration") + fmt.Printf("Num error for CLIENT_SESSION duration") + return err } } } if err := scanner.Err(); err != nil { - t.Fatalf("cal.log read error") + fmt.Printf("cal.log read error") + return err } + return nil } diff --git a/tests/unittest/rac_maint/main_test.go b/tests/unittest/rac_maint/main_test.go index 00f7be74..46710e10 100644 --- a/tests/unittest/rac_maint/main_test.go +++ b/tests/unittest/rac_maint/main_test.go @@ -34,6 +34,7 @@ func cfg() (map[string]string, map[string]string, testutil.WorkerType) { } func before() error { + os.Setenv("PARALLEL", "1") pfx := os.Getenv("MGMT_TABLE_PREFIX") if pfx == "" { pfx = "hera" @@ -46,9 +47,8 @@ func TestMain(m *testing.M) { os.Exit(testutil.UtilMain(m, cfg, before)) } -func TestRacMaint(t *testing.T) { - logger.GetLogger().Log(logger.Debug, "TestRacMaint begin +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n") - +func TestRacMaintWithStatusChange(t *testing.T) { + logger.GetLogger().Log(logger.Debug, "TestRacMaintWithStatusChange begin +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n") shard := 0 db, err := sql.Open("heraloop", fmt.Sprintf("%d:0:0", shard)) if err != nil { @@ -58,7 +58,7 @@ func TestRacMaint(t *testing.T) { db.SetMaxIdleConns(0) defer db.Close() - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() // cleanup and insert one row in the table conn, err := db.Conn(ctx) @@ -86,7 +86,7 @@ func TestRacMaint(t *testing.T) { time.Sleep(4100 * time.Millisecond) if 0 != testutil.RegexCountFile("Rac maint activating, worker", "hera.log") { - t.Fatalf("should not have rac maint activation") + t.Fatalf("should not have rac maint activation instance ID: 15") } tx, err = conn.BeginTx(ctx, nil) @@ -103,16 +103,277 @@ func TestRacMaint(t *testing.T) { t.Fatalf("Error prep ins %s\n", err.Error()) } // mysql uses instId 0 since there isn't instid's - insS.Exec(0, "F", time.Now().Unix()+1, "hera-test", hostname) + insS.Exec(0, "U", time.Now().Unix()+1, "hera-test", hostname) + err = tx.Commit() + if err != nil { + t.Fatalf("Error 2commit %s\n", err.Error()) + } + + time.Sleep(4100 * time.Millisecond) + if 0 != testutil.RegexCountFile("Rac maint activating, worker", "hera.log") { + t.Fatalf("RAC Maint should not activate for status U") + } + + if 0 == testutil.RegexCountFile("module:HERA-TEST", "cal.log") { + t.Fatalf("Status 'U' should log the RACMAINT_INFO_CHANGE event") + } + + // mysql uses instId 0 since there isn't instid's + tx, err = conn.BeginTx(ctx, nil) + if err != nil { + t.Fatalf("Error 2commit %s\n", err.Error()) + } + stmt2, err := tx.PrepareContext(ctx, "/*cmd*/insert into "+tableName+" (inst_id, status, status_time, module, machine) values (?,?,?,?,?)") + if err != nil { + t.Fatalf("Error prep ins %s\n", err.Error()) + } + // mysql uses instId 0 since there isn't instid's + timeInMillis := time.Now().Unix() + 1 + stmt2.Exec(0, "R", timeInMillis, "hera-test", hostname) + stmt2.Exec(0, "R", timeInMillis, "hera-test_taf", hostname) + err = tx.Commit() + if err != nil { + t.Fatalf("Error 2commit %s\n", err.Error()) + } + + time.Sleep(4100 * time.Millisecond) + + if 0 == testutil.RegexCountFile("Rac maint activating, worker", "hera.log") { + t.Fatalf("requires rac maint activation for main module status") + } + + if 0 == testutil.RegexCountFile("module:HERA-TEST_TAF", "cal.log") { + t.Fatalf("Status 'U' should log the RACMAINT_INFO_CHANGE event") + } + if 0 != testutil.RegexCountFile("invalid_status", "cal.log") { + t.Fatalf("ram maint status 'U' should not skip with invalid-status event") + } + time.Sleep(4100 * time.Millisecond) + //Clear logs + testutil.ClearLogsData() + + tx, err = conn.BeginTx(ctx, nil) + if err != nil { + t.Fatalf("Error to start transaction %s\n", err.Error()) + } + stmt2, err = tx.PrepareContext(ctx, "/*cmd*/insert into "+tableName+" (inst_id, status, status_time, module, machine) values (?,?,?,?,?)") + if err != nil { + t.Fatalf("Error prep ins %s\n", err.Error()) + } + // mysql uses instId 0 since there isn't instid's + timeInMillis = time.Now().Unix() + 1 + stmt2.Exec(0, "U", timeInMillis, "hera-test", hostname) + stmt2.Exec(0, "U", timeInMillis, "hera-test_taf", hostname) + err = tx.Commit() + if err != nil { + t.Fatalf("Error 2commit %s\n", err.Error()) + } + + time.Sleep(4100 * time.Millisecond) + + if 0 == testutil.RegexCountFile("module:HERA-TEST_TAF", "cal.log") { + t.Fatalf("Status 'U' should log the RACMAINT_INFO_CHANGE event") + } + + if 0 == testutil.RegexCountFile("module:HERA-TEST", "cal.log") { + t.Fatalf("Status 'U' should log the RACMAINT_INFO_CHANGE event") + } + + if 2 == testutil.RegexCountFile("RACMAINT_INFO_CHANGE", "cal.log") { + t.Fatalf("We have 2 insert queries with different status, so should log the RACMAINT_INFO_CHANGE event for every change") + } + + if 0 != testutil.RegexCountFile("invalid_status", "cal.log") { + t.Fatalf("ram maint status 'U' should not skip with invalid-status event") + } + + //Clear logs + testutil.ClearLogsData() + + tx, err = conn.BeginTx(ctx, nil) + if err != nil { + t.Fatalf("Error to start transaction %s\n", err.Error()) + } + stmt2, err = tx.PrepareContext(ctx, "/*cmd*/insert into "+tableName+" (inst_id, status, status_time, module, machine) values (?,?,?,?,?)") + if err != nil { + t.Fatalf("Error prep ins %s\n", err.Error()) + } + // mysql uses instId 0 since there isn't instid's + timeInMillis = time.Now().Unix() + 1 + stmt2.Exec(0, "F", timeInMillis, "hera-test", hostname) + stmt2.Exec(0, "F", timeInMillis, "hera-test_taf", hostname) err = tx.Commit() if err != nil { t.Fatalf("Error 2commit %s\n", err.Error()) } time.Sleep(4100 * time.Millisecond) + if 0 == testutil.RegexCountFile("Rac maint activating, worker", "hera.log") { - t.Fatalf("missed rac maint activation") + t.Fatalf("requires rac maint activation for main module status") + } + + if 0 == testutil.RegexCountFile("module:HERA-TEST_TAF", "cal.log") { + t.Fatalf("Status 'F' should log the RACMAINT_INFO_CHANGE event") + } + + if 0 == testutil.RegexCountFile("module:HERA-TEST", "cal.log") { + t.Fatalf("Status 'F' should log the RACMAINT_INFO_CHANGE event") + } + + if 2 == testutil.RegexCountFile("RACMAINT_INFO_CHANGE", "cal.log") { + t.Fatalf("We have 2 insert queries with different status, so should log the RACMAINT_INFO_CHANGE event for every change") + } + + if 0 != testutil.RegexCountFile("invalid_status", "cal.log") { + t.Fatalf("ram maint status 'U' should not skip with invalid-status event") + } + + logger.GetLogger().Log(logger.Debug, "TestRacMaintWithStatusChange done -------------------------------------------------------------") +} + +func TestRcMaintInvalidStatus(t *testing.T) { + logger.GetLogger().Log(logger.Debug, "TestRacMaintInvalidStatus begin +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n") + shard := 0 + db, err := sql.Open("heraloop", fmt.Sprintf("%d:0:0", shard)) + if err != nil { + t.Fatal("Error starting Mux:", err) + return + } + db.SetMaxIdleConns(0) + defer db.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + // cleanup and insert one row in the table + conn, err := db.Conn(ctx) + if err != nil { + t.Fatalf("Error getting connection %s\n", err.Error()) + } + defer conn.Close() + tx, _ := conn.BeginTx(ctx, nil) + stmt, _ := tx.PrepareContext(ctx, "/*cmd*/delete from "+tableName) + _, err = stmt.Exec() + if err != nil { + t.Fatalf("Error preparing test (delete table) %s\n", err.Error()) + } + stmt, _ = tx.PrepareContext(ctx, "/*cmd*/insert into "+tableName+" (inst_id, status, status_time, module, machine) values (?,?,?,?,?)") + hostname, _ := os.Hostname() + // how to do inst_id + _, err = stmt.Exec(15 /*max instid*/, "F", time.Now().Unix()+2, "hera-test", hostname) + if err != nil { + t.Fatalf("Error preparing test (create row in table) %s\n", err.Error()) + } + err = tx.Commit() + if err != nil { + t.Fatalf("Error commit %s\n", err.Error()) + } + + time.Sleep(4100 * time.Millisecond) + if 0 != testutil.RegexCountFile("Rac maint activating, worker", "hera.log") { + t.Fatalf("should not have rac maint activation") + } + + tx, err = conn.BeginTx(ctx, nil) + if err != nil { + t.Fatalf("Error 2commit %s\n", err.Error()) + } + insS, err := tx.PrepareContext(ctx, "/*cmd*/insert into "+tableName+" (inst_id, status, status_time, module, machine) values (?,?,?,?,?)") + if err != nil { + t.Fatalf("Error prep ins %s\n", err.Error()) + } + // mysql uses instId 0 since there isn't instid's + insS.Exec(0, "T", time.Now().Unix()+1, "hera-test", hostname) + err = tx.Commit() + if err != nil { + t.Fatalf("Error 2commit %s\n", err.Error()) + } + + time.Sleep(4100 * time.Millisecond) + + if 0 == testutil.RegexCountFile("invalid_status", "cal.log") { + t.Fatalf("ram maint should skip with invalid-status event") + } + if 0 != testutil.RegexCountFile("Rac maint activating, worker", "hera.log") { + t.Fatalf("should not have rac maint activation for invalid status") + } + + logger.GetLogger().Log(logger.Debug, "TestRacMaintInvalidStatus done -------------------------------------------------------------") +} + +func TestRcMaintNoChangeInTMAndStatus(t *testing.T) { + logger.GetLogger().Log(logger.Debug, "TestRcMaintNoChangeInTM begin +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n") + shard := 0 + db, err := sql.Open("heraloop", fmt.Sprintf("%d:0:0", shard)) + if err != nil { + t.Fatal("Error starting Mux:", err) + return + } + db.SetMaxIdleConns(0) + defer db.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + // cleanup and insert one row in the table + conn, err := db.Conn(ctx) + if err != nil { + t.Fatalf("Error getting connection %s\n", err.Error()) + } + defer conn.Close() + tx, _ := conn.BeginTx(ctx, nil) + stmt, _ := tx.PrepareContext(ctx, "/*cmd*/delete from "+tableName) + _, err = stmt.Exec() + if err != nil { + t.Fatalf("Error preparing test (delete table) %s\n", err.Error()) + } + hostname, _ := os.Hostname() + insS, err := tx.PrepareContext(ctx, "/*cmd*/insert into "+tableName+" (inst_id, status, status_time, module, machine) values (?,?,?,?,?)") + if err != nil { + t.Fatalf("Error prep ins %s\n", err.Error()) + } + // mysql uses instId 0 since there isn't instid's + timeValInMillis := time.Now().Unix() + 1 + insS.Exec(0, "U", timeValInMillis, "hera-test", hostname) + err = tx.Commit() + if err != nil { + t.Fatalf("Error 2commit %s\n", err.Error()) + } + + time.Sleep(4100 * time.Millisecond) + if 0 != testutil.RegexCountFile("Rac maint activating, worker", "hera.log") { + t.Fatalf("RAC Maint should not activate for status U") } - logger.GetLogger().Log(logger.Debug, "TestRacMaint done -------------------------------------------------------------") + if 0 == testutil.RegexCountFile("module:HERA-TEST", "cal.log") { + t.Fatalf("Status 'U' should log the RACMAINT_INFO_CHANGE event") + } + + //Clear logs + testutil.ClearLogsData() + // mysql uses instId 0 since there isn't instid's + tx, err = conn.BeginTx(ctx, nil) + if err != nil { + t.Fatalf("Error 2commit %s\n", err.Error()) + } + stmt2, err := tx.PrepareContext(ctx, "/*cmd*/insert into "+tableName+" (inst_id, status, status_time, module, machine) values (?,?,?,?,?)") + if err != nil { + t.Fatalf("Error prep ins %s\n", err.Error()) + } + + stmt2.Exec(0, "U", timeValInMillis, "hera-test", hostname) + err = tx.Commit() + if err != nil { + t.Fatalf("Error 2commit %s\n", err.Error()) + } + + time.Sleep(4100 * time.Millisecond) + + if 0 != testutil.RegexCountFile("Rac maint activating, worker", "hera.log") { + t.Fatalf("RAC Maint should not activate for status U") + } + + if 0 != testutil.RegexCountFile("module:HERA-TEST", "cal.log") { + t.Fatalf("Status 'U' same TM value should not log the RACMAINT_INFO_CHANGE event") + } + logger.GetLogger().Log(logger.Debug, "TestRcMaintNoChangeInTM done -------------------------------------------------------------") } diff --git a/tests/unittest/sharding_str/main_test.go b/tests/unittest/sharding_str/main_test.go index 063c81f9..79329bcc 100644 --- a/tests/unittest/sharding_str/main_test.go +++ b/tests/unittest/sharding_str/main_test.go @@ -107,7 +107,7 @@ BEGIN END; / */ -func setupShardMap(t *testing.T) { +func setupShardMap() { testutil.RunDML("DROP TABLE IF EXISTS test_str_sk") testutil.RunDML("create table test_str_sk (email_addr varchar(64), note varchar(64))") testutil.RunDML("DROP TABLE IF EXISTS hera_shard_map") @@ -119,7 +119,7 @@ func setupShardMap(t *testing.T) { func TestShardingStr(t *testing.T) { logger.GetLogger().Log(logger.Debug, "TestShardingStr function, creating tables and setting up shard map") - setupShardMap(t) + setupShardMap() logger.GetLogger().Log(logger.Debug, "TestShardingStr begin +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n") diff --git a/tests/unittest/sqlEvict/main_test.go b/tests/unittest/sqlEvict/main_test.go new file mode 100644 index 00000000..5cf84296 --- /dev/null +++ b/tests/unittest/sqlEvict/main_test.go @@ -0,0 +1,172 @@ +package main + +import ( + "context" + "database/sql" + "fmt" + "os" + "testing" + "time" + + //"github.com/paypal/hera/client/gosqldriver" + _ "github.com/paypal/hera/client/gosqldriver/tcp" /*to register the driver*/ + + "github.com/paypal/hera/tests/unittest/testutil" + "github.com/paypal/hera/utility/logger" +) + +var mx testutil.Mux +var tableName string +var max_conn float64 + +func cfg() (map[string]string, map[string]string, testutil.WorkerType) { + + appcfg := make(map[string]string) + // best to chose an "unique" port in case golang runs tests in paralel + appcfg["bind_port"] = "31002" + appcfg["log_level"] = "5" + appcfg["log_file"] = "hera.log" + appcfg["sharding_cfg_reload_interval"] = "0" + appcfg["rac_sql_interval"] = "0" + appcfg["child.executable"] = "mysqlworker" + appcfg["bind_eviction_names"] = "p" + appcfg["bind_eviction_threshold_pct"] = "50" + + appcfg["request_backlog_timeout"] = "1000" + appcfg["soft_eviction_probability"] = "100" + + opscfg := make(map[string]string) + max_conn = 25 + opscfg["opscfg.default.server.max_connections"] = fmt.Sprintf("%d", int(max_conn)) + opscfg["opscfg.default.server.log_level"] = "5" + + opscfg["opscfg.default.server.saturation_recover_threshold"] = "10" + //opscfg["opscfg.default.server.saturation_recover_throttle_rate"]= "100" + opscfg["opscfg.hera.server.saturation_recover_throttle_rate"] = "100" + // saturation_recover_throttle_rate + + return appcfg, opscfg, testutil.MySQLWorker +} + +func before() error { + fmt.Printf("before run mysql") + testutil.RunMysql("create table sleep_info (id bigint, seconds float);") + testutil.RunMysql("insert into sleep_info (id,seconds) values(10, 0.01);") + testutil.RunMysql("insert into sleep_info (id,seconds) values(100, 0.1);") + testutil.RunMysql("insert into sleep_info (id,seconds) values(1600, 2.6);") + testutil.RunMysql("insert into sleep_info (id,seconds) values(21001111, 0.1);") + testutil.RunMysql("insert into sleep_info (id,seconds) values(22001111, 0.1);") + testutil.RunMysql("insert into sleep_info (id,seconds) values(29001111, 2.9);") + out, err := testutil.RunMysql(`DELIMITER $$ +CREATE FUNCTION sleep_option (id bigint) +RETURNS float +DETERMINISTIC +BEGIN + declare dur float; + declare rv bigint; + select max(seconds) into dur from sleep_info where sleep_info.id=id; + select sleep(dur) into rv; + RETURN dur; +END$$ +DELIMITER ;`) + if err != nil { + fmt.Printf("err after run mysql " + err.Error()) + return nil + } + fmt.Printf("after run mysql " + out) // */ + return nil +} + +func TestMain(m *testing.M) { + logger.GetLogger().Log(logger.Debug, "begin 20230918kkang TestMain") + fmt.Printf("TestMain 20230918kkang\n") + os.Exit(testutil.UtilMain(m, cfg, before)) +} + +func sleepyQ(conn *sql.Conn, delayRow int) error { + stmt, err := conn.PrepareContext(context.Background(), "select * from sleep_info where ( seconds > sleep_option(?) or seconds > 0.0 )") + if err != nil { + fmt.Printf("Error preparing sleepyQ %s\n", err.Error()) + return err + } + defer stmt.Close() + rows, err := stmt.Query(delayRow) + if err != nil { + fmt.Printf("Error query sleepyQ %s\n", err.Error()) + return err + } + defer rows.Close() + return nil +} + +func simpleEvict() { + db, err := sql.Open("hera", "127.0.0.1:31002") + if err != nil { + fmt.Printf("Error db %s\n", err.Error()) + return + } + db.SetConnMaxLifetime(2 * time.Second) + db.SetMaxIdleConns(0) + db.SetMaxOpenConns(22111) + defer db.Close() + + conn, err := db.Conn(context.Background()) + if err != nil { + fmt.Printf("Error conn %s\n", err.Error()) + return + } + defer conn.Close() + sleepyQ(conn, 1600) + + for i := 0; i < int(max_conn)+1; i++ { + conn, err := db.Conn(context.Background()) + if err != nil { + fmt.Printf("Error #%d conn %s\n", i, err.Error()) + continue + } + defer conn.Close() + fmt.Printf("connected %d\n", i) + go sleepyQ(conn, 1600) + } + fmt.Printf("done with bklg setup\n") + for i := 0; i < 55; i++ { + conn, err := db.Conn(context.Background()) + if err != nil { + fmt.Printf("Error #%d conn %s\n", i, err.Error()) + continue + } + defer conn.Close() + fmt.Printf("connected %d heartbeat\n", i) + go sleepyQ(conn, 10) + time.Sleep(20 * time.Millisecond) + } + fmt.Printf("done waiting for bklg\n") +} + +var normCliErr error + +func NormCliErr() error { + if normCliErr == nil { + normCliErr = fmt.Errorf("normal client got error") + } + return normCliErr +} + +func TestSqlEvict(t *testing.T) { + logger.GetLogger().Log(logger.Debug, "TestSqlEvict begin +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n") + simpleEvict() + if testutil.RegexCountFile("HERA-100: backlog timeout", "hera.log") == 0 { + t.Fatal("backlog timeout was not triggered") + } // */ + /* if (testutil.RegexCountFile("coordinator dispatchrequest: no worker HERA-102: backlog eviction", "hera.log") == 0) { + t.Fatal("backlog eviction was not triggered") + } // */ + if testutil.RegexCountFile("coordinator dispatchrequest: no worker HERA-104: saturation soft sql eviction", "hera.log") == 0 { + t.Fatal("soft eviction was not triggered") + } + if testutil.RegexCountFile("coordinator dispatchrequest: stranded conn HERA-101: saturation kill", "hera.log") == 0 { + t.Fatal("eviction was not triggered") + } + logger.GetLogger().Log(logger.Debug, "TestSqlEvict stop +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n") + time.Sleep(2 * time.Second) +} // */ diff --git a/tests/unittest/testall.sh b/tests/unittest/testall.sh index 5b378562..92fce600 100755 --- a/tests/unittest/testall.sh +++ b/tests/unittest/testall.sh @@ -19,7 +19,10 @@ do fi if [ 0 != $rv ] then - #grep ^ *.log + echo "--- HERA_LOG ---" + grep ^ hera.log + echo "--- CAL LOG ---" + grep ^ cal.log popd #exit $rv overall=1 diff --git a/tests/unittest/testutil/main.go b/tests/unittest/testutil/main.go index 7e43e53a..9096e1ce 100644 --- a/tests/unittest/testutil/main.go +++ b/tests/unittest/testutil/main.go @@ -73,10 +73,13 @@ func UtilMain(m *testing.M, cfg cfgFunc, before beforeFunc) int { } } - code := m.Run() - teardown() - if testing.Verbose() { - saveLogs() + if m != nil { + code := m.Run() + teardown() + if testing.Verbose() { + saveLogs() + } + return code } - return code + return 0 } diff --git a/tests/unittest/testutil/regen/cmd.go b/tests/unittest/testutil/regen/cmd.go new file mode 100644 index 00000000..617a3abe --- /dev/null +++ b/tests/unittest/testutil/regen/cmd.go @@ -0,0 +1,123 @@ +/* +Copyright 2023 PayPal Inc + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +/* +Regenerates and rewrites code from the "unittest" integration tests to +go1.20 integration executables +*/ + +import ( + "bufio" + "fmt" + "os" + "strings" +) + +func main() { + if len(os.Args) <= 1 { + fmt.Printf("need cmd arg\n") + os.Exit(3) + } + cmd := os.Args[1] + fmt.Printf("cmd is %s\n",cmd) + + if cmd == "rewrite" { + if len(os.Args) <= 2 { + fmt.Printf("need path like tests/unittest/querybindblocker") + os.Exit(4) + } + path := os.Args[2] + doRewrite(path) + } +} + +func doRewrite(path string) { + // open path/main_test.go + // write path/main.go + + f, err := os.Open(os.Getenv("GOPATH")+ "/src/github.com/paypal/hera/"+ path + "/main_test.go") + if err != nil { + fmt.Printf("err opening %s/main_test.go %v\n",path, err) + os.Exit(5) + } + defer f.Close() + + // TODO may clobber existing file + fo, err := os.Create(os.Getenv("GOPATH")+ "/src/github.com/paypal/hera/"+ path + "/main.go") + if err != nil { + fmt.Printf("err create %s/main.go %v\n",path, err) + os.Exit(6) + } + defer fo.Close() + + + scnln:= bufio.NewScanner(f) + numln := 0 + fns := make([]string,0) + beforeFnName := "nil" + findBeforeFnName := "testutil.UtilMain(m, cfg, " + for scnln.Scan() { + numln += 1 + err = scnln.Err() + if err != nil { + fmt.Printf("err read line %d %s %v\n", numln, path, err) + break + } + lnstr := scnln.Text() + + /* + heartbeat doesn't wrap with os.Exit + unittest/querybindblocker/main_test.go: os.Exit(testutil.UtilMain(m, cfg, nil)) + unittest/mysql_recycle/main_test.go: os.Exit(testutil.UtilMain(m, cfg, before)) + unittest/mysql_autocommit/main_test.go: os.Exit(testutil.UtilMain(m, cfg, setupDb)) + */ + if strings.Contains(lnstr, findBeforeFnName) { + _, after, _ := strings.Cut(lnstr, findBeforeFnName) + lastIdx := len(after) + if after[lastIdx-2] == ')' { + lastIdx -= 2 + } else if after[lastIdx-1] == ')' { + lastIdx -= 1 + } + beforeFnName = after[:lastIdx] + } + + if strings.HasPrefix(lnstr, "func Test") && !strings.Contains(lnstr,"TestMain(") { + before, after, _/*found*/ := strings.Cut(lnstr, "(t *testing.T)") + fo.WriteString(before + "()" + after + "\n") + fns = append(fns, before[5:]) + } else if strings.Contains(lnstr, "t.Fatal") { + // also works for t.Fatalf + before, after, _ := strings.Cut(lnstr, "t.Fatal") + fo.WriteString(before + "testutil.Fatal" + after + "\n") + } else { + fo.WriteString(lnstr+"\n") + } + } + if err != nil { + fmt.Printf("inner look break, now exit") + os.Exit(7) + } + fo.WriteString("func main() {\n") + fo.WriteString("\ttestutil.UtilMain(nil, cfg, "+beforeFnName+")\n") + for _, fnname := range fns { + fo.WriteString("\t"+fnname+"()\n") + } + fo.WriteString("}\n") + fmt.Printf("all done %d lines and %d fn's in %s\n",numln,len(fns),path) +} diff --git a/tests/unittest/testutil/util.go b/tests/unittest/testutil/util.go index 8753b457..77317e64 100644 --- a/tests/unittest/testutil/util.go +++ b/tests/unittest/testutil/util.go @@ -81,6 +81,7 @@ func BackupAndClear(logbasename, grpName string) { bakname := "" for { bakname = fmt.Sprintf("%s%d.log", logbasename, num) + num += 1 _, err := os.Stat(bakname) if os.IsNotExist(err) { break @@ -111,8 +112,11 @@ func RunMysql(sql string) (string, error) { cmd.Stdin = strings.NewReader(sql) var cmdOutBuf bytes.Buffer cmd.Stdout = &cmdOutBuf - cmd.Run() - return cmdOutBuf.String(), nil + err := cmd.Run() + if err != nil { + logger.GetLogger().Log(logger.Debug, "RunMysql", "sql=", sql, "err=", err, "out=", cmdOutBuf.String()) + } + return cmdOutBuf.String(), err } func RunDML(dml string) error { @@ -150,6 +154,7 @@ func RunDML(dml string) error { func RegexCount(regex string) int { return RegexCountFile(regex, "hera.log") } + func RegexCountFile(regex string, filename string) int { time.Sleep(10 * time.Millisecond) fa, err := regexp.Compile(regex) @@ -218,3 +223,36 @@ func ComputeScuttleId(shardKey interface{}, maxScuttles string) (uint64, error) } return 0, errors.New(fmt.Sprintf("Failed to compute scuttle ID for shardKey: %v", shardKey)) } + +func Fatal(msg ...interface{}) { + fmt.Println(msg...) + os.Exit(2) +} + +func Fatalf(str string, msg ...interface{}) { + fmt.Printf(str, msg...) + os.Exit(1) +} + +func ClearLogsData() { + heraFile, err := os.OpenFile(runFolder+"/hera.log", os.O_WRONLY|os.O_TRUNC, 0644) + if err != nil { + fmt.Printf("failed to clear Hera log file") + return + } + defer heraFile.Close() + + statelogFile, err := os.OpenFile(runFolder+"/state.log", os.O_WRONLY|os.O_TRUNC, 0644) + if err != nil { + fmt.Printf("failed to clear Hera log file") + return + } + defer statelogFile.Close() + + calLogFile, err := os.OpenFile(runFolder+"/cal.log", os.O_WRONLY|os.O_TRUNC, 0644) + if err != nil { + fmt.Printf("failed to clear Hera log file") + return + } + defer calLogFile.Close() +} diff --git a/tests/unittest2/bindEvict/main_test.go b/tests/unittest2/bindEvict/main_test.go deleted file mode 100644 index f857dc53..00000000 --- a/tests/unittest2/bindEvict/main_test.go +++ /dev/null @@ -1,376 +0,0 @@ -package main - -import ( - "context" - "database/sql" - "fmt" - "os" - "testing" - "time" - - - //"github.com/paypal/hera/client/gosqldriver" - _ "github.com/paypal/hera/client/gosqldriver/tcp" /*to register the driver*/ - - "github.com/paypal/hera/tests/unittest/testutil" - "github.com/paypal/hera/utility/logger" - "github.com/paypal/hera/lib" -) - -var mx testutil.Mux -var tableName string -var max_conn float64 - -func cfg() (map[string]string, map[string]string, testutil.WorkerType) { - - appcfg := make(map[string]string) - // best to chose an "unique" port in case golang runs tests in paralel - appcfg["bind_port"] = "31002" - appcfg["log_level"] = "5" - appcfg["log_file"] = "hera.log" - appcfg["sharding_cfg_reload_interval"] = "0" - appcfg["rac_sql_interval"] = "0" - appcfg["child.executable"] = "mysqlworker" - appcfg["bind_eviction_names"] = "p" - - appcfg["request_backlog_timeout"]="1000"; - appcfg["soft_eviction_probability"]="100"; - - opscfg := make(map[string]string) - max_conn = 25 - opscfg["opscfg.default.server.max_connections"] = fmt.Sprintf("%d", int(max_conn)) - opscfg["opscfg.default.server.log_level"] = "5" - - - opscfg["opscfg.default.server.saturation_recover_threshold"]= "10" - //opscfg["opscfg.default.server.saturation_recover_throttle_rate"]= "100" - opscfg["opscfg.hera.server.saturation_recover_throttle_rate"]= "100" - // saturation_recover_throttle_rate - - return appcfg, opscfg, testutil.MySQLWorker -} - -func before() error { - fmt.Printf("before run mysql") - testutil.RunMysql("create table sleep_info (id bigint, seconds float)") - testutil.RunMysql("insert into sleep_info (id,seconds) values(10, 0.01)") - testutil.RunMysql("insert into sleep_info (id,seconds) values(100, 0.1)") - testutil.RunMysql("insert into sleep_info (id,seconds) values(1600, 2.6)") - testutil.RunMysql("insert into sleep_info (id,seconds) values(21001111, 0.1)") - testutil.RunMysql("insert into sleep_info (id,seconds) values(22001111, 0.1)") - testutil.RunMysql("insert into sleep_info (id,seconds) values(29001111, 2.9)") - out,err := testutil.RunMysql(`DELIMITER $$ -CREATE FUNCTION sleep_option (id bigint) -RETURNS float -DETERMINISTIC -BEGIN - declare dur float; - declare rv bigint; - select max(seconds) into dur from sleep_info where sleep_info.id=id; - select sleep(dur) into rv; - RETURN dur; -END$$ -DELIMITER ;`); - if err != nil { - fmt.Printf("err after run mysql "+err.Error()) - return nil - } - fmt.Printf("after run mysql "+out) // */ - return nil -} - -func TestMain(m *testing.M) { - os.Exit(testutil.UtilMain(m, cfg, before)) -} - - -func sleepyQ(conn *sql.Conn, delayRow int) (error) { - stmt, err := conn.PrepareContext(context.Background(), "select * from sleep_info where ( seconds > sleep_option(?) or seconds > 0.0 )") - if err != nil { - fmt.Printf("Error preparing sleepyQ %s\n", err.Error()) - return err - } - defer stmt.Close() - rows, err := stmt.Query(delayRow) - if err != nil { - fmt.Printf("Error query sleepyQ %s\n", err.Error()) - return err - } - defer rows.Close() - return nil -} - -func simpleEvict() { - db, err := sql.Open("hera", "127.0.0.1:31002") - if err != nil { - fmt.Printf("Error db %s\n", err.Error()) - return - } - db.SetConnMaxLifetime(2 * time.Second) - db.SetMaxIdleConns(0) - db.SetMaxOpenConns(22111) - defer db.Close() - - conn, err := db.Conn(context.Background()) - if err != nil { - fmt.Printf("Error conn %s\n", err.Error()) - return - } - defer conn.Close() - sleepyQ(conn, 1600) - - for i := 0; i < int(max_conn)+1 ; i++ { - conn, err := db.Conn(context.Background()) - if err != nil { - fmt.Printf("Error #%d conn %s\n", i, err.Error()) - continue - } - defer conn.Close() - fmt.Printf("connected %d\n", i) - go sleepyQ(conn,1600) - } - fmt.Printf("done with bklg setup\n") - for i := 0; i < 55 ; i++ { - conn, err := db.Conn(context.Background()) - if err != nil { - fmt.Printf("Error #%d conn %s\n", i, err.Error()) - continue - } - defer conn.Close() - fmt.Printf("connected %d heartbeat\n", i) - go sleepyQ(conn,10) - time.Sleep(20 * time.Millisecond) - } - fmt.Printf("done waiting for bklg\n") -} - -func TestSqlEvict(t *testing.T) { - logger.GetLogger().Log(logger.Debug, "TestSqlEvict begin +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n") - simpleEvict() - if (testutil.RegexCountFile("HERA-100: backlog timeout", "hera.log") == 0) { - t.Fatal("backlog timeout was not triggered") - } // */ - /* if (testutil.RegexCountFile("coordinator dispatchrequest: no worker HERA-102: backlog eviction", "hera.log") == 0) { - t.Fatal("backlog eviction was not triggered") - } // */ - if (testutil.RegexCountFile("coordinator dispatchrequest: no worker HERA-104: saturation soft sql eviction", "hera.log") == 0) { - t.Fatal("soft eviction was not triggered") - } - if (testutil.RegexCountFile("coordinator dispatchrequest: stranded conn HERA-101: saturation kill", "hera.log") == 0) { - t.Fatal("eviction was not triggered") - } - logger.GetLogger().Log(logger.Debug, "TestSqlEvict stop +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n") - time.Sleep(2*time.Second) -} - -func fastAndSlowBinds() (error) { - db, err := sql.Open("hera", "127.0.0.1:31002") - if err != nil { - fmt.Printf("Error db %s\n", err.Error()) - return err - } - db.SetConnMaxLifetime(22 * time.Second) - db.SetMaxIdleConns(0) - db.SetMaxOpenConns(22111) - defer db.Close() - - // client threads of slow queries - var stop2 int - var badCliErr string - mkClients(1+int(max_conn*1.6), &stop2, 29001111, "badClient", &badCliErr, db) - time.Sleep(3100 * time.Millisecond) - /* if (testutil.RegexCountFile("BIND_THROTTLE", "cal.log") == 0) { - return fmt.Errorf("BIND_THROTTLE was not triggered") - } - if (testutil.RegexCountFile("BIND_EVICT", "cal.log") == 0) { - return fmt.Errorf("BIND_EVICT was not triggered") - } // */ - - // start normal clients after initial backlog timeouts - var normCliErrStr string - var stop int - mkClients(1, &stop, 21001111, "n client", &normCliErrStr, db) - time.Sleep(1100 * time.Millisecond) - - // if we throttle down or stop, it restores - stop2 = 1 // stop bad clients - lib.GetConfig().BindEvictionDecrPerSec = 11500.1 - defer func(){ lib.GetConfig().BindEvictionDecrPerSec = 1.1 }() - time.Sleep(1 * time.Second) - conn, err := db.Conn(context.Background()) - if err != nil { - fmt.Printf("Error conn %s\n", err.Error()) - return err - } - defer conn.Close() - err = sleepyQ(conn,29001111) - if err != nil { - msg := fmt.Sprintf("test failed, throttle down didn't restore") - fmt.Printf("%s", msg) - return fmt.Errorf("%s", msg) - } - - stop = 1 - if len(normCliErrStr) != 0 { - return NormCliErr() - } - return nil -} -var normCliErr error -func NormCliErr() (error) { - if normCliErr == nil { - normCliErr = fmt.Errorf("normal client got error") - } - return normCliErr -} - - -func partialBadLoad(fracBad float64) (error) { - db, err := sql.Open("hera", "127.0.0.1:31002") - if err != nil { - fmt.Printf("Error db %s\n", err.Error()) - return err - } - db.SetConnMaxLifetime(111 * time.Second) - db.SetMaxIdleConns(0) - db.SetMaxOpenConns(22111) - defer db.Close() - - - // client threads of slow queries - var stop2 int - var stop3 int - var badCliErr string - var cliErr string - numBad := int(max_conn*fracBad) - numNorm := int(max_conn*2.1)+1 - numBad - fmt.Printf("spawning clients bad%d norm%d\n", numBad, numNorm) - mkClients(numBad, &stop2, 29001111, "badClient", &badCliErr, db) - mkClients(numNorm, &stop3, 100, "normClient", &cliErr, db) // bind value is short, so bindevict won't trigger - time.Sleep(3100 * time.Millisecond) - //time.Sleep(33100 * time.Millisecond) - - // start normal clients after initial backlog timeouts - var stop int - var normCliErrStr string - mkClients(1, &stop, 21001111, "n client", &normCliErrStr, db) - time.Sleep(1100 * time.Millisecond) - - // if we throttle down or stop, it restores - stop2 = 1 // stop bad clients - stop3 = 1 - lib.GetConfig().BindEvictionDecrPerSec = 11333.1 - defer func(){lib.GetConfig().BindEvictionDecrPerSec = 1.1 }() - time.Sleep(1 * time.Second) - conn, err := db.Conn(context.Background()) - if err != nil { - fmt.Printf("Error conn %s\n", err.Error()) - return err - } - defer conn.Close() - err = sleepyQ(conn,29001111) - if err != nil { - msg := fmt.Sprintf("test failed, throttle down didn't restore") - fmt.Printf("%s", msg) - return fmt.Errorf("%s", msg) - } - - stop = 1 - // tolerate soft eviction on normal client when we did not use bind eviction - if len(normCliErrStr) != 0 { - return NormCliErr() - } // */ - return nil -} - -func mkClients(num int, stop *int, bindV int, grpName string, outErr *string, db *sql.DB) { - for i := 0 ; i < num ; i++ { - go func (clientId int) { - count := 0 - var conn *sql.Conn - var err error - var curErr string - for *stop == 0 { - nowStr := time.Now().Format("15:04:05.000000 ") - if conn == nil { - conn, err = db.Conn(context.Background()) - fmt.Printf(grpName+" connected %d\n", clientId) - if err != nil { - fmt.Printf(nowStr + grpName+" Error %d conn %s\n", clientId, err.Error()) - time.Sleep(7 * time.Millisecond) - continue - } - } - - fmt.Printf(nowStr + grpName+"%d loop%d %s\n", clientId, count, time.Now().Format("20060102j150405.000000")) - err := sleepyQ(conn,bindV) - if err != nil { - if err.Error() == curErr { - fmt.Printf(nowStr +grpName+"%d same err twice\n", clientId) - conn.Close() - conn = nil - } else { - curErr = err.Error() - *outErr = curErr - fmt.Printf(nowStr + grpName +"%d err %s\n", clientId, curErr) - } - } - count++ - time.Sleep(10 * time.Millisecond) - } - fmt.Printf(time.Now().Format("15:04:05.000000 ") + grpName+"%d END loop%d\n", clientId, count) - }(i) - } -} - - -func TestBindEvict(t *testing.T) { - // we would like to clear hera.log, but even if we try, lots of messages still go there - logger.GetLogger().Log(logger.Debug, "TestBindEvict +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n") - err := fastAndSlowBinds() - if err != nil { - t.Fatalf("main step function returned err %s", err.Error()) - } - if (testutil.RegexCountFile("BIND_THROTTLE", "cal.log") == 0) { - t.Fatalf("BIND_THROTTLE was not triggered") - } - if (testutil.RegexCountFile("BIND_EVICT", "cal.log") == 0) { - t.Fatalf("BIND_EVICT was not triggered") - } - logger.GetLogger().Log(logger.Debug, "TestBindEvict stop +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n") -} - -func TestBindLess(t *testing.T) { - // we would like to clear hera.log, but even if we try, lots of messages still go there - logger.GetLogger().Log(logger.Debug, "TestBindLess +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n") - testutil.BackupAndClear("cal", "BindLess start") - testutil.BackupAndClear("hera", "BindLess start") - err := partialBadLoad(0.10) - if err != nil && err != NormCliErr() { - t.Fatalf("main step function returned err %s", err.Error()) - } - if (testutil.RegexCountFile("BIND_THROTTLE", "cal.log") > 0) { - t.Fatalf("BIND_THROTTLE should not trigger") - } - if (testutil.RegexCountFile("BIND_EVICT", "cal.log") > 0) { - t.Fatalf("BIND_EVICT should not trigger") - } - if (testutil.RegexCountFile("HERA-10", "hera.log") == 0) { - t.Fatal("backlog timeout or saturation was not triggered") - } // */ - - if true { - logger.GetLogger().Log(logger.Debug, "TestBindLess midpt +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n") - err := partialBadLoad(0.3) - if err != nil { - // t.Fatalf("main step function returned err %s", err.Error()) // can be triggered since test only has one sql - } - if (testutil.RegexCountFile("BIND_THROTTLE", "cal.log") == 0) { - t.Fatalf("BIND_THROTTLE should trigger") - } - if (testutil.RegexCountFile("BIND_EVICT", "cal.log") == 0) { - t.Fatalf("BIND_EVICT should trigger") - } - } // endif - logger.GetLogger().Log(logger.Debug, "TestBindLess done +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n") -} // */ diff --git a/tests/unittest2/oracleHighLoadAdj/cal_client.cdb b/tests/unittest3/oracleHighLoadAdj/cal_client.cdb similarity index 100% rename from tests/unittest2/oracleHighLoadAdj/cal_client.cdb rename to tests/unittest3/oracleHighLoadAdj/cal_client.cdb diff --git a/tests/unittest2/oracleHighLoadAdj/cal_client.txt.sample b/tests/unittest3/oracleHighLoadAdj/cal_client.txt.sample similarity index 100% rename from tests/unittest2/oracleHighLoadAdj/cal_client.txt.sample rename to tests/unittest3/oracleHighLoadAdj/cal_client.txt.sample diff --git a/tests/unittest2/oracleHighLoadAdj/cdbmake.py b/tests/unittest3/oracleHighLoadAdj/cdbmake.py similarity index 100% rename from tests/unittest2/oracleHighLoadAdj/cdbmake.py rename to tests/unittest3/oracleHighLoadAdj/cdbmake.py diff --git a/tests/unittest2/oracleHighLoadAdj/cmd/rude.go b/tests/unittest3/oracleHighLoadAdj/cmd/rude.go similarity index 100% rename from tests/unittest2/oracleHighLoadAdj/cmd/rude.go rename to tests/unittest3/oracleHighLoadAdj/cmd/rude.go diff --git a/tests/unittest2/oracleHighLoadAdj/hera.txt.sample b/tests/unittest3/oracleHighLoadAdj/hera.txt.sample similarity index 100% rename from tests/unittest2/oracleHighLoadAdj/hera.txt.sample rename to tests/unittest3/oracleHighLoadAdj/hera.txt.sample diff --git a/tests/unittest2/oracleHighLoadAdj/main_test.go b/tests/unittest3/oracleHighLoadAdj/main_test.go similarity index 88% rename from tests/unittest2/oracleHighLoadAdj/main_test.go rename to tests/unittest3/oracleHighLoadAdj/main_test.go index a6a3e63a..5112c719 100644 --- a/tests/unittest2/oracleHighLoadAdj/main_test.go +++ b/tests/unittest3/oracleHighLoadAdj/main_test.go @@ -141,10 +141,12 @@ func cfg() (map[string]string, map[string]string, testutil.WorkerType) { } func before() error { + fmt.Printf("before function") return nil } func TestMain(m *testing.M) { + fmt.Printf("TestMain function") os.Exit(testutil.UtilMain(m, cfg, before)) } @@ -158,6 +160,7 @@ func mkConn(t *testing.T, db *sql.DB) (*sql.Conn, context.CancelFunc) { } func TestBadPassword(t *testing.T) { + fmt.Printf("badPass disabled function") return logger.GetLogger().Log(logger.Debug, "TestBadPassword +++++++++++++") @@ -201,6 +204,7 @@ func TestBadPassword(t *testing.T) { } func TestLimitConcurrentInit(t *testing.T) { + fmt.Printf("T Lim Concur Init function") logger.GetLogger().Log(logger.Debug, "TestLimitConcurrentInit +++++++++++++") if max_conn < 15 { t.Error("max_conn likely too low to see TestLimitConcurrentInit") @@ -211,6 +215,8 @@ func TestLimitConcurrentInit(t *testing.T) { } func TestSkipOciBreak(t *testing.T) { + logMsg := "" + fmt.Printf("Skip OCI break function") logger.GetLogger().Log(logger.Debug, "TestSkipOciBreak +++++++++++++") hostname, _ := os.Hostname() fmt.Println("Hostname: ", hostname) @@ -219,14 +225,17 @@ func TestSkipOciBreak(t *testing.T) { t.Fatal("Error db conn", err) return } + logMsg = "db conn ok"; fmt.Printf(logMsg); logger.GetLogger().Log(logger.Debug, logMsg) db.SetMaxIdleConns(0) defer db.Close() conn, _ := mkConn(t, db) defer conn.Close() + logMsg = "pre tbl rm"; fmt.Printf(logMsg); logger.GetLogger().Log(logger.Debug, logMsg) execSql(t, conn, "delete from resilience_at_load", false) // execSql() commits, mux releases conn + logMsg = "add load"; fmt.Printf(logMsg); logger.GetLogger().Log(logger.Debug, logMsg) // simulate high load numConn := 6 stuckConn := make([]*sql.Conn, numConn) @@ -236,11 +245,13 @@ func TestSkipOciBreak(t *testing.T) { stuckConn[i] = c stuckTx[i] = execSql(t, c, fmt.Sprintf("insert into resilience_at_load(id,note)values(%d,'stuckConn')", 1000+i), true) } + logMsg = "add load done"; fmt.Printf(logMsg); logger.GetLogger().Log(logger.Debug, logMsg) time.Sleep(1000*time.Millisecond) // helper starts sql and rudely stops // first with insert which stays in wait state for the client // we want to keep current behavior + logMsg = "doing rude.go"; fmt.Printf(logMsg); logger.GetLogger().Log(logger.Debug, logMsg) out, err := exec.Command(os.Getenv("GOROOT")+"/bin/go", "run", "cmd/rude.go", "insert").Output() if err != nil { fmt.Printf("go run rude.go - output %s", out) @@ -249,12 +260,14 @@ func TestSkipOciBreak(t *testing.T) { time.Sleep(50*time.Millisecond) // check for behavior we want + logMsg = "pre chk"; fmt.Printf(logMsg); logger.GetLogger().Log(logger.Debug, logMsg) if testutil.RegexCountFile("is high load, skipping", "hera.log") != 0 { t.Fatal("skip oci break, on waiting db txn") } // slow query, client timesout/crashes + logMsg = "add slow clients"; fmt.Printf(logMsg); logger.GetLogger().Log(logger.Debug, logMsg) out, err = exec.Command(os.Getenv("GOROOT")+"/bin/go", "run", "cmd/rude.go", "usleep").Output() if err != nil { fmt.Printf("go run rude.go - output %s", out) @@ -263,16 +276,19 @@ func TestSkipOciBreak(t *testing.T) { time.Sleep(50*time.Millisecond) // check for behavior we want + logMsg = "pre chk, slow"; fmt.Printf(logMsg); logger.GetLogger().Log(logger.Debug, logMsg) if testutil.RegexCountFile("is high load, skipping", "hera.log") < 1 { t.Fatal("Error did not skip oci break") } // start restore // release stuckConn + logMsg = "pre restore"; fmt.Printf(logMsg); logger.GetLogger().Log(logger.Debug, logMsg) for i := 0; i < numConn; i++ { stuckTx[i].Rollback() stuckConn[i].Close() } + logMsg = "chk restore"; fmt.Printf(logMsg); logger.GetLogger().Log(logger.Debug, logMsg) acpt4 := 0 for i := 0; i < 33; i++ { acpt4, _ = testutil.StatelogGetField(2) @@ -285,6 +301,7 @@ func TestSkipOciBreak(t *testing.T) { if int(max_conn) != acpt4 { t.Fatal("conn's did not restore") } + logMsg = "skip oci break test done"; fmt.Printf(logMsg); logger.GetLogger().Log(logger.Debug, logMsg) logger.GetLogger().Log(logger.Debug, "TestSkipOciBreak +++++++++++++ done") } diff --git a/tests/unittest2/oracleHighLoadAdj/occ.cdb b/tests/unittest3/oracleHighLoadAdj/occ.cdb similarity index 100% rename from tests/unittest2/oracleHighLoadAdj/occ.cdb rename to tests/unittest3/oracleHighLoadAdj/occ.cdb diff --git a/tests/unittest3/oracleHighLoadAdj/setup.sql b/tests/unittest3/oracleHighLoadAdj/setup.sql new file mode 100755 index 00000000..8a71288f --- /dev/null +++ b/tests/unittest3/oracleHighLoadAdj/setup.sql @@ -0,0 +1,37 @@ +create or replace function cur_micros +return number +is + rv number; + upper number; +begin + select to_number(to_char(current_timestamp,'SSFF')) into rv from dual; + select to_number(to_char(current_timestamp,'MI')) into upper from dual; + rv := rv + 1000000 * 60 * upper; + -- adding hh24 overflows + return rv; +end; +/ +select cur_micros() from dual; +select cur_micros() as chkStmtSpeed from dual; +create or replace function usleep (micros in number) +return number +is + finish number; + cur number; +begin + cur := cur_micros(); + finish := cur + micros; + while cur < finish loop + cur := cur_micros(); + end loop; + return cur-finish+micros; +end; +/ +select current_timestamp from dual; +select usleep(2111000) from dual; +select current_timestamp from dual; +create public synonym usleep for usleep; +--grant execute on usleep to app; + +create table resilience_at_load ( id number, note varchar2(333) ); +create public synonym resilience_at_load for resilience_at_load; diff --git a/tests/unittest2/oracleHighLoadAdj/version.cdb b/tests/unittest3/oracleHighLoadAdj/version.cdb similarity index 100% rename from tests/unittest2/oracleHighLoadAdj/version.cdb rename to tests/unittest3/oracleHighLoadAdj/version.cdb diff --git a/tests/unittest3/testall.sh b/tests/unittest3/testall.sh new file mode 100755 index 00000000..07d44757 --- /dev/null +++ b/tests/unittest3/testall.sh @@ -0,0 +1,77 @@ +set -e # exit on cmd return +if [ x$GOPATH = x ] +then + # when running in github actions workflow + export GOPATH=$PWD/testrun + mkdir -p testrun/src/github.com/paypal + ln -s $PWD testrun/src/github.com/paypal/hera +fi + +sudo apt install libboost-regex-dev -y +wget -nv https://download.oracle.com/otn_software/linux/instantclient/1919000/instantclient-basiclite-linux.x64-19.19.0.0.0dbru.zip https://download.oracle.com/otn_software/linux/instantclient/1920000/instantclient-sqlplus-linux.x64-19.20.0.0.0dbru.zip +curl -O https://download.oracle.com/otn_software/linux/instantclient/1919000/instantclient-sdk-linux.x64-19.19.0.0.0dbru.zip +echo 409b867f76c701ccba47f9278363b204137fc92444c317b36b60da35669453a99bd02a3c84b1b9b92c54fd94929a0eff instantclient-sqlplus-linux.x64-19.20.0.0.0dbru.zip >> SHA384 +echo bb68094a12e754fc633874e8c2b4c4d38a45a65a5a536195d628d968fca72d7a5006a62a7b1fdd92a29134a06605d2b4 instantclient-basiclite-linux.x64-19.19.0.0.0dbru.zip >> SHA384 +echo 5999f2333a9b73426c7af589ab13480f015c2cbd82bb395c7347ade37cc7040a833a398e9ce947ae2781365bd3a2e371 instantclient-sdk-linux.x64-19.19.0.0.0dbru.zip >> SHA384 +sha384sum -c SHA384 +pubdir=$PWD + +pushd /opt +mkdir instantclient_19 +ln -s instantclient_19 instantclient_19_17 +ln -s instantclient_19 instantclient_19_19 +ln -s instantclient_19 instantclient_19_20 +unzip $pubdir/instantclient-basiclite-linux.x64-19.19.0.0.0dbru.zip +unzip $pubdir/instantclient-sdk-linux.x64-19.19.0.0.0dbru.zip +unzip $pubdir/instantclient-sqlplus-linux.x64-19.20.0.0.0dbru.zip +popd + +export ORACLE_HOME=/opt/instantclient_19 +mkdir -p $ORACLE_HOME/network/admin +echo 'TEST3=(DESCRIPTION=(ADDRESS_LIST=(ADDRESS=(PROTOCOL=TCP)(Host=localhost)(PORT=1521))(CONNECT_DATA=(SERVICE_NAME=XEPDB1)(FAILOVER_MODE=(TYPE=SESSION)(METHOD=BASIC)(RETRIES=1000)(DELAY=5)))))' > $ORACLE_HOME/network/admin/tnsnames.ora +find $ORACLE_HOME/network -ls +export TWO_TASK=TEST3 +export TNS_ADMIN=./ +export OPS_CFG_FILE=occ.cdb +export username=system +export password=1.2.8MomOfferExpand +if [ x$LD_LIBRARY_PATH = x ] +then + export LD_LIBRARY_PATH=$ORACLE_HOME +else + export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$ORACLE_HOME +fi + +# make oracle worker +pushd worker/cppworker/worker +make -f ../build/makefile19 -j 3 +mkdir -p $GOPATH/bin +cp -v oracleworker $GOPATH/bin/ +popd + + +# run test with oracle +overall=0 +for d in `ls $GOPATH/src/github.com/paypal/hera/tests/unittest3 | grep -vE '(testall.sh)'` +do + pushd $GOPATH/src/github.com/paypal/hera/tests/unittest3/$d + if [ -f setup.sql ] + then + cat setup.sql |$ORACLE_HOME/sqlplus $username/$password@$TWO_TASK + fi + cp -v $GOPATH/bin/oracleworker . + $GOROOT/bin/go test -c . + ./$d.test -test.v | tee /dev/null + rv=$? + if [ 0 != $rv ] + then + echo $d failing $d + grep ^ *.log + overall=$rv + fi + tail *.log + echo $d test done $rv + grep -E '(FAIL([^O]|$)|PASS)' -A1 *.log + popd +done +exit $overall diff --git a/worker/cppworker/worker/OCCChild.cpp b/worker/cppworker/worker/OCCChild.cpp index b46b2eb0..91da8a18 100644 --- a/worker/cppworker/worker/OCCChild.cpp +++ b/worker/cppworker/worker/OCCChild.cpp @@ -1098,7 +1098,7 @@ int OCCChild::handle_command(const int _cmd, std::string &_line) } } } - } + } c->SetStatus(CAL::TRANS_OK); // SQL errors are logged to CAL separately delete c; c = NULL; @@ -1330,32 +1330,29 @@ int OCCChild::handle_command(const int _cmd, std::string &_line) m_writer->write(OCC_MARKDOWN); break; } - //send server Info - std::string server_info = CalTransaction::GetCurrentPoolInfo(); - - m_writer->add(OCC_OK, server_info); - eor(is_in_transaction() ? EORMessage::IN_TRANSACTION : EORMessage::FREE); - m_writer->write(); - - WRITE_LOG_ENTRY(logfile, LOG_VERBOSE, "Client info: %s", _line.c_str()); - WRITE_LOG_ENTRY(logfile, LOG_VERBOSE, "Server Info: %s", server_info.c_str()); - //m_writer->write(OCC_OK); // Make client to proceed. - - - // Set the client info only if it's not already set - if(client_info.empty()) + std::string client_info = _line; + if (client_info.length() == 0) { - client_info = _line; - process_pool_info(client_info); + client_info = "unknown"; + } + WRITE_LOG_ENTRY(logfile, LOG_VERBOSE, "Client info: %s", client_info.c_str()); + CalEvent e(CAL::EVENT_TYPE_CLIENT_INFO, client_info, CAL::TRANS_OK); + /* std::string poolStack = _line; + std::string client_info; - unsigned int last_idx = client_info.rfind(CLIENT_NAME_PREFIX); - if (last_idx != std::string::npos) - { - m_client_name.clear(); // Clear previous data if-any, little paranoia. - m_client_name = client_info.substr(last_idx + CLIENT_NAME_PREFIX.length()); - StringUtil::trim(m_client_name); // Remove any white-spaces - } + StringUtil::tokenize(poolStack, client_info, '|'); + if (client_info.length() == 0) + { + client_info = "unknown"; + } + WRITE_LOG_ENTRY(logfile, LOG_VERBOSE, "Client info: %s | poolStack: %s", client_info.c_str(), poolStack.c_str()); + CalEvent e(CAL::EVENT_TYPE_CLIENT_INFO, client_info, CAL::TRANS_OK); + if (CalClient::is_poolstack_enabled()) { + WRITE_LOG_ENTRY(logfile, LOG_DEBUG, "set poolStack: %s", poolStack.c_str()); + CalTransaction::SetParentStack(poolStack, std::string("CLIENT_INFO")); } + e.AddPoolStack(); */ + // CalEvent e(CAL::EVENT_TYPE_CLIENT_INFO, client_info, CAL::TRANS_OK, poolStack); if (cur_stmt != NULL) CalEvent e(CAL::EVENT_TYPE_MESSAGE, "CLIENT_INFO_IN_TXN", "0"); @@ -1381,9 +1378,14 @@ int OCCChild::handle_command(const int _cmd, std::string &_line) OCIAttrSet((dvoid *)authp, OCI_HTYPE_SESSION, (dvoid *) const_cast(m_corr_id.c_str()), m_corr_id.length(), OCI_ATTR_ACTION, errhp); if (cur_stmt != NULL) - CalEvent e(CAL::EVENT_TYPE_MESSAGE, "CORRID_IN_TXN", "0"); - else if (!is_in_transaction()) - set_dedicated(false); // this command has no response, so setting this flag here + CalEvent e(CAL::EVENT_TYPE_MESSAGE, "CORRID_IN_TXN", "0"); + + // 325:0 18:2006 CorrId=NotSet,22:11 ClientApp&PoolStack,18:2006 CorrId=NotSet,170:25 /*shard=0*/ SELECT inst_id...0 + // Removing this dedicated flag reset -- If worker receives the above command -- this below flag will reset the dedicated flag when it encounters the second corrId. + // This will create additional CLIENT_SESSION for the same request. + // This was not a problem earlier because the session will start during the prepare command, whereas, now it begins as part of the CLIENT_INFO command. + // else if (!is_in_transaction()) + // set_dedicated(false); // this command has no response, so setting this flag here break; diff --git a/worker/cppworker/worker/Worker.cpp b/worker/cppworker/worker/Worker.cpp index 6d2384c1..32da8bbb 100644 --- a/worker/cppworker/worker/Worker.cpp +++ b/worker/cppworker/worker/Worker.cpp @@ -443,6 +443,8 @@ void Worker::run() client_session.start_session("API", m_cal_client_session_name); client_session.get_session_transaction()->AddData("worker_pid", getpid()); client_session.get_session_transaction()->AddData("sid", m_sid); + client_session.get_session_transaction()->AddData("rac_id", m_connected_id); + client_session.get_session_transaction()->AddData("db_uname", m_db_uname); client_session.set_status(CAL::TRANS_OK); // internal queries' error overwrite status so reset it. } WRITE_LOG_ENTRY(logfile, LOG_VERBOSE, "Session started"); diff --git a/worker/mysqlworker/adapter.go b/worker/mysqlworker/adapter.go index 653854f3..5d98e58d 100644 --- a/worker/mysqlworker/adapter.go +++ b/worker/mysqlworker/adapter.go @@ -92,16 +92,22 @@ func (adapter *mysqlAdapter) InitDB() (*sql.DB, error) { } db.Close() } - attempt = attempt + 1 + // Adding additional logging for error scenario + if err != nil { + if logger.GetLogger().V(logger.Warning) { + logger.GetLogger().Log(logger.Warning, fmt.Sprintf("Failed to connect datasource=%s, retry-attempt=%d and error=%v", curDs, attempt, err.Error())) + } + } // read only connection if logger.GetLogger().V(logger.Warning) { - logger.GetLogger().Log(logger.Warning, "recycling, got read-only conn " /*+curDs*/ +fmt.Sprintf("retry-attempt=%d", attempt-1)) + logger.GetLogger().Log(logger.Warning, "recycling, got read-only conn " /*+curDs*/ +fmt.Sprintf("retry-attempt=%d", attempt)) } - evt := cal.NewCalEvent("INITDB", "RECYCLE_ON_READ_ONLY", cal.TransWarning, fmt.Sprintf("retry-attempt=%d", attempt-1)) + evt := cal.NewCalEvent("INITDB", "RECYCLE_ON_READ_ONLY", cal.TransWarning, fmt.Sprintf("retry-attempt=%d", attempt)) evt.SetStatus(cal.TransError) evt.Completed() + attempt = attempt + 1 pwdStr := fmt.Sprintf("password%d", attempt) pass = os.Getenv(pwdStr) } @@ -113,12 +119,13 @@ func (adapter *mysqlAdapter) InitDB() (*sql.DB, error) { err = errors.New("cannot use read-only conn " + curDs) } if is_writable { + calTrans.SetStatus(cal.TransOK) //Reset the URL status to success break } } calTrans.Completed() if err != nil { - spread := 11 * time.Second + time.Duration(rand.Intn(11000999888)/*ns*/) + spread := 11*time.Second + time.Duration(rand.Intn(2000999888) /*ns*/) logger.GetLogger().Log(logger.Warning, "onErr sleeping "+spread.String()) time.Sleep(spread) } diff --git a/worker/shared/cmdprocessor.go b/worker/shared/cmdprocessor.go index 1388825c..8871da0d 100644 --- a/worker/shared/cmdprocessor.go +++ b/worker/shared/cmdprocessor.go @@ -156,6 +156,8 @@ type CmdProcessor struct { lastErr error // the FNV hash of the SQL, for logging sqlHash uint32 + // corr_id for logging + m_corr_id string // the name of the cal TXN calSessionTxnName string heartbeat bool @@ -207,11 +209,56 @@ func (cp *CmdProcessor) ProcessCmd(ns *netstring.Netstring) error { outloop: switch ns.Cmd { case common.CmdClientCalCorrelationID: - // - // @TODO parse out correlationid. - // - if cp.calSessionTxn != nil { - cp.calSessionTxn.SetCorrelationID("@todo") + // The below parsing expects the value to be of the format: CorrId=Val&IgnoreTail + // Usually, CorrId is at the beginning of the request. For now, minimal parsing should be sufficient. + // To-do: Do a full parse if required + if logger.GetLogger().V(logger.Verbose) { + logger.GetLogger().Log(logger.Verbose, "CmdClientCalCorrelationID:", string(ns.Payload), string(ns.Serialized)) + } + cp.m_corr_id = "unset" + if ns != nil { + cid := string(ns.Payload) + pos := strings.Index(cid, "&") + if pos != -1 { + cid = cid[:pos] + } + pos = strings.Index(cid, "=") + if pos != -1 && strings.Compare(cid[:pos], "CorrId") == 0 { + if len(cid[pos+1:]) > 0 && len(cid[pos+1:]) <= 32 { + cp.m_corr_id = cid[pos+1:] + logger.GetLogger().Log(logger.Verbose, "corr_id_=",cp.m_corr_id) + } else { + logger.GetLogger().Log(logger.Verbose, "CmdClientCalCorrelationID: corrid not in expected format:", string(ns.Payload)) + } + } else { + logger.GetLogger().Log(logger.Verbose, "CmdClientCalCorrelationID: Payload not in expected format:", string(ns.Payload)) + } + } + case common.CmdClientInfo: + if logger.GetLogger().V(logger.Verbose) { + logger.GetLogger().Log(logger.Verbose, "CmdClientInfo:", string(ns.Payload), string(ns.Serialized)) + } + if len(string(ns.Payload)) > 0 { + logger.GetLogger().Log(logger.Verbose, "len clientApplication:", len(string(ns.Payload))) + logger.GetLogger().Log(logger.Verbose, "clientApplication:", string(ns.Payload)) + // splits := strings.Split(string(ns.Payload), "|") + // if (len(splits) == 2) { + // logger.GetLogger().Log(logger.Verbose, "len clientApplication:", len(splits[0])) + // logger.GetLogger().Log(logger.Verbose, "len poolStack:", len(splits[1])) + // if len(splits[0]) == 0 { + // logger.GetLogger().Log(logger.Verbose, "clientApplication: unknown") + // } else { + // logger.GetLogger().Log(logger.Verbose, "clientApplication:", splits[0]) + // } + // logger.GetLogger().Log(logger.Verbose, "poolStack:", splits[1]) + // // + // // @TODO Add CLIENT_INFO event inside calSessionTxn + // // + // } else { + // logger.GetLogger().Log(logger.Debug, "CmdClientInfo: Payload not in expected Client&PoolStack format:", string(ns.Payload)) + // } + } else { + logger.GetLogger().Log(logger.Verbose, "clientApplication: unknown") } case common.CmdPrepare, common.CmdPrepareV2, common.CmdPrepareSpecial: cp.dedicated = true @@ -237,7 +284,9 @@ outloop: cp.hasResult, startTrans = cp.sqlParser.Parse(sqlQuery) if cp.calSessionTxn == nil { cp.calSessionTxn = cal.NewCalTransaction(cal.TransTypeAPI, cp.calSessionTxnName, cal.TransOK, "", cal.DefaultTGName) + cp.calSessionTxn.AddDataStr("corrid", cp.m_corr_id) } + cp.m_corr_id = "unset" // Reset after logging cp.calSessionTxn.SendSQLData(string(ns.Payload)) cp.sqlHash = utility.GetSQLHash(string(ns.Payload)) cp.queryScope.SqlHash = fmt.Sprintf("%d", cp.sqlHash) @@ -628,9 +677,9 @@ outloop: if writeCols[i].Valid { outstr = cp.adapter.ProcessResult(cts[i].DatabaseTypeName(), writeCols[i].String) } - if logger.GetLogger().V(logger.Debug) { + /* if logger.GetLogger().V(logger.Debug) { logger.GetLogger().Log(logger.Debug, "query result", outstr) - } + } // causes high volume logs and can cause CI test issues */ nss = append(nss, netstring.NewNetstringFrom(common.RcValue, []byte(outstr))) } }