pkg/asynq
2
0
Fork 0
mirror of https://github.com/hibiken/asynq.git synced 2026-05-11 06:10:10 +08:00
Code Issues Projects Releases Wiki Activity

Extract redis logic to type rdb

Browse Source
This commit is contained in:
Ken Hibino
2019-11-19 19:44:41 -08:00
parent 4c5b6081de
commit 85a04cbabb
4 changed files with 204 additions and 132 deletions
Download Patch File Download Diff File Expand all files Collapse all files

44
manager.go
View File

@@ -1,16 +1,15 @@
package asynq
import (
"encoding/json"
"fmt"
"log"
"math"
"math/rand"
"time"
"github.com/go-redis/redis/v7"
)
type manager struct {
rdb *redis.Client
rdb *rdb
handler TaskHandler
@@ -22,7 +21,7 @@ type manager struct {
done chan struct{}
}
func newManager(rdb *redis.Client, numWorkers int, handler TaskHandler) *manager {
func newManager(rdb *rdb, numWorkers int, handler TaskHandler) *manager {
return &manager{
rdb: rdb,
handler: handler,
@@ -63,22 +62,21 @@ func (m *manager) start() {
func (m *manager) processTasks() {
// pull message out of the queue and process it
// TODO(hibiken): sort the list of queues in order of priority
res, err := m.rdb.BLPop(5*time.Second, listQueues(m.rdb)...).Result() // NOTE: BLPOP needs to time out because if case a new queue is added.
msg, err := m.rdb.bpop(5*time.Second, m.rdb.listQueues()...)
if err != nil {
if err != redis.Nil {
log.Printf("BLPOP command failed: %v\n", err)
switch err {
case errQueuePopTimeout:
// timed out, this is a normal behavior.
return
case errDeserializeTask:
log.Println("[Servere Error] could not parse json encoded message")
return
default:
log.Printf("[Servere Error] unexpected error while pulling message out of queues: %v\n", err)
return
}
return
}
q, data := res[0], res[1]
fmt.Printf("perform task %v from %s\n", data, q)
var msg taskMessage
err = json.Unmarshal([]byte(data), &msg)
if err != nil {
log.Printf("[Servere Error] could not parse json encoded message %s: %v", data, err)
return
}
t := &Task{Type: msg.Type, Payload: msg.Payload}
m.sema <- struct{}{} // acquire token
go func(task *Task) {
@@ -87,7 +85,7 @@ func (m *manager) processTasks() {
if err != nil {
if msg.Retried >= msg.Retry {
fmt.Println("Retry exhausted!!!")
if err := kill(m.rdb, &msg); err != nil {
if err := m.rdb.kill(msg); err != nil {
log.Printf("[SERVER ERROR] could not add task %+v to 'dead' set\n", err)
}
return
@@ -97,7 +95,7 @@ func (m *manager) processTasks() {
fmt.Printf("[DEBUG] retying the task in %v\n", retryAt.Sub(time.Now()))
msg.Retried++
msg.ErrorMsg = err.Error()
if err := zadd(m.rdb, retry, float64(retryAt.Unix()), &msg); err != nil {
if err := m.rdb.zadd(retry, float64(retryAt.Unix()), msg); err != nil {
// TODO(hibiken): Not sure how to handle this error
log.Printf("[SEVERE ERROR] could not add msg %+v to 'retry' set: %v\n", msg, err)
return
@@ -105,3 +103,11 @@ func (m *manager) processTasks() {
}
}(t)
}
// delaySeconds returns a number seconds to delay before retrying.
// Formula taken from https://github.com/mperham/sidekiq.
func delaySeconds(count int) time.Duration {
r := rand.New(rand.NewSource(time.Now().UnixNano()))
s := int(math.Pow(float64(count), 4)) + 15 + (r.Intn(30) * (count + 1))
return time.Duration(s) * time.Second
}