v0.18.4

CHANGELOG.md

@@ -7,6 +7,29 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.18.4] - 2020-08-17
+
+### Fixed
+
+- Scheduler methods are now thread-safe. It's now safe to call `Register` and `Unregister` concurrently.
+
+## [0.18.3] - 2020-08-09
+
+### Changed
+
+- `Client.Enqueue` no longer enqueues tasks with empty typename; Error message is returned.
+
+## [0.18.2] - 2020-07-15
+
+### Changed
+
+- Changed `Queue` function to not to convert the provided queue name to lowercase. Queue names are now case-sensitive.
+- `QueueInfo.MemoryUsage` is now an approximate usage value.
+
+### Fixed
+
+- Fixed latency issue around memory usage (see https://github.com/hibiken/asynq/issues/309).
+
 ## [0.18.1] - 2020-07-04
 
 ### Changed

README.md

@@ -28,8 +28,8 @@ Task queues are used as a mechanism to distribute work across multiple machines.
 - Scheduling of tasks
 - [Retries](https://github.com/hibiken/asynq/wiki/Task-Retry) of failed tasks
 - Automatic recovery of tasks in the event of a worker crash
-- [Weighted priority queues](https://github.com/hibiken/asynq/wiki/Priority-Queues#weighted-priority-queues)
+- [Weighted priority queues](https://github.com/hibiken/asynq/wiki/Queue-Priority#weighted-priority)
-- [Strict priority queues](https://github.com/hibiken/asynq/wiki/Priority-Queues#strict-priority-queues)
+- [Strict priority queues](https://github.com/hibiken/asynq/wiki/Queue-Priority#strict-priority)
 - Low latency to add a task since writes are fast in Redis
 - De-duplication of tasks using [unique option](https://github.com/hibiken/asynq/wiki/Unique-Tasks)
 - Allow [timeout and deadline per task](https://github.com/hibiken/asynq/wiki/Task-Timeout-and-Cancelation)
@@ -91,7 +91,7 @@ type ImageResizePayload struct {
 //----------------------------------------------
 
 func NewEmailDeliveryTask(userID int, tmplID string) (*asynq.Task, error) {
-    payload, err := json.Marshal(EmailDeliveryPayload{UserID: userID, TemplateID: templID})
+    payload, err := json.Marshal(EmailDeliveryPayload{UserID: userID, TemplateID: tmplID})
     if err != nil {
         return nil, err
     }
@@ -129,7 +129,7 @@ type ImageProcessor struct {
     // ... fields for struct
 }
 
-func (p *ImageProcessor) ProcessTask(ctx context.Context, t *asynq.Task) error {
+func (processor *ImageProcessor) ProcessTask(ctx context.Context, t *asynq.Task) error {
     var p ImageResizePayload
     if err := json.Unmarshal(t.Payload(), &p); err != nil {
         return fmt.Errorf("json.Unmarshal failed: %v: %w", err, asynq.SkipRetry)
@@ -140,7 +140,7 @@ func (p *ImageProcessor) ProcessTask(ctx context.Context, t *asynq.Task) error {
 }
 
 func NewImageProcessor() *ImageProcessor {
-    // ... return an instance
+	return &ImageProcessor{}
 }
 ```
 

client.go

@@ -93,10 +93,8 @@ func (n retryOption) Type() OptionType   { return MaxRetryOpt }
 func (n retryOption) Value() interface{} { return int(n) }
 
 // Queue returns an option to specify the queue to enqueue the task into.
-//
-// Queue name is case-insensitive and the lowercased version is used.
 func Queue(qname string) Option {
-	return queueOption(strings.ToLower(qname))
+	return queueOption(qname)
 }
 
 func (qname queueOption) String() string     { return fmt.Sprintf("Queue(%q)", string(qname)) }
@@ -207,11 +205,11 @@ func composeOptions(opts ...Option) (option, error) {
 		case retryOption:
 			res.retry = int(opt)
 		case queueOption:
-			trimmed := strings.TrimSpace(string(opt))
+			qname := string(opt)
-			if err := base.ValidateQueueName(trimmed); err != nil {
+			if err := base.ValidateQueueName(qname); err != nil {
 				return option{}, err
 			}
-			res.queue = trimmed
+			res.queue = qname
 		case timeoutOption:
 			res.timeout = time.Duration(opt)
 		case deadlineOption:
@@ -269,6 +267,9 @@ func (c *Client) Close() error {
 //
 // If no ProcessAt or ProcessIn options are provided, the task will be pending immediately.
 func (c *Client) Enqueue(task *Task, opts ...Option) (*TaskInfo, error) {
+	if strings.TrimSpace(task.Type()) == "" {
+		return nil, fmt.Errorf("task typename cannot be empty")
+	}
 	c.mu.Lock()
 	if defaults, ok := c.opts[task.Type()]; ok {
 		opts = append(defaults, opts...)

client_test.go

@@ -287,13 +287,13 @@ func TestClientEnqueue(t *testing.T) {
 			},
 		},
 		{
-			desc: "Queue option should be case-insensitive",
+			desc: "Queue option should be case sensitive",
 			task: task,
 			opts: []Option{
-				Queue("HIGH"),
+				Queue("MyQueue"),
 			},
 			wantInfo: &TaskInfo{
-				Queue:         "high",
+				Queue:         "MyQueue",
 				Type:          task.Type(),
 				Payload:       task.Payload(),
 				State:         TaskStatePending,
@@ -306,12 +306,12 @@ func TestClientEnqueue(t *testing.T) {
 				NextProcessAt: now,
 			},
 			wantPending: map[string][]*base.TaskMessage{
-				"high": {
+				"MyQueue": {
 					{
 						Type:     task.Type(),
 						Payload:  task.Payload(),
 						Retry:    defaultMaxRetry,
-						Queue:    "high",
+						Queue:    "MyQueue",
 						Timeout:  int64(defaultTimeout.Seconds()),
 						Deadline: noDeadline.Unix(),
 					},
@@ -585,6 +585,16 @@ func TestClientEnqueueError(t *testing.T) {
 				Queue(""),
 			},
 		},
+		{
+			desc: "With empty task typename",
+			task: NewTask("", h.JSON(map[string]interface{}{})),
+			opts: []Option{},
+		},
+		{
+			desc: "With blank task typename",
+			task: NewTask("    ", h.JSON(map[string]interface{}{})),
+			opts: []Option{},
+		},
 	}
 
 	for _, tc := range tests {

inspector.go

@@ -50,6 +50,7 @@ type QueueInfo struct {
 	Queue string
 
 	// Total number of bytes that the queue and its tasks require to be stored in redis.
+	// It is an approximate memory usage value in bytes since the value is computed by sampling.
 	MemoryUsage int64
 
 	// Size is the total number of tasks in the queue.

internal/base/base.go

@@ -10,6 +10,7 @@ import (
 	"crypto/md5"
 	"encoding/hex"
 	"fmt"
+	"strings"
 	"sync"
 	"time"
 
@@ -22,7 +23,7 @@ import (
 )
 
 // Version of asynq library and CLI.
-const Version = "0.18.1"
+const Version = "0.18.4"
 
 // DefaultQueueName is the queue name used if none are specified by user.
 const DefaultQueueName = "default"
@@ -85,7 +86,7 @@ func TaskStateFromString(s string) (TaskState, error) {
 // ValidateQueueName validates a given qname to be used as a queue name.
 // Returns nil if valid, otherwise returns non-nil error.
 func ValidateQueueName(qname string) error {
-	if len(qname) == 0 {
+	if len(strings.TrimSpace(qname)) == 0 {
 		return fmt.Errorf("queue name must contain one or more characters")
 	}
 	return nil

internal/rdb/inspect.go

@@ -26,7 +26,8 @@ type Stats struct {
 	// Name of the queue (e.g. "default", "critical").
 	Queue string
 	// MemoryUsage is the total number of bytes the queue and its tasks require
-	// to be stored in redis.
+	// to be stored in redis. It is an approximate memory usage value in bytes
+	// since the value is computed by sampling.
 	MemoryUsage int64
 	// Paused indicates whether the queue is paused.
 	// If true, tasks in the queue should not be processed.
@@ -172,31 +173,82 @@ func (r *RDB) CurrentStats(qname string) (*Stats, error) {
 	return stats, nil
 }
 
+// Computes memory usage for the given queue by sampling tasks
+// from each redis list/zset. Returns approximate memory usage value
+// in bytes.
+//
+// KEYS[1] -> asynq:{qname}:active
+// KEYS[2] -> asynq:{qname}:pending
+// KEYS[3] -> asynq:{qname}:scheduled
+// KEYS[4] -> asynq:{qname}:retry
+// KEYS[5] -> asynq:{qname}:archived
+//
+// ARGV[1] -> asynq:{qname}:t:
+// ARGV[2] -> sample_size (e.g 20)
+var memoryUsageCmd = redis.NewScript(`
+local sample_size = tonumber(ARGV[2])
+if sample_size <= 0 then
+    return redis.error_reply("sample size must be a positive number")
+end
+local memusg = 0
+for i=1,2 do
+    local ids = redis.call("LRANGE", KEYS[i], 0, sample_size - 1)
+    local sample_total = 0
+    if (table.getn(ids) > 0) then
+        for _, id in ipairs(ids) do
+            local bytes = redis.call("MEMORY", "USAGE", ARGV[1] .. id)
+            sample_total = sample_total + bytes
+        end
+        local n = redis.call("LLEN", KEYS[i])
+        local avg = sample_total / table.getn(ids)
+        memusg = memusg + (avg * n)
+    end
+    local m = redis.call("MEMORY", "USAGE", KEYS[i])
+    if (m) then
+        memusg = memusg + m
+    end
+end
+for i=3,5 do
+    local ids = redis.call("ZRANGE", KEYS[i], 0, sample_size - 1)
+    local sample_total = 0
+    if (table.getn(ids) > 0) then
+        for _, id in ipairs(ids) do
+            local bytes = redis.call("MEMORY", "USAGE", ARGV[1] .. id)
+            sample_total = sample_total + bytes
+        end
+        local n = redis.call("ZCARD", KEYS[i])
+        local avg = sample_total / table.getn(ids)
+        memusg = memusg + (avg * n)
+    end
+    local m = redis.call("MEMORY", "USAGE", KEYS[i])
+    if (m) then
+        memusg = memusg + m
+    end
+end
+return memusg
+`)
+
 func (r *RDB) memoryUsage(qname string) (int64, error) {
 	var op errors.Op = "rdb.memoryUsage"
-	var (
+	const sampleSize = 20
-		keys   []string
+	keys := []string{
-		data   []string
+		base.ActiveKey(qname),
-		cursor uint64
+		base.PendingKey(qname),
-		err    error
+		base.ScheduledKey(qname),
-	)
+		base.RetryKey(qname),
-	for {
+		base.ArchivedKey(qname),
-		data, cursor, err = r.client.Scan(cursor, fmt.Sprintf("asynq:{%s}*", qname), 100).Result()
-		if err != nil {
-			return 0, errors.E(op, errors.Unknown, &errors.RedisCommandError{Command: "scan", Err: err})
-		}
-		keys = append(keys, data...)
-		if cursor == 0 {
-			break
-		}
 	}
-	var usg int64
+	argv := []interface{}{
-	for _, k := range keys {
+		base.TaskKeyPrefix(qname),
-		n, err := r.client.MemoryUsage(k).Result()
+		sampleSize,
-		if err != nil {
+	}
-			return 0, errors.E(op, errors.Unknown, &errors.RedisCommandError{Command: "memory usage", Err: err})
+	res, err := memoryUsageCmd.Run(r.client, keys, argv...).Result()
-		}
+	if err != nil {
-		usg += n
+		return 0, errors.E(op, errors.Unknown, fmt.Sprintf("redis eval error: %v", err))
+	}
+	usg, err := cast.ToInt64E(res)
+	if err != nil {
+		return 0, errors.E(op, errors.Internal, fmt.Sprintf("could not cast script return value to int64"))
 	}
 	return usg, nil
 }

scheduler.go

@@ -19,6 +19,8 @@ import (
 )
 
 // A Scheduler kicks off tasks at regular intervals based on the user defined schedule.
+//
+// Schedulers are safe for concurrent use by multiple goroutines.
 type Scheduler struct {
 	id         string
 	state      *base.ServerState
@@ -30,6 +32,9 @@ type Scheduler struct {
 	done       chan struct{}
 	wg         sync.WaitGroup
 	errHandler func(task *Task, opts []Option, err error)
+
+	// guards idmap
+	mu sync.Mutex
 	// idmap maps Scheduler's entry ID to cron.EntryID
 	// to avoid using cron.EntryID as the public API of
 	// the Scheduler.
@@ -154,17 +159,22 @@ func (s *Scheduler) Register(cronspec string, task *Task, opts ...Option) (entry
 	if err != nil {
 		return "", err
 	}
+	s.mu.Lock()
 	s.idmap[job.id.String()] = cronID
+	s.mu.Unlock()
 	return job.id.String(), nil
 }
 
 // Unregister removes a registered entry by entry ID.
 // Unregister returns a non-nil error if no entries were found for the given entryID.
 func (s *Scheduler) Unregister(entryID string) error {
+	s.mu.Lock()
+	defer s.mu.Unlock()
 	cronID, ok := s.idmap[entryID]
 	if !ok {
 		return fmt.Errorf("asynq: no scheduler entry found")
 	}
+	delete(s.idmap, entryID)
 	s.cron.Remove(cronID)
 	return nil
 }

servemux.go

@@ -98,7 +98,7 @@ func (mux *ServeMux) Handle(pattern string, handler Handler) {
 	mux.mu.Lock()
 	defer mux.mu.Unlock()
 
-	if pattern == "" {
+	if strings.TrimSpace(pattern) == "" {
 		panic("asynq: invalid pattern")
 	}
 	if handler == nil {

server.go

@@ -295,6 +295,9 @@ func NewServer(r RedisConnOpt, cfg Config) *Server {
 	}
 	queues := make(map[string]int)
 	for qname, p := range cfg.Queues {
+		if err := base.ValidateQueueName(qname); err != nil {
+			continue // ignore invalid queue names
+		}
 		if p > 0 {
 			queues[qname] = p
 		}