Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Upgrade to goredis v8 #15

Open
wants to merge 3 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 6 additions & 4 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,8 @@ import (
)

func main() {
p, err := redisqueue.NewProducerWithOptions(&redisqueue.ProducerOptions{
ctx := context.Background()
p, err := redisqueue.NewProducerWithContextOptions(ctx, &redisqueue.ProducerOptions{
StreamMaxLength: 10000,
ApproximateMaxLength: true,
})
Expand All @@ -70,7 +71,7 @@ func main() {
}

for i := 0; i < 1000; i++ {
err := p.Enqueue(&redisqueue.Message{
err := p.EnqueueWithContext(ctx, &redisqueue.Message{
Stream: "redisqueue:test",
Values: map[string]interface{}{
"index": i,
Expand Down Expand Up @@ -100,7 +101,8 @@ import (
)

func main() {
c, err := redisqueue.NewConsumerWithOptions(&redisqueue.ConsumerOptions{
ctx := context.Background()
c, err := redisqueue.NewConsumerWithContextOptions(ctx, &redisqueue.ConsumerOptions{
VisibilityTimeout: 60 * time.Second,
BlockingTimeout: 5 * time.Second,
ReclaimInterval: 1 * time.Second,
Expand All @@ -121,7 +123,7 @@ func main() {
}()

fmt.Println("starting")
c.Run()
c.Run(ctx)
fmt.Println("stopped")
}

Expand Down
39 changes: 20 additions & 19 deletions consumer.go
Original file line number Diff line number Diff line change
@@ -1,12 +1,13 @@
package redisqueue

import (
"context"
"net"
"os"
"sync"
"time"

"github.com/go-redis/redis/v7"
"github.com/go-redis/redis/v8"
"github.com/pkg/errors"
)

Expand Down Expand Up @@ -58,7 +59,7 @@ type ConsumerOptions struct {
RedisClient redis.UniversalClient
// RedisOptions allows you to configure the underlying Redis connection.
// More info here:
// https://pkg.go.dev/github.com/go-redis/redis/v7?tab=doc#Options.
// https://pkg.go.dev/github.com/go-redis/redis/v8?tab=doc#Options.
//
// This field is used if RedisClient field is nil.
RedisOptions *RedisOptions
Expand Down Expand Up @@ -98,15 +99,15 @@ var defaultConsumerOptions = &ConsumerOptions{
// to the hostname, GroupName to "redisqueue", VisibilityTimeout to 60 seconds,
// BufferSize to 100, and Concurrency to 10. In most production environments,
// you'll want to use NewConsumerWithOptions.
func NewConsumer() (*Consumer, error) {
return NewConsumerWithOptions(defaultConsumerOptions)
func NewConsumerWithContext(ctx context.Context) (*Consumer, error) {
return NewConsumerWithContextOptions(ctx, defaultConsumerOptions)
}

// NewConsumerWithOptions creates a Consumer with custom ConsumerOptions. If
// Name is left empty, it defaults to the hostname; if GroupName is left empty,
// it defaults to "redisqueue"; if BlockingTimeout is 0, it defaults to 5
// seconds; if ReclaimInterval is 0, it defaults to 1 second.
func NewConsumerWithOptions(options *ConsumerOptions) (*Consumer, error) {
func NewConsumerWithContextOptions(ctx context.Context, options *ConsumerOptions) (*Consumer, error) {
hostname, _ := os.Hostname()

if options.Name == "" {
Expand All @@ -130,7 +131,7 @@ func NewConsumerWithOptions(options *ConsumerOptions) (*Consumer, error) {
r = newRedisClient(options.RedisOptions)
}

if err := redisPreflightChecks(r); err != nil {
if err := redisPreflightChecks(ctx, r); err != nil {
return nil, err
}

Expand Down Expand Up @@ -182,15 +183,15 @@ func (c *Consumer) Register(stream string, fn ConsumerFunc) {
// Run will terminate early. The same will happen if an error occurs when
// creating the consumer group in Redis. Run will block until Shutdown is called
// and all of the in-flight messages have been processed.
func (c *Consumer) Run() {
func (c *Consumer) Run(ctx context.Context) {
if len(c.consumers) == 0 {
c.Errors <- errors.New("at least one consumer function needs to be registered")
return
}

for stream, consumer := range c.consumers {
c.streams = append(c.streams, stream)
err := c.redis.XGroupCreateMkStream(stream, c.options.GroupName, consumer.id).Err()
err := c.redis.XGroupCreateMkStream(ctx, stream, c.options.GroupName, consumer.id).Err()
// ignoring the BUSYGROUP error makes this a noop
if err != nil && err.Error() != "BUSYGROUP Consumer Group name already exists" {
c.Errors <- errors.Wrap(err, "error creating consumer group")
Expand All @@ -202,8 +203,8 @@ func (c *Consumer) Run() {
c.streams = append(c.streams, ">")
}

go c.reclaim()
go c.poll()
go c.reclaim(ctx)
go c.poll(ctx)

stop := newSignalHandler()
go func() {
Expand All @@ -214,7 +215,7 @@ func (c *Consumer) Run() {
c.wg.Add(c.options.Concurrency)

for i := 0; i < c.options.Concurrency; i++ {
go c.work()
go c.work(ctx)
}

c.wg.Wait()
Expand All @@ -237,7 +238,7 @@ func (c *Consumer) Shutdown() {
// If VisibilityTimeout is 0, this function returns early and no messages are
// reclaimed. It checks the list of pending messages according to
// ReclaimInterval.
func (c *Consumer) reclaim() {
func (c *Consumer) reclaim(ctx context.Context) {
if c.options.VisibilityTimeout == 0 {
return
}
Expand All @@ -256,7 +257,7 @@ func (c *Consumer) reclaim() {
end := "+"

for {
res, err := c.redis.XPendingExt(&redis.XPendingExtArgs{
res, err := c.redis.XPendingExt(ctx, &redis.XPendingExtArgs{
Stream: stream,
Group: c.options.GroupName,
Start: start,
Expand All @@ -276,7 +277,7 @@ func (c *Consumer) reclaim() {

for _, r := range res {
if r.Idle >= c.options.VisibilityTimeout {
claimres, err := c.redis.XClaim(&redis.XClaimArgs{
claimres, err := c.redis.XClaim(ctx, &redis.XClaimArgs{
Stream: stream,
Group: c.options.GroupName,
Consumer: c.options.Name,
Expand All @@ -297,7 +298,7 @@ func (c *Consumer) reclaim() {
// exists, the only way we can get it out of the
// pending state is to acknowledge it.
if err == redis.Nil {
err = c.redis.XAck(stream, c.options.GroupName, r.ID).Err()
err = c.redis.XAck(ctx, stream, c.options.GroupName, r.ID).Err()
if err != nil {
c.Errors <- errors.Wrapf(err, "error acknowledging after failed claim for %q stream and %q message", stream, r.ID)
continue
Expand All @@ -324,7 +325,7 @@ func (c *Consumer) reclaim() {
// messages for this consumer to process. It blocks for up to 5 seconds instead
// of blocking indefinitely so that it can periodically check to see if Shutdown
// was called.
func (c *Consumer) poll() {
func (c *Consumer) poll(ctx context.Context) {
for {
select {
case <-c.stopPoll:
Expand All @@ -335,7 +336,7 @@ func (c *Consumer) poll() {
}
return
default:
res, err := c.redis.XReadGroup(&redis.XReadGroupArgs{
res, err := c.redis.XReadGroup(ctx, &redis.XReadGroupArgs{
Group: c.options.GroupName,
Consumer: c.options.Name,
Streams: c.streams,
Expand Down Expand Up @@ -378,7 +379,7 @@ func (c *Consumer) enqueue(stream string, msgs []redis.XMessage) {
// channel, it calls the corrensponding ConsumerFunc depending on the stream it
// came from. If no error is returned from the ConsumerFunc, the message is
// acknowledged in Redis.
func (c *Consumer) work() {
func (c *Consumer) work(ctx context.Context) {
defer c.wg.Done()

for {
Expand All @@ -389,7 +390,7 @@ func (c *Consumer) work() {
c.Errors <- errors.Wrapf(err, "error calling ConsumerFunc for %q stream and %q message", msg.Stream, msg.ID)
continue
}
err = c.redis.XAck(msg.Stream, c.options.GroupName, msg.ID).Err()
err = c.redis.XAck(ctx, msg.Stream, c.options.GroupName, msg.ID).Err()
if err != nil {
c.Errors <- errors.Wrapf(err, "error acknowledging after success for %q stream and %q message", msg.Stream, msg.ID)
continue
Expand Down
Loading