From ec16c2080fa38f8c0ca975fae51e73daaa4ce947 Mon Sep 17 00:00:00 2001 From: Derrick Date: Mon, 2 Nov 2020 17:47:15 -0800 Subject: [PATCH 01/14] Add support for cascading retries --- events/config.go | 5 +++ events/consumer.go | 82 ++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 85 insertions(+), 2 deletions(-) diff --git a/events/config.go b/events/config.go index b402d21c..8d8eb270 100644 --- a/events/config.go +++ b/events/config.go @@ -2,12 +2,14 @@ package events import ( "errors" + "github.com/Shopify/sarama" "github.com/kelseyhightower/envconfig" ) const DeadLetterSuffix = "-dead-letters" +// CloudEventsConfig describes the configuration for a consumer of cloud events from a Kafka topic type CloudEventsConfig struct { EventSource string `envconfig:"CLOUD_EVENTS_SOURCE" required:"true"` KafkaBrokers []string `envconfig:"KAFKA_BROKERS" required:"true"` @@ -19,6 +21,9 @@ type CloudEventsConfig struct { KafkaVersion string `envconfig:"KAFKA_VERSION" required:"true"` KafkaUsername string `envconfig:"KAFKA_USERNAME" required:"false"` KafkaPassword string `envconfig:"KAFKA_PASSWORD" required:"false"` + KafkaDelay int `envconfig:"KAFKA_DELAY" default:"0"` + CascadeDelays []int `envconfig:"KAFKA_CASCADE_DELAYS" default:""` + CascadePattern string `envconfig:"KAFKA_CASCADE_PATTERN" default:"%s-delay-%d"` SaramaConfig *sarama.Config } diff --git a/events/consumer.go b/events/consumer.go index 7d0e91de..3ad2d122 100644 --- a/events/consumer.go +++ b/events/consumer.go @@ -2,12 +2,15 @@ package events import ( "context" + "fmt" + "log" + "sync" + "time" + "github.com/Shopify/sarama" "github.com/cloudevents/sdk-go/protocol/kafka_sarama/v2" cloudevents "github.com/cloudevents/sdk-go/v2" "github.com/cloudevents/sdk-go/v2/binding" - "log" - "sync" ) type SaramaConsumer struct { @@ -19,6 +22,62 @@ type SaramaConsumer struct { deadLetterProducer *KafkaCloudEventsProducer } +//CascadingEventConsumer an event consumer that cascaded failures +type CascadingEventConsumer struct { + Consumers []EventConsumer +} + +// NewCascadingCloudEventsConsumer create a cascading events consumer +func NewCascadingCloudEventsConsumer(config *CloudEventsConfig) (EventConsumer, error) { + topic := config.KafkaTopic + delay := config.KafkaDelay + var consumers []EventConsumer + for nextDelay := range config.CascadeDelays { + newconfig := *config + newconfig.KafkaDelay = delay + newconfig.KafkaTopic = topic + newconfig.KafkaDeadLettersTopic = fmt.Sprintf(config.CascadePattern, config.KafkaTopic, nextDelay) + consumer, err := NewSaramaCloudEventsConsumer(&newconfig) + if err != nil { + return nil, err + } + consumers = append(consumers, consumer) + topic = newconfig.KafkaDeadLettersTopic + delay = nextDelay + } + newconfig := *config + newconfig.KafkaDelay = delay + newconfig.KafkaTopic = topic + consumer, err := NewSaramaCloudEventsConsumer(&newconfig) + if err != nil { + return nil, err + } + consumers = append(consumers, consumer) + + return &CascadingEventConsumer{ + Consumers: consumers, + }, nil +} + +//RegisterHandler registers the handler with all consumers in the cascade +func (c *CascadingEventConsumer) RegisterHandler(handler EventHandler) { + for _, consumer := range c.Consumers { + consumer.RegisterHandler(handler) + } +} + +//Start starts an async consumer goproc (that may sleep) +func (c *CascadingEventConsumer) Start(ctx context.Context) error { + for _, consumer := range c.Consumers { + err := consumer.Start(ctx) + if err != nil { + return err + } + } + return nil +} + +//NewSaramaCloudEventsConsumer creates a new cloud events consumer func NewSaramaCloudEventsConsumer(config *CloudEventsConfig) (EventConsumer, error) { if err := validateConsumerConfig(config); err != nil { return nil, err @@ -32,19 +91,36 @@ func NewSaramaCloudEventsConsumer(config *CloudEventsConfig) (EventConsumer, err }, nil } +//Setup marks a consumer to be ready func (s *SaramaConsumer) Setup(session sarama.ConsumerGroupSession) error { // Mark the consumer as ready close(s.ready) return nil } +//Cleanup frees any resources func (s *SaramaConsumer) Cleanup(session sarama.ConsumerGroupSession) error { return nil } +//ConsumeClaim synchronously consumes all the messages in a claim, optionally delaying consumption a fixed amount of time func (s *SaramaConsumer) ConsumeClaim(session sarama.ConsumerGroupSession, claim sarama.ConsumerGroupClaim) error { + delay := time.Second * time.Duration(s.config.KafkaDelay) for message := range claim.Messages() { m := kafka_sarama.NewMessageFromConsumerMessage(message) + timestamp := message.Timestamp + scheduledTime := timestamp.Add(delay) + remaining := time.Now().Sub(scheduledTime) + if delay == 0 || remaining > 0 { + timer := time.NewTimer(remaining) + select { + case <-timer.C: + break + case <-session.Context().Done(): + timer.Stop() + return nil + } + } // just ignore non-cloud event messages if rs, rserr := binding.ToEvent(context.Background(), m); rserr == nil { s.handleCloudEvent(*rs) @@ -76,10 +152,12 @@ func (s *SaramaConsumer) sendToDeadLetterTopic(ce cloudevents.Event) { } } +//RegisterHandler register a handler to process events func (s *SaramaConsumer) RegisterHandler(handler EventHandler) { s.handlers = append(s.handlers, handler) } +//Start starts an async consumer goproc (that may sleep) func (s *SaramaConsumer) Start(ctx context.Context) error { if err := s.initialize(); err != nil { return err From 57e22081f8f3b93a41d62008f44f867ec427c4b6 Mon Sep 17 00:00:00 2001 From: Derrick Date: Tue, 3 Nov 2020 13:54:33 -0800 Subject: [PATCH 02/14] Support exponential backoff retries --- events/config.go | 32 +++++++++++-------- events/consumer.go | 79 ++++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 92 insertions(+), 19 deletions(-) diff --git a/events/config.go b/events/config.go index 8d8eb270..855766ec 100644 --- a/events/config.go +++ b/events/config.go @@ -2,6 +2,7 @@ package events import ( "errors" + "time" "github.com/Shopify/sarama" "github.com/kelseyhightower/envconfig" @@ -11,19 +12,24 @@ const DeadLetterSuffix = "-dead-letters" // CloudEventsConfig describes the configuration for a consumer of cloud events from a Kafka topic type CloudEventsConfig struct { - EventSource string `envconfig:"CLOUD_EVENTS_SOURCE" required:"true"` - KafkaBrokers []string `envconfig:"KAFKA_BROKERS" required:"true"` - KafkaConsumerGroup string `envconfig:"KAFKA_CONSUMER_GROUP" required:"false"` - KafkaTopic string `envconfig:"KAFKA_TOPIC" default:"events"` - KafkaDeadLettersTopic string `envconfig:"KAFKA_DEAD_LETTERS_TOPIC"` - KafkaTopicPrefix string `envconfig:"KAFKA_TOPIC_PREFIX" required:"true"` - KafkaRequireSSL bool `envconfig:"KAFKA_REQUIRE_SSL" required:"true"` - KafkaVersion string `envconfig:"KAFKA_VERSION" required:"true"` - KafkaUsername string `envconfig:"KAFKA_USERNAME" required:"false"` - KafkaPassword string `envconfig:"KAFKA_PASSWORD" required:"false"` - KafkaDelay int `envconfig:"KAFKA_DELAY" default:"0"` - CascadeDelays []int `envconfig:"KAFKA_CASCADE_DELAYS" default:""` - CascadePattern string `envconfig:"KAFKA_CASCADE_PATTERN" default:"%s-delay-%d"` + EventSource string `envconfig:"CLOUD_EVENTS_SOURCE" required:"true"` + KafkaBrokers []string `envconfig:"KAFKA_BROKERS" required:"true"` + KafkaConsumerGroup string `envconfig:"KAFKA_CONSUMER_GROUP" required:"false"` + KafkaTopic string `envconfig:"KAFKA_TOPIC" default:"events"` + KafkaDeadLettersTopic string `envconfig:"KAFKA_DEAD_LETTERS_TOPIC"` + KafkaTopicPrefix string `envconfig:"KAFKA_TOPIC_PREFIX" required:"true"` + KafkaRequireSSL bool `envconfig:"KAFKA_REQUIRE_SSL" required:"true"` + KafkaVersion string `envconfig:"KAFKA_VERSION" required:"true"` + KafkaUsername string `envconfig:"KAFKA_USERNAME" required:"false"` + KafkaPassword string `envconfig:"KAFKA_PASSWORD" required:"false"` + KafkaDelay time.Duration `envconfig:"KAFKA_DELAY" default:"0s"` + CascadeDelays []time.Duration `envconfig:"KAFKA_CASCADE_DELAYS" default:""` + CascadePattern string `envconfig:"KAFKA_CASCADE_PATTERN" default:"%s-delay-%d"` + RetryInitialDelay time.Duration `envconfig:"KAFKA_RETRY_INITIAL_DELAY" default:"5s"` + RetryMaxDelay time.Duration `envconfig:"KAFKA_RETRY_MAX_DELAY" default:"24h"` + RetryFactor float32 `envconfig:"KAFKA_RETRY_FACTOR" default:"1.5"` + RetryAddend time.Duration `envconfig:"KAFKA_RETRY_ADDEND" default:"0s"` + RetryMaxAttempts int `envconfig:"KAFKA_RETRY_MAX_ATTEMPTS" default:"0"` SaramaConfig *sarama.Config } diff --git a/events/consumer.go b/events/consumer.go index 3ad2d122..c3310e54 100644 --- a/events/consumer.go +++ b/events/consumer.go @@ -13,6 +13,58 @@ import ( "github.com/cloudevents/sdk-go/v2/binding" ) +//ExponentialRetry implements exponential backoff +type ExponentialRetry struct { + attempt int + maxAttempts int + delay time.Duration + maxDelay time.Duration + initialDelay time.Duration + multiplier float32 + adder time.Duration +} + +//RetryAlgorithm implements a retry mechanism +type RetryAlgorithm interface { + Retry() bool // returns true after possible delay if should attempt retry +} + +//NewExponentialRetry creates a new exponential backoff mechanism +func NewExponentialRetry(config *CloudEventsConfig) *ExponentialRetry { + return &ExponentialRetry{ + attempt: 0, + delay: 0, + maxDelay: config.RetryMaxDelay, + initialDelay: config.RetryInitialDelay, + multiplier: config.RetryFactor, + adder: config.RetryAddend, + maxAttempts: config.RetryMaxAttempts, + } +} + +// Retry implements retry algorithm +func (r *ExponentialRetry) Retry(ctx context.Context) bool { + if r.attempt == r.maxAttempts { + return false + } + if r.attempt == 0 { + r.delay = r.initialDelay + } else { + r.delay = time.Duration(int64(r.multiplier * float32(r.delay))) + r.delay = r.delay + time.Duration(r.adder)*time.Second + if r.delay > r.maxDelay { + r.delay = r.maxDelay + } + } + r.attempt = r.attempt + 1 + select { + case <-time.After(r.delay): + return true + case <-ctx.Done(): + return false + } +} + type SaramaConsumer struct { config *CloudEventsConfig consumerGroup sarama.ConsumerGroup @@ -20,6 +72,7 @@ type SaramaConsumer struct { topic string handlers []EventHandler deadLetterProducer *KafkaCloudEventsProducer + retry *ExponentialRetry } //CascadingEventConsumer an event consumer that cascaded failures @@ -28,11 +81,12 @@ type CascadingEventConsumer struct { } // NewCascadingCloudEventsConsumer create a cascading events consumer +// For every delay in config.CascadeDelays, a new consumer in addition to the one for the primary topic func NewCascadingCloudEventsConsumer(config *CloudEventsConfig) (EventConsumer, error) { topic := config.KafkaTopic delay := config.KafkaDelay var consumers []EventConsumer - for nextDelay := range config.CascadeDelays { + for _, nextDelay := range config.CascadeDelays { newconfig := *config newconfig.KafkaDelay = delay newconfig.KafkaTopic = topic @@ -88,6 +142,7 @@ func NewSaramaCloudEventsConsumer(config *CloudEventsConfig) (EventConsumer, err ready: make(chan bool), topic: config.GetPrefixedTopic(), handlers: make([]EventHandler, 0), + retry: NewExponentialRetry(config), }, nil } @@ -123,7 +178,11 @@ func (s *SaramaConsumer) ConsumeClaim(session sarama.ConsumerGroupSession, claim } // just ignore non-cloud event messages if rs, rserr := binding.ToEvent(context.Background(), m); rserr == nil { - s.handleCloudEvent(*rs) + err := s.handleCloudEvent(session.Context(), *rs) + if err != nil { + log.Printf("failed to process event %s", err) + return err + } } session.MarkMessage(message, "") } @@ -131,7 +190,7 @@ func (s *SaramaConsumer) ConsumeClaim(session sarama.ConsumerGroupSession, claim return nil } -func (s *SaramaConsumer) handleCloudEvent(ce cloudevents.Event) { +func (s *SaramaConsumer) handleCloudEvent(ctx context.Context, ce cloudevents.Event) error { var errors []error for _, handler := range s.handlers { if handler.CanHandle(ce) { @@ -141,15 +200,23 @@ func (s *SaramaConsumer) handleCloudEvent(ce cloudevents.Event) { } } if len(errors) != 0 { - log.Printf("Sending event %v to dead-letter topic due to handler error(s): %v", ce.ID(), errors) - s.sendToDeadLetterTopic(ce) + if s.retry.Retry(ctx) { + return s.handleCloudEvent(ctx, ce) + } + if s.config.IsDeadLettersEnabled() { + return s.sendToDeadLetterTopic(ce) + } + return fmt.Errorf("no dead letter topic, skipping event %v, %v", ce.ID(), errors) } + return nil } -func (s *SaramaConsumer) sendToDeadLetterTopic(ce cloudevents.Event) { +func (s *SaramaConsumer) sendToDeadLetterTopic(ce cloudevents.Event) error { if err := s.deadLetterProducer.SendCloudEvent(context.Background(), ce); err != nil { log.Printf("Failed to send event %v to dead-letter topic: %v", ce, err) + return err } + return nil } //RegisterHandler register a handler to process events From 12ed0985e12ded5465369ddd6a3e54d24f8c3438 Mon Sep 17 00:00:00 2001 From: Derrick Date: Tue, 3 Nov 2020 13:59:51 -0800 Subject: [PATCH 03/14] Fix addend --- events/consumer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/events/consumer.go b/events/consumer.go index c3310e54..f39626a7 100644 --- a/events/consumer.go +++ b/events/consumer.go @@ -51,7 +51,7 @@ func (r *ExponentialRetry) Retry(ctx context.Context) bool { r.delay = r.initialDelay } else { r.delay = time.Duration(int64(r.multiplier * float32(r.delay))) - r.delay = r.delay + time.Duration(r.adder)*time.Second + r.delay = r.delay + r.adder if r.delay > r.maxDelay { r.delay = r.maxDelay } From ce2a13227bc73fcc1fd1dc19eeab94006cde1565 Mon Sep 17 00:00:00 2001 From: Derrick Date: Thu, 5 Nov 2020 13:16:57 -0800 Subject: [PATCH 04/14] Pass in index of delay topic --- events/consumer.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/events/consumer.go b/events/consumer.go index f39626a7..477eb4d1 100644 --- a/events/consumer.go +++ b/events/consumer.go @@ -86,11 +86,11 @@ func NewCascadingCloudEventsConsumer(config *CloudEventsConfig) (EventConsumer, topic := config.KafkaTopic delay := config.KafkaDelay var consumers []EventConsumer - for _, nextDelay := range config.CascadeDelays { + for i, nextDelay := range config.CascadeDelays { newconfig := *config newconfig.KafkaDelay = delay newconfig.KafkaTopic = topic - newconfig.KafkaDeadLettersTopic = fmt.Sprintf(config.CascadePattern, config.KafkaTopic, nextDelay) + newconfig.KafkaDeadLettersTopic = fmt.Sprintf(config.CascadePattern, config.KafkaTopic, i, nextDelay) consumer, err := NewSaramaCloudEventsConsumer(&newconfig) if err != nil { return nil, err From bc3957d2a2df58647639b74c6f461389e8df110b Mon Sep 17 00:00:00 2001 From: Derrick Date: Thu, 5 Nov 2020 13:41:49 -0800 Subject: [PATCH 05/14] Fix boundary condition where max delay is exceeded. --- events/consumer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/events/consumer.go b/events/consumer.go index 477eb4d1..d12565da 100644 --- a/events/consumer.go +++ b/events/consumer.go @@ -53,7 +53,7 @@ func (r *ExponentialRetry) Retry(ctx context.Context) bool { r.delay = time.Duration(int64(r.multiplier * float32(r.delay))) r.delay = r.delay + r.adder if r.delay > r.maxDelay { - r.delay = r.maxDelay + return false } } r.attempt = r.attempt + 1 From de42b719eb182cbd9100f93b46844001b7c742cc Mon Sep 17 00:00:00 2001 From: Derrick Date: Thu, 5 Nov 2020 13:42:30 -0800 Subject: [PATCH 06/14] Fix boundary condition where max delay is exceeded. --- events/consumer.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/events/consumer.go b/events/consumer.go index d12565da..7c014ebc 100644 --- a/events/consumer.go +++ b/events/consumer.go @@ -52,9 +52,9 @@ func (r *ExponentialRetry) Retry(ctx context.Context) bool { } else { r.delay = time.Duration(int64(r.multiplier * float32(r.delay))) r.delay = r.delay + r.adder - if r.delay > r.maxDelay { - return false - } + } + if r.delay > r.maxDelay { + return false } r.attempt = r.attempt + 1 select { From 90a6b18ec12a6b7c297ae3536cd60140f8e9aeae Mon Sep 17 00:00:00 2001 From: Derrick Date: Thu, 5 Nov 2020 13:45:00 -0800 Subject: [PATCH 07/14] Fix max attempts logic --- events/consumer.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/events/consumer.go b/events/consumer.go index 7c014ebc..d2eddb33 100644 --- a/events/consumer.go +++ b/events/consumer.go @@ -44,10 +44,11 @@ func NewExponentialRetry(config *CloudEventsConfig) *ExponentialRetry { // Retry implements retry algorithm func (r *ExponentialRetry) Retry(ctx context.Context) bool { - if r.attempt == r.maxAttempts { + r.attempt = r.attempt + 1 + if r.attempt > r.maxAttempts { return false } - if r.attempt == 0 { + if r.attempt == 1 { r.delay = r.initialDelay } else { r.delay = time.Duration(int64(r.multiplier * float32(r.delay))) @@ -56,7 +57,7 @@ func (r *ExponentialRetry) Retry(ctx context.Context) bool { if r.delay > r.maxDelay { return false } - r.attempt = r.attempt + 1 + select { case <-time.After(r.delay): return true From 107bac9d41889b039740cc780d545e301891af9f Mon Sep 17 00:00:00 2001 From: Derrick Date: Thu, 5 Nov 2020 13:47:19 -0800 Subject: [PATCH 08/14] Add comments on retry mechanism --- events/consumer.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/events/consumer.go b/events/consumer.go index d2eddb33..9624c1ab 100644 --- a/events/consumer.go +++ b/events/consumer.go @@ -30,6 +30,8 @@ type RetryAlgorithm interface { } //NewExponentialRetry creates a new exponential backoff mechanism +// retries will not be attempted if the delay exceeds maxDelay, so if initialDelay exceeds maxDelay, then no retries will be attempted +// further retries will not be attempted if maxAttempts have already been tried, so if you set maxAttempts to 0, then no retries will be attempted func NewExponentialRetry(config *CloudEventsConfig) *ExponentialRetry { return &ExponentialRetry{ attempt: 0, From 0f536943e35c30005ef6aebc6b52656a0f278ba8 Mon Sep 17 00:00:00 2001 From: Derrick Date: Fri, 6 Nov 2020 10:04:49 -0800 Subject: [PATCH 09/14] Add comments --- events/config.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/events/config.go b/events/config.go index 855766ec..683b9617 100644 --- a/events/config.go +++ b/events/config.go @@ -33,6 +33,7 @@ type CloudEventsConfig struct { SaramaConfig *sarama.Config } +//NewConfig creates a new cloud events config func NewConfig() *CloudEventsConfig { cfg := &CloudEventsConfig{} cfg.SaramaConfig = sarama.NewConfig() @@ -40,6 +41,7 @@ func NewConfig() *CloudEventsConfig { return cfg } +//LoadFromEnv loads the configuration from environment variables func (k *CloudEventsConfig) LoadFromEnv() error { if err := envconfig.Process("", k); err != nil { return err @@ -67,10 +69,12 @@ func (k *CloudEventsConfig) LoadFromEnv() error { return nil } +//GetPrefixedTopic returns the fully resolved topic name func (k *CloudEventsConfig) GetPrefixedTopic() string { return k.KafkaTopicPrefix + k.KafkaTopic } +//GetDeadLettersTopic returns the fully resolved dead letter topic name or an empty string if there is no dead letter topic func (k *CloudEventsConfig) GetDeadLettersTopic() string { if k.KafkaDeadLettersTopic == "" { return k.KafkaDeadLettersTopic @@ -78,6 +82,7 @@ func (k *CloudEventsConfig) GetDeadLettersTopic() string { return k.KafkaTopicPrefix + k.KafkaDeadLettersTopic } +//IsDeadLettersEnabled returns true iff there is a dead letter topic func (k *CloudEventsConfig) IsDeadLettersEnabled() bool { return k.GetDeadLettersTopic() != "" } From 84aa9691f52166d08abf6b0596222fd2e01b82c6 Mon Sep 17 00:00:00 2001 From: Derrick Date: Fri, 6 Nov 2020 10:11:03 -0800 Subject: [PATCH 10/14] Remove vendoring --- go.mod | 3 + go.sum | 16 + tracing/init.go | 30 + vendor/github.com/Shopify/sarama/.gitignore | 29 - .../github.com/Shopify/sarama/.golangci.yml | 77 - vendor/github.com/Shopify/sarama/CHANGELOG.md | 979 --- vendor/github.com/Shopify/sarama/LICENSE | 20 - vendor/github.com/Shopify/sarama/Makefile | 31 - vendor/github.com/Shopify/sarama/README.md | 36 - vendor/github.com/Shopify/sarama/Vagrantfile | 14 - .../github.com/Shopify/sarama/acl_bindings.go | 138 - .../Shopify/sarama/acl_create_request.go | 89 - .../Shopify/sarama/acl_create_response.go | 94 - .../Shopify/sarama/acl_delete_request.go | 62 - .../Shopify/sarama/acl_delete_response.go | 163 - .../Shopify/sarama/acl_describe_request.go | 39 - .../Shopify/sarama/acl_describe_response.go | 91 - .../github.com/Shopify/sarama/acl_filter.go | 78 - vendor/github.com/Shopify/sarama/acl_types.go | 55 - .../sarama/add_offsets_to_txn_request.go | 57 - .../sarama/add_offsets_to_txn_response.go | 49 - .../sarama/add_partitions_to_txn_request.go | 81 - .../sarama/add_partitions_to_txn_response.go | 114 - vendor/github.com/Shopify/sarama/admin.go | 934 -- .../Shopify/sarama/alter_configs_request.go | 126 - .../Shopify/sarama/alter_configs_response.go | 101 - .../alter_partition_reassignments_request.go | 130 - .../alter_partition_reassignments_response.go | 157 - .../Shopify/sarama/api_versions_request.go | 29 - .../Shopify/sarama/api_versions_response.go | 93 - .../Shopify/sarama/async_producer.go | 1158 --- .../Shopify/sarama/balance_strategy.go | 1061 --- vendor/github.com/Shopify/sarama/broker.go | 1441 --- vendor/github.com/Shopify/sarama/client.go | 1070 --- vendor/github.com/Shopify/sarama/compress.go | 194 - vendor/github.com/Shopify/sarama/config.go | 765 -- .../Shopify/sarama/config_resource_type.go | 18 - vendor/github.com/Shopify/sarama/consumer.go | 914 -- .../Shopify/sarama/consumer_group.go | 876 -- .../Shopify/sarama/consumer_group_members.go | 96 - .../sarama/consumer_metadata_request.go | 38 - .../sarama/consumer_metadata_response.go | 82 - .../Shopify/sarama/control_record.go | 72 - .../github.com/Shopify/sarama/crc32_field.go | 86 - .../sarama/create_partitions_request.go | 125 - .../sarama/create_partitions_response.go | 109 - .../Shopify/sarama/create_topics_request.go | 178 - .../Shopify/sarama/create_topics_response.go | 127 - .../github.com/Shopify/sarama/decompress.go | 63 - .../Shopify/sarama/delete_groups_request.go | 34 - .../Shopify/sarama/delete_groups_response.go | 74 - .../Shopify/sarama/delete_records_request.go | 130 - .../Shopify/sarama/delete_records_response.go | 162 - .../Shopify/sarama/delete_topics_request.go | 52 - .../Shopify/sarama/delete_topics_response.go | 82 - .../sarama/describe_configs_request.go | 116 - .../sarama/describe_configs_response.go | 327 - .../Shopify/sarama/describe_groups_request.go | 34 - .../sarama/describe_groups_response.go | 191 - .../sarama/describe_log_dirs_request.go | 87 - .../sarama/describe_log_dirs_response.go | 229 - vendor/github.com/Shopify/sarama/dev.yml | 10 - .../Shopify/sarama/docker-compose.yml | 134 - .../Shopify/sarama/encoder_decoder.go | 94 - .../Shopify/sarama/end_txn_request.go | 54 - .../Shopify/sarama/end_txn_response.go | 48 - vendor/github.com/Shopify/sarama/errors.go | 385 - .../Shopify/sarama/fetch_request.go | 295 - .../Shopify/sarama/fetch_response.go | 546 -- .../sarama/find_coordinator_request.go | 65 - .../sarama/find_coordinator_response.go | 96 - vendor/github.com/Shopify/sarama/go.mod | 35 - vendor/github.com/Shopify/sarama/go.sum | 88 - .../Shopify/sarama/gssapi_kerberos.go | 258 - .../Shopify/sarama/heartbeat_request.go | 51 - .../Shopify/sarama/heartbeat_response.go | 36 - .../sarama/init_producer_id_request.go | 47 - .../sarama/init_producer_id_response.go | 59 - .../github.com/Shopify/sarama/interceptors.go | 43 - .../Shopify/sarama/join_group_request.go | 167 - .../Shopify/sarama/join_group_response.go | 139 - .../Shopify/sarama/kerberos_client.go | 46 - .../Shopify/sarama/leave_group_request.go | 44 - .../Shopify/sarama/leave_group_response.go | 36 - .../github.com/Shopify/sarama/length_field.go | 99 - .../Shopify/sarama/list_groups_request.go | 28 - .../Shopify/sarama/list_groups_response.go | 73 - .../list_partition_reassignments_request.go | 98 - .../list_partition_reassignments_response.go | 169 - vendor/github.com/Shopify/sarama/message.go | 174 - .../github.com/Shopify/sarama/message_set.go | 111 - .../Shopify/sarama/metadata_request.go | 85 - .../Shopify/sarama/metadata_response.go | 325 - vendor/github.com/Shopify/sarama/metrics.go | 43 - .../github.com/Shopify/sarama/mockbroker.go | 415 - .../github.com/Shopify/sarama/mockkerberos.go | 123 - .../Shopify/sarama/mockresponses.go | 1121 --- .../Shopify/sarama/offset_commit_request.go | 214 - .../Shopify/sarama/offset_commit_response.go | 114 - .../Shopify/sarama/offset_fetch_request.go | 104 - .../Shopify/sarama/offset_fetch_response.go | 201 - .../Shopify/sarama/offset_manager.go | 591 -- .../Shopify/sarama/offset_request.go | 160 - .../Shopify/sarama/offset_response.go | 178 - .../Shopify/sarama/packet_decoder.go | 67 - .../Shopify/sarama/packet_encoder.go | 72 - .../github.com/Shopify/sarama/partitioner.go | 217 - .../github.com/Shopify/sarama/prep_encoder.go | 202 - .../Shopify/sarama/produce_request.go | 258 - .../Shopify/sarama/produce_response.go | 212 - .../github.com/Shopify/sarama/produce_set.go | 273 - .../github.com/Shopify/sarama/real_decoder.go | 426 - .../github.com/Shopify/sarama/real_encoder.go | 208 - vendor/github.com/Shopify/sarama/record.go | 116 - .../github.com/Shopify/sarama/record_batch.go | 225 - vendor/github.com/Shopify/sarama/records.go | 203 - vendor/github.com/Shopify/sarama/request.go | 191 - .../Shopify/sarama/response_header.go | 31 - vendor/github.com/Shopify/sarama/sarama.go | 110 - .../sarama/sasl_authenticate_request.go | 33 - .../sarama/sasl_authenticate_response.go | 48 - .../Shopify/sarama/sasl_handshake_request.go | 38 - .../Shopify/sarama/sasl_handshake_response.go | 42 - .../sarama/sticky_assignor_user_data.go | 124 - .../Shopify/sarama/sync_group_request.go | 104 - .../Shopify/sarama/sync_group_response.go | 45 - .../Shopify/sarama/sync_producer.go | 149 - vendor/github.com/Shopify/sarama/timestamp.go | 40 - .../sarama/txn_offset_commit_request.go | 130 - .../sarama/txn_offset_commit_response.go | 87 - vendor/github.com/Shopify/sarama/utils.go | 229 - vendor/github.com/Shopify/sarama/zstd.go | 28 - .../sdk-go/protocol/kafka_sarama/v2/LICENSE | 201 - .../sdk-go/protocol/kafka_sarama/v2/doc.go | 4 - .../sdk-go/protocol/kafka_sarama/v2/go.mod | 15 - .../sdk-go/protocol/kafka_sarama/v2/go.sum | 156 - .../protocol/kafka_sarama/v2/message.go | 138 - .../sdk-go/protocol/kafka_sarama/v2/option.go | 23 - .../protocol/kafka_sarama/v2/protocol.go | 133 - .../protocol/kafka_sarama/v2/receiver.go | 175 - .../sdk-go/protocol/kafka_sarama/v2/sender.go | 83 - .../kafka_sarama/v2/write_producer_message.go | 158 - .../github.com/cloudevents/sdk-go/v2/LICENSE | 201 - .../github.com/cloudevents/sdk-go/v2/alias.go | 173 - .../sdk-go/v2/binding/binary_writer.go | 47 - .../cloudevents/sdk-go/v2/binding/doc.go | 63 - .../cloudevents/sdk-go/v2/binding/encoding.go | 40 - .../sdk-go/v2/binding/event_message.go | 103 - .../sdk-go/v2/binding/finish_message.go | 37 - .../sdk-go/v2/binding/format/doc.go | 7 - .../sdk-go/v2/binding/format/format.go | 78 - .../cloudevents/sdk-go/v2/binding/message.go | 141 - .../sdk-go/v2/binding/spec/attributes.go | 136 - .../cloudevents/sdk-go/v2/binding/spec/doc.go | 8 - .../v2/binding/spec/match_exact_version.go | 76 - .../sdk-go/v2/binding/spec/spec.go | 184 - .../sdk-go/v2/binding/structured_writer.go | 17 - .../cloudevents/sdk-go/v2/binding/to_event.go | 126 - .../sdk-go/v2/binding/transformer.go | 37 - .../cloudevents/sdk-go/v2/binding/write.go | 174 - .../cloudevents/sdk-go/v2/client/client.go | 249 - .../sdk-go/v2/client/client_default.go | 26 - .../sdk-go/v2/client/client_observed.go | 101 - .../sdk-go/v2/client/defaulters.go | 52 - .../cloudevents/sdk-go/v2/client/doc.go | 6 - .../sdk-go/v2/client/http_receiver.go | 45 - .../cloudevents/sdk-go/v2/client/invoker.go | 99 - .../sdk-go/v2/client/observability.go | 96 - .../cloudevents/sdk-go/v2/client/options.go | 85 - .../cloudevents/sdk-go/v2/client/receiver.go | 189 - .../cloudevents/sdk-go/v2/context/context.go | 105 - .../cloudevents/sdk-go/v2/context/doc.go | 5 - .../cloudevents/sdk-go/v2/context/logger.go | 43 - .../cloudevents/sdk-go/v2/context/retry.go | 71 - .../sdk-go/v2/event/content_type.go | 42 - .../sdk-go/v2/event/data_content_encoding.go | 11 - .../sdk-go/v2/event/datacodec/codec.go | 73 - .../v2/event/datacodec/codec_observed.go | 50 - .../sdk-go/v2/event/datacodec/doc.go | 5 - .../sdk-go/v2/event/datacodec/json/data.go | 51 - .../v2/event/datacodec/json/data_observed.go | 30 - .../sdk-go/v2/event/datacodec/json/doc.go | 4 - .../v2/event/datacodec/json/observability.go | 51 - .../v2/event/datacodec/observability.go | 51 - .../sdk-go/v2/event/datacodec/text/data.go | 25 - .../v2/event/datacodec/text/data_observed.go | 30 - .../sdk-go/v2/event/datacodec/text/doc.go | 4 - .../v2/event/datacodec/text/observability.go | 51 - .../sdk-go/v2/event/datacodec/xml/data.go | 35 - .../v2/event/datacodec/xml/data_observed.go | 30 - .../sdk-go/v2/event/datacodec/xml/doc.go | 4 - .../v2/event/datacodec/xml/observability.go | 51 - .../cloudevents/sdk-go/v2/event/doc.go | 4 - .../cloudevents/sdk-go/v2/event/event.go | 134 - .../cloudevents/sdk-go/v2/event/event_data.go | 113 - .../sdk-go/v2/event/event_interface.go | 97 - .../sdk-go/v2/event/event_marshal.go | 324 - .../sdk-go/v2/event/event_observability.go | 77 - .../sdk-go/v2/event/event_reader.go | 98 - .../sdk-go/v2/event/event_validation.go | 45 - .../sdk-go/v2/event/event_writer.go | 112 - .../sdk-go/v2/event/eventcontext.go | 120 - .../sdk-go/v2/event/eventcontext_v03.go | 307 - .../v2/event/eventcontext_v03_reader.go | 93 - .../v2/event/eventcontext_v03_writer.go | 98 - .../sdk-go/v2/event/eventcontext_v1.go | 297 - .../sdk-go/v2/event/eventcontext_v1_reader.go | 98 - .../sdk-go/v2/event/eventcontext_v1_writer.go | 92 - .../cloudevents/sdk-go/v2/event/extensions.go | 24 - .../distributed_tracing_extension.go | 164 - .../cloudevents/sdk-go/v2/extensions/doc.go | 2 - .../github.com/cloudevents/sdk-go/v2/go.mod | 28 - .../github.com/cloudevents/sdk-go/v2/go.sum | 115 - .../sdk-go/v2/observability/doc.go | 4 - .../sdk-go/v2/observability/keys.go | 22 - .../sdk-go/v2/observability/observer.go | 87 - .../cloudevents/sdk-go/v2/protocol/doc.go | 21 - .../cloudevents/sdk-go/v2/protocol/error.go | 37 - .../v2/protocol/http/abuse_protection.go | 121 - .../sdk-go/v2/protocol/http/doc.go | 4 - .../sdk-go/v2/protocol/http/headers.go | 33 - .../sdk-go/v2/protocol/http/message.go | 158 - .../sdk-go/v2/protocol/http/options.go | 237 - .../sdk-go/v2/protocol/http/protocol.go | 330 - .../v2/protocol/http/protocol_lifecycle.go | 125 - .../sdk-go/v2/protocol/http/protocol_retry.go | 107 - .../sdk-go/v2/protocol/http/result.go | 55 - .../sdk-go/v2/protocol/http/retries_result.go | 54 - .../sdk-go/v2/protocol/http/write_request.go | 136 - .../v2/protocol/http/write_responsewriter.go | 121 - .../cloudevents/sdk-go/v2/protocol/inbound.go | 47 - .../sdk-go/v2/protocol/lifecycle.go | 18 - .../sdk-go/v2/protocol/outbound.go | 44 - .../cloudevents/sdk-go/v2/protocol/result.go | 122 - .../cloudevents/sdk-go/v2/staticcheck.conf | 3 - .../cloudevents/sdk-go/v2/types/allocate.go | 36 - .../cloudevents/sdk-go/v2/types/doc.go | 41 - .../cloudevents/sdk-go/v2/types/timestamp.go | 70 - .../cloudevents/sdk-go/v2/types/uri.go | 77 - .../cloudevents/sdk-go/v2/types/uriref.go | 77 - .../cloudevents/sdk-go/v2/types/value.go | 330 - vendor/github.com/davecgh/go-spew/LICENSE | 15 - .../github.com/davecgh/go-spew/spew/bypass.go | 145 - .../davecgh/go-spew/spew/bypasssafe.go | 38 - .../github.com/davecgh/go-spew/spew/common.go | 341 - .../github.com/davecgh/go-spew/spew/config.go | 306 - vendor/github.com/davecgh/go-spew/spew/doc.go | 211 - .../github.com/davecgh/go-spew/spew/dump.go | 509 -- .../github.com/davecgh/go-spew/spew/format.go | 419 - .../github.com/davecgh/go-spew/spew/spew.go | 148 - .../github.com/eapache/go-resiliency/LICENSE | 22 - .../eapache/go-resiliency/breaker/README.md | 34 - .../eapache/go-resiliency/breaker/breaker.go | 161 - .../eapache/go-xerial-snappy/.gitignore | 24 - .../eapache/go-xerial-snappy/.travis.yml | 7 - .../eapache/go-xerial-snappy/LICENSE | 21 - .../eapache/go-xerial-snappy/README.md | 13 - .../eapache/go-xerial-snappy/fuzz.go | 16 - .../eapache/go-xerial-snappy/snappy.go | 131 - vendor/github.com/eapache/queue/.gitignore | 23 - vendor/github.com/eapache/queue/.travis.yml | 7 - vendor/github.com/eapache/queue/LICENSE | 21 - vendor/github.com/eapache/queue/README.md | 16 - vendor/github.com/eapache/queue/queue.go | 102 - vendor/github.com/globalsign/mgo/.gitignore | 2 - vendor/github.com/globalsign/mgo/.travis.yml | 49 - .../github.com/globalsign/mgo/CONTRIBUTING.md | 14 - vendor/github.com/globalsign/mgo/LICENSE | 25 - vendor/github.com/globalsign/mgo/Makefile | 5 - vendor/github.com/globalsign/mgo/README.md | 105 - vendor/github.com/globalsign/mgo/auth.go | 467 - vendor/github.com/globalsign/mgo/bson/LICENSE | 25 - .../github.com/globalsign/mgo/bson/README.md | 12 - vendor/github.com/globalsign/mgo/bson/bson.go | 836 -- .../globalsign/mgo/bson/compatibility.go | 29 - .../github.com/globalsign/mgo/bson/decimal.go | 312 - .../github.com/globalsign/mgo/bson/decode.go | 1055 --- .../github.com/globalsign/mgo/bson/encode.go | 645 -- vendor/github.com/globalsign/mgo/bson/json.go | 384 - .../github.com/globalsign/mgo/bson/stream.go | 90 - vendor/github.com/globalsign/mgo/bulk.go | 366 - .../globalsign/mgo/changestreams.go | 357 - vendor/github.com/globalsign/mgo/cluster.go | 704 -- .../github.com/globalsign/mgo/coarse_time.go | 62 - vendor/github.com/globalsign/mgo/doc.go | 35 - vendor/github.com/globalsign/mgo/gridfs.go | 782 -- .../globalsign/mgo/internal/json/LICENSE | 27 - .../globalsign/mgo/internal/json/decode.go | 1685 ---- .../globalsign/mgo/internal/json/encode.go | 1260 --- .../globalsign/mgo/internal/json/extension.go | 95 - .../globalsign/mgo/internal/json/fold.go | 143 - .../globalsign/mgo/internal/json/indent.go | 141 - .../globalsign/mgo/internal/json/scanner.go | 697 -- .../globalsign/mgo/internal/json/stream.go | 510 -- .../globalsign/mgo/internal/json/tags.go | 44 - .../globalsign/mgo/internal/sasl/sasl.c | 77 - .../globalsign/mgo/internal/sasl/sasl.go | 142 - .../mgo/internal/sasl/sasl_windows.c | 122 - .../mgo/internal/sasl/sasl_windows.go | 142 - .../mgo/internal/sasl/sasl_windows.h | 7 - .../mgo/internal/sasl/sspi_windows.c | 96 - .../mgo/internal/sasl/sspi_windows.h | 70 - .../globalsign/mgo/internal/scram/scram.go | 266 - vendor/github.com/globalsign/mgo/log.go | 133 - vendor/github.com/globalsign/mgo/queue.go | 91 - vendor/github.com/globalsign/mgo/raceoff.go | 5 - vendor/github.com/globalsign/mgo/raceon.go | 5 - vendor/github.com/globalsign/mgo/saslimpl.go | 11 - vendor/github.com/globalsign/mgo/saslstub.go | 11 - vendor/github.com/globalsign/mgo/server.go | 609 -- vendor/github.com/globalsign/mgo/session.go | 5656 ------------ vendor/github.com/globalsign/mgo/socket.go | 756 -- vendor/github.com/globalsign/mgo/stats.go | 184 - vendor/github.com/golang/snappy/.gitignore | 16 - vendor/github.com/golang/snappy/AUTHORS | 15 - vendor/github.com/golang/snappy/CONTRIBUTORS | 37 - vendor/github.com/golang/snappy/LICENSE | 27 - vendor/github.com/golang/snappy/README | 107 - vendor/github.com/golang/snappy/decode.go | 237 - .../github.com/golang/snappy/decode_amd64.go | 14 - .../github.com/golang/snappy/decode_amd64.s | 490 -- .../github.com/golang/snappy/decode_other.go | 101 - vendor/github.com/golang/snappy/encode.go | 285 - .../github.com/golang/snappy/encode_amd64.go | 29 - .../github.com/golang/snappy/encode_amd64.s | 730 -- .../github.com/golang/snappy/encode_other.go | 238 - vendor/github.com/golang/snappy/go.mod | 1 - vendor/github.com/golang/snappy/snappy.go | 98 - vendor/github.com/google/uuid/.travis.yml | 9 - vendor/github.com/google/uuid/CONTRIBUTING.md | 10 - vendor/github.com/google/uuid/CONTRIBUTORS | 9 - vendor/github.com/google/uuid/LICENSE | 27 - vendor/github.com/google/uuid/README.md | 19 - vendor/github.com/google/uuid/dce.go | 80 - vendor/github.com/google/uuid/doc.go | 12 - vendor/github.com/google/uuid/go.mod | 1 - vendor/github.com/google/uuid/hash.go | 53 - vendor/github.com/google/uuid/marshal.go | 37 - vendor/github.com/google/uuid/node.go | 90 - vendor/github.com/google/uuid/node_js.go | 12 - vendor/github.com/google/uuid/node_net.go | 33 - vendor/github.com/google/uuid/sql.go | 59 - vendor/github.com/google/uuid/time.go | 123 - vendor/github.com/google/uuid/util.go | 43 - vendor/github.com/google/uuid/uuid.go | 245 - vendor/github.com/google/uuid/version1.go | 44 - vendor/github.com/google/uuid/version4.go | 38 - .../github.com/hashicorp/go-uuid/.travis.yml | 12 - vendor/github.com/hashicorp/go-uuid/LICENSE | 363 - vendor/github.com/hashicorp/go-uuid/README.md | 8 - vendor/github.com/hashicorp/go-uuid/go.mod | 1 - vendor/github.com/hashicorp/go-uuid/uuid.go | 83 - .../github.com/hashicorp/golang-lru/LICENSE | 362 - .../hashicorp/golang-lru/simplelru/lru.go | 177 - .../golang-lru/simplelru/lru_interface.go | 39 - vendor/github.com/jcmturner/gofork/LICENSE | 27 - .../jcmturner/gofork/encoding/asn1/README.md | 5 - .../jcmturner/gofork/encoding/asn1/asn1.go | 1003 --- .../jcmturner/gofork/encoding/asn1/common.go | 173 - .../jcmturner/gofork/encoding/asn1/marshal.go | 659 -- .../gofork/x/crypto/pbkdf2/pbkdf2.go | 98 - .../kelseyhightower/envconfig/.travis.yml | 13 - .../kelseyhightower/envconfig/LICENSE | 19 - .../kelseyhightower/envconfig/MAINTAINERS | 2 - .../kelseyhightower/envconfig/README.md | 192 - .../kelseyhightower/envconfig/doc.go | 8 - .../kelseyhightower/envconfig/env_os.go | 7 - .../kelseyhightower/envconfig/env_syscall.go | 7 - .../kelseyhightower/envconfig/envconfig.go | 382 - .../kelseyhightower/envconfig/go.mod | 1 - .../kelseyhightower/envconfig/usage.go | 164 - vendor/github.com/klauspost/compress/LICENSE | 28 - .../klauspost/compress/fse/README.md | 79 - .../klauspost/compress/fse/bitreader.go | 122 - .../klauspost/compress/fse/bitwriter.go | 168 - .../klauspost/compress/fse/bytereader.go | 47 - .../klauspost/compress/fse/compress.go | 684 -- .../klauspost/compress/fse/decompress.go | 374 - .../github.com/klauspost/compress/fse/fse.go | 144 - .../klauspost/compress/huff0/.gitignore | 1 - .../klauspost/compress/huff0/README.md | 87 - .../klauspost/compress/huff0/bitreader.go | 329 - .../klauspost/compress/huff0/bitwriter.go | 197 - .../klauspost/compress/huff0/bytereader.go | 54 - .../klauspost/compress/huff0/compress.go | 651 -- .../klauspost/compress/huff0/decompress.go | 1146 --- .../klauspost/compress/huff0/huff0.go | 260 - .../klauspost/compress/snappy/.gitignore | 16 - .../klauspost/compress/snappy/AUTHORS | 15 - .../klauspost/compress/snappy/CONTRIBUTORS | 37 - .../klauspost/compress/snappy/LICENSE | 27 - .../klauspost/compress/snappy/README | 107 - .../klauspost/compress/snappy/decode.go | 237 - .../klauspost/compress/snappy/decode_amd64.go | 14 - .../klauspost/compress/snappy/decode_amd64.s | 482 -- .../klauspost/compress/snappy/decode_other.go | 115 - .../klauspost/compress/snappy/encode.go | 285 - .../klauspost/compress/snappy/encode_amd64.go | 29 - .../klauspost/compress/snappy/encode_amd64.s | 730 -- .../klauspost/compress/snappy/encode_other.go | 238 - .../klauspost/compress/snappy/runbench.cmd | 2 - .../klauspost/compress/snappy/snappy.go | 98 - .../klauspost/compress/zstd/README.md | 427 - .../klauspost/compress/zstd/bitreader.go | 136 - .../klauspost/compress/zstd/bitwriter.go | 169 - .../klauspost/compress/zstd/blockdec.go | 739 -- .../klauspost/compress/zstd/blockenc.go | 837 -- .../compress/zstd/blocktype_string.go | 85 - .../klauspost/compress/zstd/bytebuf.go | 127 - .../klauspost/compress/zstd/bytereader.go | 88 - .../klauspost/compress/zstd/decoder.go | 540 -- .../compress/zstd/decoder_options.go | 84 - .../klauspost/compress/zstd/dict.go | 104 - .../klauspost/compress/zstd/enc_better.go | 518 -- .../klauspost/compress/zstd/enc_dfast.go | 678 -- .../klauspost/compress/zstd/enc_fast.go | 755 -- .../klauspost/compress/zstd/enc_params.go | 157 - .../klauspost/compress/zstd/encoder.go | 560 -- .../compress/zstd/encoder_options.go | 267 - .../klauspost/compress/zstd/framedec.go | 494 -- .../klauspost/compress/zstd/frameenc.go | 115 - .../klauspost/compress/zstd/fse_decoder.go | 385 - .../klauspost/compress/zstd/fse_encoder.go | 726 -- .../klauspost/compress/zstd/fse_predefined.go | 158 - .../klauspost/compress/zstd/hash.go | 77 - .../klauspost/compress/zstd/history.go | 89 - .../compress/zstd/internal/xxhash/LICENSE.txt | 22 - .../compress/zstd/internal/xxhash/README.md | 58 - .../compress/zstd/internal/xxhash/xxhash.go | 238 - .../zstd/internal/xxhash/xxhash_amd64.go | 13 - .../zstd/internal/xxhash/xxhash_amd64.s | 215 - .../zstd/internal/xxhash/xxhash_other.go | 76 - .../zstd/internal/xxhash/xxhash_safe.go | 11 - .../klauspost/compress/zstd/seqdec.go | 485 -- .../klauspost/compress/zstd/seqenc.go | 115 - .../klauspost/compress/zstd/snappy.go | 436 - .../klauspost/compress/zstd/zstd.go | 144 - .../lightstep/tracecontext.go/LICENSE | 21 - .../tracecontext.go/traceparent/package.go | 192 - .../tracecontext.go/tracestate/package.go | 123 - vendor/github.com/pierrec/lz4/.gitignore | 34 - vendor/github.com/pierrec/lz4/.travis.yml | 24 - vendor/github.com/pierrec/lz4/LICENSE | 28 - vendor/github.com/pierrec/lz4/README.md | 90 - vendor/github.com/pierrec/lz4/block.go | 413 - vendor/github.com/pierrec/lz4/debug.go | 23 - vendor/github.com/pierrec/lz4/debug_stub.go | 7 - vendor/github.com/pierrec/lz4/decode_amd64.go | 8 - vendor/github.com/pierrec/lz4/decode_amd64.s | 375 - vendor/github.com/pierrec/lz4/decode_other.go | 98 - vendor/github.com/pierrec/lz4/errors.go | 30 - .../pierrec/lz4/internal/xxh32/xxh32zero.go | 223 - vendor/github.com/pierrec/lz4/lz4.go | 113 - vendor/github.com/pierrec/lz4/lz4_go1.10.go | 29 - .../github.com/pierrec/lz4/lz4_notgo1.10.go | 29 - vendor/github.com/pierrec/lz4/reader.go | 335 - vendor/github.com/pierrec/lz4/writer.go | 408 - .../github.com/rcrowley/go-metrics/.gitignore | 9 - .../rcrowley/go-metrics/.travis.yml | 22 - vendor/github.com/rcrowley/go-metrics/LICENSE | 29 - .../github.com/rcrowley/go-metrics/README.md | 171 - .../github.com/rcrowley/go-metrics/counter.go | 112 - .../github.com/rcrowley/go-metrics/debug.go | 80 - vendor/github.com/rcrowley/go-metrics/ewma.go | 138 - .../github.com/rcrowley/go-metrics/gauge.go | 120 - .../rcrowley/go-metrics/gauge_float64.go | 125 - .../rcrowley/go-metrics/graphite.go | 113 - .../rcrowley/go-metrics/healthcheck.go | 61 - .../rcrowley/go-metrics/histogram.go | 202 - vendor/github.com/rcrowley/go-metrics/json.go | 31 - vendor/github.com/rcrowley/go-metrics/log.go | 100 - .../github.com/rcrowley/go-metrics/memory.md | 285 - .../github.com/rcrowley/go-metrics/meter.go | 251 - .../github.com/rcrowley/go-metrics/metrics.go | 13 - .../rcrowley/go-metrics/opentsdb.go | 119 - .../rcrowley/go-metrics/registry.go | 373 - .../github.com/rcrowley/go-metrics/runtime.go | 216 - .../rcrowley/go-metrics/runtime_cgo.go | 10 - .../go-metrics/runtime_gccpufraction.go | 9 - .../rcrowley/go-metrics/runtime_no_cgo.go | 7 - .../go-metrics/runtime_no_gccpufraction.go | 9 - .../github.com/rcrowley/go-metrics/sample.go | 616 -- .../github.com/rcrowley/go-metrics/syslog.go | 78 - .../github.com/rcrowley/go-metrics/timer.go | 329 - .../rcrowley/go-metrics/validate.sh | 10 - .../github.com/rcrowley/go-metrics/writer.go | 100 - vendor/github.com/xdg/scram/.gitignore | 0 vendor/github.com/xdg/scram/.travis.yml | 11 - vendor/github.com/xdg/scram/LICENSE | 175 - vendor/github.com/xdg/scram/README.md | 71 - vendor/github.com/xdg/scram/client.go | 130 - vendor/github.com/xdg/scram/client_conv.go | 149 - vendor/github.com/xdg/scram/common.go | 97 - vendor/github.com/xdg/scram/doc.go | 24 - vendor/github.com/xdg/scram/parse.go | 205 - vendor/github.com/xdg/scram/scram.go | 66 - vendor/github.com/xdg/scram/server.go | 50 - vendor/github.com/xdg/scram/server_conv.go | 151 - vendor/github.com/xdg/stringprep/.gitignore | 0 vendor/github.com/xdg/stringprep/.travis.yml | 10 - vendor/github.com/xdg/stringprep/LICENSE | 175 - vendor/github.com/xdg/stringprep/README.md | 27 - vendor/github.com/xdg/stringprep/bidi.go | 73 - vendor/github.com/xdg/stringprep/doc.go | 10 - vendor/github.com/xdg/stringprep/error.go | 14 - vendor/github.com/xdg/stringprep/map.go | 21 - vendor/github.com/xdg/stringprep/profile.go | 75 - vendor/github.com/xdg/stringprep/saslprep.go | 52 - vendor/github.com/xdg/stringprep/set.go | 36 - vendor/github.com/xdg/stringprep/tables.go | 3215 ------- vendor/go.opencensus.io/.gitignore | 9 - vendor/go.opencensus.io/.travis.yml | 17 - vendor/go.opencensus.io/AUTHORS | 1 - vendor/go.opencensus.io/CONTRIBUTING.md | 63 - vendor/go.opencensus.io/Gopkg.lock | 231 - vendor/go.opencensus.io/Gopkg.toml | 36 - vendor/go.opencensus.io/LICENSE | 202 - vendor/go.opencensus.io/Makefile | 96 - vendor/go.opencensus.io/README.md | 263 - vendor/go.opencensus.io/appveyor.yml | 25 - vendor/go.opencensus.io/go.mod | 12 - vendor/go.opencensus.io/go.sum | 61 - vendor/go.opencensus.io/internal/internal.go | 37 - vendor/go.opencensus.io/internal/sanitize.go | 50 - .../internal/tagencoding/tagencoding.go | 75 - .../internal/traceinternals.go | 53 - .../go.opencensus.io/metric/metricdata/doc.go | 19 - .../metric/metricdata/exemplar.go | 38 - .../metric/metricdata/label.go | 35 - .../metric/metricdata/metric.go | 46 - .../metric/metricdata/point.go | 193 - .../metric/metricdata/type_string.go | 16 - .../metric/metricdata/unit.go | 27 - .../metric/metricproducer/manager.go | 78 - .../metric/metricproducer/producer.go | 28 - vendor/go.opencensus.io/opencensus.go | 21 - .../go.opencensus.io/plugin/ochttp/client.go | 117 - .../plugin/ochttp/client_stats.go | 143 - vendor/go.opencensus.io/plugin/ochttp/doc.go | 19 - .../plugin/ochttp/propagation/b3/b3.go | 123 - .../propagation/tracecontext/propagation.go | 187 - .../go.opencensus.io/plugin/ochttp/route.go | 61 - .../go.opencensus.io/plugin/ochttp/server.go | 449 - .../ochttp/span_annotating_client_trace.go | 169 - .../go.opencensus.io/plugin/ochttp/stats.go | 292 - .../go.opencensus.io/plugin/ochttp/trace.go | 239 - .../plugin/ochttp/wrapped_body.go | 44 - vendor/go.opencensus.io/resource/resource.go | 164 - vendor/go.opencensus.io/stats/doc.go | 69 - .../go.opencensus.io/stats/internal/record.go | 25 - vendor/go.opencensus.io/stats/measure.go | 109 - .../go.opencensus.io/stats/measure_float64.go | 55 - .../go.opencensus.io/stats/measure_int64.go | 55 - vendor/go.opencensus.io/stats/record.go | 117 - vendor/go.opencensus.io/stats/units.go | 25 - .../stats/view/aggregation.go | 120 - .../stats/view/aggregation_data.go | 293 - .../go.opencensus.io/stats/view/collector.go | 86 - vendor/go.opencensus.io/stats/view/doc.go | 47 - vendor/go.opencensus.io/stats/view/export.go | 58 - vendor/go.opencensus.io/stats/view/view.go | 221 - .../stats/view/view_to_metric.go | 140 - vendor/go.opencensus.io/stats/view/worker.go | 281 - .../stats/view/worker_commands.go | 186 - vendor/go.opencensus.io/tag/context.go | 43 - vendor/go.opencensus.io/tag/doc.go | 26 - vendor/go.opencensus.io/tag/key.go | 45 - vendor/go.opencensus.io/tag/map.go | 229 - vendor/go.opencensus.io/tag/map_codec.go | 239 - vendor/go.opencensus.io/tag/metadata.go | 52 - vendor/go.opencensus.io/tag/profile_19.go | 31 - vendor/go.opencensus.io/tag/profile_not19.go | 23 - vendor/go.opencensus.io/tag/validate.go | 56 - vendor/go.opencensus.io/trace/basetypes.go | 119 - vendor/go.opencensus.io/trace/config.go | 86 - vendor/go.opencensus.io/trace/doc.go | 53 - vendor/go.opencensus.io/trace/evictedqueue.go | 38 - vendor/go.opencensus.io/trace/export.go | 97 - .../trace/internal/internal.go | 22 - vendor/go.opencensus.io/trace/lrumap.go | 37 - .../trace/propagation/propagation.go | 108 - vendor/go.opencensus.io/trace/sampling.go | 75 - vendor/go.opencensus.io/trace/spanbucket.go | 130 - vendor/go.opencensus.io/trace/spanstore.go | 306 - vendor/go.opencensus.io/trace/status_codes.go | 37 - vendor/go.opencensus.io/trace/trace.go | 598 -- vendor/go.opencensus.io/trace/trace_go11.go | 32 - .../go.opencensus.io/trace/trace_nongo11.go | 25 - .../trace/tracestate/tracestate.go | 147 - vendor/go.uber.org/atomic/.codecov.yml | 15 - vendor/go.uber.org/atomic/.gitignore | 11 - vendor/go.uber.org/atomic/.travis.yml | 27 - vendor/go.uber.org/atomic/LICENSE.txt | 19 - vendor/go.uber.org/atomic/Makefile | 51 - vendor/go.uber.org/atomic/README.md | 36 - vendor/go.uber.org/atomic/atomic.go | 351 - vendor/go.uber.org/atomic/error.go | 55 - vendor/go.uber.org/atomic/glide.lock | 17 - vendor/go.uber.org/atomic/glide.yaml | 6 - vendor/go.uber.org/atomic/string.go | 49 - vendor/go.uber.org/multierr/.codecov.yml | 15 - vendor/go.uber.org/multierr/.gitignore | 1 - vendor/go.uber.org/multierr/.travis.yml | 33 - vendor/go.uber.org/multierr/CHANGELOG.md | 28 - vendor/go.uber.org/multierr/LICENSE.txt | 19 - vendor/go.uber.org/multierr/Makefile | 74 - vendor/go.uber.org/multierr/README.md | 23 - vendor/go.uber.org/multierr/error.go | 401 - vendor/go.uber.org/multierr/glide.lock | 19 - vendor/go.uber.org/multierr/glide.yaml | 8 - vendor/go.uber.org/zap/.codecov.yml | 17 - vendor/go.uber.org/zap/.gitignore | 28 - vendor/go.uber.org/zap/.readme.tmpl | 108 - vendor/go.uber.org/zap/.travis.yml | 21 - vendor/go.uber.org/zap/CHANGELOG.md | 327 - vendor/go.uber.org/zap/CODE_OF_CONDUCT.md | 75 - vendor/go.uber.org/zap/CONTRIBUTING.md | 81 - vendor/go.uber.org/zap/FAQ.md | 155 - vendor/go.uber.org/zap/LICENSE.txt | 19 - vendor/go.uber.org/zap/Makefile | 76 - vendor/go.uber.org/zap/README.md | 136 - vendor/go.uber.org/zap/array.go | 320 - vendor/go.uber.org/zap/buffer/buffer.go | 115 - vendor/go.uber.org/zap/buffer/pool.go | 49 - vendor/go.uber.org/zap/check_license.sh | 17 - vendor/go.uber.org/zap/config.go | 243 - vendor/go.uber.org/zap/doc.go | 113 - vendor/go.uber.org/zap/encoder.go | 75 - vendor/go.uber.org/zap/error.go | 80 - vendor/go.uber.org/zap/field.go | 310 - vendor/go.uber.org/zap/flag.go | 39 - vendor/go.uber.org/zap/glide.lock | 76 - vendor/go.uber.org/zap/glide.yaml | 35 - vendor/go.uber.org/zap/global.go | 168 - vendor/go.uber.org/zap/global_go112.go | 26 - vendor/go.uber.org/zap/global_prego112.go | 26 - vendor/go.uber.org/zap/http_handler.go | 81 - .../zap/internal/bufferpool/bufferpool.go | 31 - .../go.uber.org/zap/internal/color/color.go | 44 - vendor/go.uber.org/zap/internal/exit/exit.go | 64 - vendor/go.uber.org/zap/level.go | 132 - vendor/go.uber.org/zap/logger.go | 305 - vendor/go.uber.org/zap/options.go | 109 - vendor/go.uber.org/zap/sink.go | 161 - vendor/go.uber.org/zap/stacktrace.go | 126 - vendor/go.uber.org/zap/sugar.go | 304 - vendor/go.uber.org/zap/time.go | 27 - vendor/go.uber.org/zap/writer.go | 99 - .../zap/zapcore/console_encoder.go | 147 - vendor/go.uber.org/zap/zapcore/core.go | 113 - vendor/go.uber.org/zap/zapcore/doc.go | 24 - vendor/go.uber.org/zap/zapcore/encoder.go | 348 - vendor/go.uber.org/zap/zapcore/entry.go | 257 - vendor/go.uber.org/zap/zapcore/error.go | 120 - vendor/go.uber.org/zap/zapcore/field.go | 212 - vendor/go.uber.org/zap/zapcore/hook.go | 68 - .../go.uber.org/zap/zapcore/json_encoder.go | 505 -- vendor/go.uber.org/zap/zapcore/level.go | 175 - .../go.uber.org/zap/zapcore/level_strings.go | 46 - vendor/go.uber.org/zap/zapcore/marshaler.go | 53 - .../go.uber.org/zap/zapcore/memory_encoder.go | 179 - vendor/go.uber.org/zap/zapcore/sampler.go | 134 - vendor/go.uber.org/zap/zapcore/tee.go | 81 - .../go.uber.org/zap/zapcore/write_syncer.go | 123 - vendor/golang.org/x/crypto/AUTHORS | 3 - vendor/golang.org/x/crypto/CONTRIBUTORS | 3 - vendor/golang.org/x/crypto/LICENSE | 27 - vendor/golang.org/x/crypto/PATENTS | 22 - vendor/golang.org/x/crypto/md4/md4.go | 122 - vendor/golang.org/x/crypto/md4/md4block.go | 89 - vendor/golang.org/x/crypto/pbkdf2/pbkdf2.go | 77 - vendor/golang.org/x/net/AUTHORS | 3 - vendor/golang.org/x/net/CONTRIBUTORS | 3 - vendor/golang.org/x/net/LICENSE | 27 - vendor/golang.org/x/net/PATENTS | 22 - .../golang.org/x/net/internal/socks/client.go | 168 - .../golang.org/x/net/internal/socks/socks.go | 317 - vendor/golang.org/x/net/proxy/dial.go | 54 - vendor/golang.org/x/net/proxy/direct.go | 31 - vendor/golang.org/x/net/proxy/per_host.go | 155 - vendor/golang.org/x/net/proxy/proxy.go | 149 - vendor/golang.org/x/net/proxy/socks5.go | 42 - vendor/golang.org/x/text/AUTHORS | 3 - vendor/golang.org/x/text/CONTRIBUTORS | 3 - vendor/golang.org/x/text/LICENSE | 27 - vendor/golang.org/x/text/PATENTS | 22 - .../golang.org/x/text/transform/transform.go | 705 -- .../x/text/unicode/norm/composition.go | 512 -- .../x/text/unicode/norm/forminfo.go | 278 - .../golang.org/x/text/unicode/norm/input.go | 109 - vendor/golang.org/x/text/unicode/norm/iter.go | 458 - .../x/text/unicode/norm/normalize.go | 609 -- .../x/text/unicode/norm/readwriter.go | 125 - .../x/text/unicode/norm/tables10.0.0.go | 7657 ---------------- .../x/text/unicode/norm/tables11.0.0.go | 7693 ----------------- .../x/text/unicode/norm/tables9.0.0.go | 7637 ---------------- .../x/text/unicode/norm/transform.go | 88 - vendor/golang.org/x/text/unicode/norm/trie.go | 54 - .../gopkg.in/jcmturner/aescts.v1/.gitignore | 14 - vendor/gopkg.in/jcmturner/aescts.v1/LICENSE | 201 - vendor/gopkg.in/jcmturner/aescts.v1/README.md | 16 - vendor/gopkg.in/jcmturner/aescts.v1/aescts.go | 186 - .../gopkg.in/jcmturner/dnsutils.v1/.gitignore | 14 - .../jcmturner/dnsutils.v1/.travis.yml | 24 - vendor/gopkg.in/jcmturner/dnsutils.v1/LICENSE | 201 - vendor/gopkg.in/jcmturner/dnsutils.v1/srv.go | 95 - vendor/gopkg.in/jcmturner/gokrb5.v7/LICENSE | 201 - .../jcmturner/gokrb5.v7/asn1tools/tools.go | 86 - .../jcmturner/gokrb5.v7/client/ASExchange.go | 189 - .../jcmturner/gokrb5.v7/client/TGSExchange.go | 103 - .../jcmturner/gokrb5.v7/client/cache.go | 110 - .../jcmturner/gokrb5.v7/client/client.go | 241 - .../jcmturner/gokrb5.v7/client/network.go | 224 - .../jcmturner/gokrb5.v7/client/passwd.go | 95 - .../jcmturner/gokrb5.v7/client/session.go | 255 - .../jcmturner/gokrb5.v7/client/settings.go | 69 - .../jcmturner/gokrb5.v7/config/error.go | 30 - .../jcmturner/gokrb5.v7/config/hosts.go | 141 - .../jcmturner/gokrb5.v7/config/krb5conf.go | 726 -- .../jcmturner/gokrb5.v7/credentials/ccache.go | 348 - .../gokrb5.v7/credentials/credentials.go | 314 - .../crypto/aes128-cts-hmac-sha1-96.go | 173 - .../crypto/aes128-cts-hmac-sha256-128.go | 135 - .../crypto/aes256-cts-hmac-sha1-96.go | 173 - .../crypto/aes256-cts-hmac-sha384-192.go | 135 - .../gokrb5.v7/crypto/common/common.go | 143 - .../jcmturner/gokrb5.v7/crypto/crypto.go | 175 - .../gokrb5.v7/crypto/des3-cbc-sha1-kd.go | 174 - .../jcmturner/gokrb5.v7/crypto/etype/etype.go | 29 - .../jcmturner/gokrb5.v7/crypto/rc4-hmac.go | 135 - .../gokrb5.v7/crypto/rfc3961/encryption.go | 125 - .../gokrb5.v7/crypto/rfc3961/keyDerivation.go | 178 - .../gokrb5.v7/crypto/rfc3961/nfold.go | 128 - .../gokrb5.v7/crypto/rfc3962/encryption.go | 89 - .../gokrb5.v7/crypto/rfc3962/keyDerivation.go | 58 - .../gokrb5.v7/crypto/rfc4757/checksum.go | 40 - .../gokrb5.v7/crypto/rfc4757/encryption.go | 80 - .../gokrb5.v7/crypto/rfc4757/keyDerivation.go | 55 - .../gokrb5.v7/crypto/rfc4757/msgtype.go | 20 - .../gokrb5.v7/crypto/rfc8009/encryption.go | 128 - .../gokrb5.v7/crypto/rfc8009/keyDerivation.go | 144 - .../jcmturner/gokrb5.v7/gssapi/MICToken.go | 202 - .../jcmturner/gokrb5.v7/gssapi/README.md | 20 - .../gokrb5.v7/gssapi/contextFlags.go | 25 - .../jcmturner/gokrb5.v7/gssapi/gssapi.go | 199 - .../jcmturner/gokrb5.v7/gssapi/wrapToken.go | 235 - .../gokrb5.v7/iana/addrtype/constants.go | 15 - .../gokrb5.v7/iana/adtype/constants.go | 23 - .../gokrb5.v7/iana/asnAppTag/constants.go | 24 - .../gokrb5.v7/iana/chksumtype/constants.go | 32 - .../jcmturner/gokrb5.v7/iana/constants.go | 5 - .../gokrb5.v7/iana/errorcode/constants.go | 155 - .../gokrb5.v7/iana/etypeID/constants.go | 101 - .../gokrb5.v7/iana/flags/constants.go | 36 - .../gokrb5.v7/iana/keyusage/constants.go | 42 - .../gokrb5.v7/iana/msgtype/constants.go | 18 - .../gokrb5.v7/iana/nametype/constants.go | 15 - .../gokrb5.v7/iana/patype/constants.go | 77 - .../gokrb5.v7/kadmin/changepasswddata.go | 23 - .../jcmturner/gokrb5.v7/kadmin/message.go | 114 - .../jcmturner/gokrb5.v7/kadmin/passwd.go | 68 - .../jcmturner/gokrb5.v7/keytab/keytab.go | 466 - .../jcmturner/gokrb5.v7/krberror/error.go | 67 - .../jcmturner/gokrb5.v7/messages/APRep.go | 64 - .../jcmturner/gokrb5.v7/messages/APReq.go | 220 - .../jcmturner/gokrb5.v7/messages/KDCRep.go | 312 - .../jcmturner/gokrb5.v7/messages/KDCReq.go | 432 - .../jcmturner/gokrb5.v7/messages/KRBCred.go | 102 - .../jcmturner/gokrb5.v7/messages/KRBError.go | 83 - .../jcmturner/gokrb5.v7/messages/KRBPriv.go | 108 - .../jcmturner/gokrb5.v7/messages/KRBSafe.go | 61 - .../jcmturner/gokrb5.v7/messages/Ticket.go | 265 - .../jcmturner/gokrb5.v7/pac/client_claims.go | 33 - .../jcmturner/gokrb5.v7/pac/client_info.go | 31 - .../gokrb5.v7/pac/credentials_info.go | 86 - .../jcmturner/gokrb5.v7/pac/device_claims.go | 33 - .../jcmturner/gokrb5.v7/pac/device_info.go | 32 - .../gokrb5.v7/pac/kerb_validation_info.go | 115 - .../jcmturner/gokrb5.v7/pac/pac_type.go | 251 - .../gokrb5.v7/pac/s4u_delegation_info.go | 26 - .../jcmturner/gokrb5.v7/pac/signature_data.go | 79 - .../gokrb5.v7/pac/supplemental_cred.go | 90 - .../jcmturner/gokrb5.v7/pac/upn_dns_info.go | 73 - .../gokrb5.v7/types/Authenticator.go | 100 - .../gokrb5.v7/types/AuthorizationData.go | 123 - .../jcmturner/gokrb5.v7/types/Cryptosystem.go | 55 - .../jcmturner/gokrb5.v7/types/HostAddress.go | 204 - .../gokrb5.v7/types/KerberosFlags.go | 124 - .../jcmturner/gokrb5.v7/types/PAData.go | 155 - .../gokrb5.v7/types/PrincipalName.go | 64 - .../jcmturner/gokrb5.v7/types/TypedData.go | 18 - vendor/gopkg.in/jcmturner/rpc.v1/LICENSE | 201 - .../jcmturner/rpc.v1/mstypes/claims.go | 134 - .../jcmturner/rpc.v1/mstypes/common.go | 10 - .../jcmturner/rpc.v1/mstypes/filetime.go | 52 - .../rpc.v1/mstypes/group_membership.go | 19 - .../rpc.v1/mstypes/kerb_sid_and_attributes.go | 23 - .../jcmturner/rpc.v1/mstypes/reader.go | 109 - .../rpc.v1/mstypes/rpc_unicode_string.go | 13 - .../gopkg.in/jcmturner/rpc.v1/mstypes/sid.go | 32 - .../rpc.v1/mstypes/user_session_key.go | 11 - .../gopkg.in/jcmturner/rpc.v1/ndr/arrays.go | 413 - .../gopkg.in/jcmturner/rpc.v1/ndr/decoder.go | 393 - vendor/gopkg.in/jcmturner/rpc.v1/ndr/error.go | 18 - .../gopkg.in/jcmturner/rpc.v1/ndr/header.go | 116 - vendor/gopkg.in/jcmturner/rpc.v1/ndr/pipe.go | 31 - .../jcmturner/rpc.v1/ndr/primitives.go | 211 - .../gopkg.in/jcmturner/rpc.v1/ndr/rawbytes.go | 61 - .../gopkg.in/jcmturner/rpc.v1/ndr/strings.go | 70 - vendor/gopkg.in/jcmturner/rpc.v1/ndr/tags.go | 69 - vendor/gopkg.in/jcmturner/rpc.v1/ndr/union.go | 57 - vendor/modules.txt | 148 - 812 files changed, 49 insertions(+), 149116 deletions(-) create mode 100644 tracing/init.go delete mode 100644 vendor/github.com/Shopify/sarama/.gitignore delete mode 100644 vendor/github.com/Shopify/sarama/.golangci.yml delete mode 100644 vendor/github.com/Shopify/sarama/CHANGELOG.md delete mode 100644 vendor/github.com/Shopify/sarama/LICENSE delete mode 100644 vendor/github.com/Shopify/sarama/Makefile delete mode 100644 vendor/github.com/Shopify/sarama/README.md delete mode 100644 vendor/github.com/Shopify/sarama/Vagrantfile delete mode 100644 vendor/github.com/Shopify/sarama/acl_bindings.go delete mode 100644 vendor/github.com/Shopify/sarama/acl_create_request.go delete mode 100644 vendor/github.com/Shopify/sarama/acl_create_response.go delete mode 100644 vendor/github.com/Shopify/sarama/acl_delete_request.go delete mode 100644 vendor/github.com/Shopify/sarama/acl_delete_response.go delete mode 100644 vendor/github.com/Shopify/sarama/acl_describe_request.go delete mode 100644 vendor/github.com/Shopify/sarama/acl_describe_response.go delete mode 100644 vendor/github.com/Shopify/sarama/acl_filter.go delete mode 100644 vendor/github.com/Shopify/sarama/acl_types.go delete mode 100644 vendor/github.com/Shopify/sarama/add_offsets_to_txn_request.go delete mode 100644 vendor/github.com/Shopify/sarama/add_offsets_to_txn_response.go delete mode 100644 vendor/github.com/Shopify/sarama/add_partitions_to_txn_request.go delete mode 100644 vendor/github.com/Shopify/sarama/add_partitions_to_txn_response.go delete mode 100644 vendor/github.com/Shopify/sarama/admin.go delete mode 100644 vendor/github.com/Shopify/sarama/alter_configs_request.go delete mode 100644 vendor/github.com/Shopify/sarama/alter_configs_response.go delete mode 100644 vendor/github.com/Shopify/sarama/alter_partition_reassignments_request.go delete mode 100644 vendor/github.com/Shopify/sarama/alter_partition_reassignments_response.go delete mode 100644 vendor/github.com/Shopify/sarama/api_versions_request.go delete mode 100644 vendor/github.com/Shopify/sarama/api_versions_response.go delete mode 100644 vendor/github.com/Shopify/sarama/async_producer.go delete mode 100644 vendor/github.com/Shopify/sarama/balance_strategy.go delete mode 100644 vendor/github.com/Shopify/sarama/broker.go delete mode 100644 vendor/github.com/Shopify/sarama/client.go delete mode 100644 vendor/github.com/Shopify/sarama/compress.go delete mode 100644 vendor/github.com/Shopify/sarama/config.go delete mode 100644 vendor/github.com/Shopify/sarama/config_resource_type.go delete mode 100644 vendor/github.com/Shopify/sarama/consumer.go delete mode 100644 vendor/github.com/Shopify/sarama/consumer_group.go delete mode 100644 vendor/github.com/Shopify/sarama/consumer_group_members.go delete mode 100644 vendor/github.com/Shopify/sarama/consumer_metadata_request.go delete mode 100644 vendor/github.com/Shopify/sarama/consumer_metadata_response.go delete mode 100644 vendor/github.com/Shopify/sarama/control_record.go delete mode 100644 vendor/github.com/Shopify/sarama/crc32_field.go delete mode 100644 vendor/github.com/Shopify/sarama/create_partitions_request.go delete mode 100644 vendor/github.com/Shopify/sarama/create_partitions_response.go delete mode 100644 vendor/github.com/Shopify/sarama/create_topics_request.go delete mode 100644 vendor/github.com/Shopify/sarama/create_topics_response.go delete mode 100644 vendor/github.com/Shopify/sarama/decompress.go delete mode 100644 vendor/github.com/Shopify/sarama/delete_groups_request.go delete mode 100644 vendor/github.com/Shopify/sarama/delete_groups_response.go delete mode 100644 vendor/github.com/Shopify/sarama/delete_records_request.go delete mode 100644 vendor/github.com/Shopify/sarama/delete_records_response.go delete mode 100644 vendor/github.com/Shopify/sarama/delete_topics_request.go delete mode 100644 vendor/github.com/Shopify/sarama/delete_topics_response.go delete mode 100644 vendor/github.com/Shopify/sarama/describe_configs_request.go delete mode 100644 vendor/github.com/Shopify/sarama/describe_configs_response.go delete mode 100644 vendor/github.com/Shopify/sarama/describe_groups_request.go delete mode 100644 vendor/github.com/Shopify/sarama/describe_groups_response.go delete mode 100644 vendor/github.com/Shopify/sarama/describe_log_dirs_request.go delete mode 100644 vendor/github.com/Shopify/sarama/describe_log_dirs_response.go delete mode 100644 vendor/github.com/Shopify/sarama/dev.yml delete mode 100644 vendor/github.com/Shopify/sarama/docker-compose.yml delete mode 100644 vendor/github.com/Shopify/sarama/encoder_decoder.go delete mode 100644 vendor/github.com/Shopify/sarama/end_txn_request.go delete mode 100644 vendor/github.com/Shopify/sarama/end_txn_response.go delete mode 100644 vendor/github.com/Shopify/sarama/errors.go delete mode 100644 vendor/github.com/Shopify/sarama/fetch_request.go delete mode 100644 vendor/github.com/Shopify/sarama/fetch_response.go delete mode 100644 vendor/github.com/Shopify/sarama/find_coordinator_request.go delete mode 100644 vendor/github.com/Shopify/sarama/find_coordinator_response.go delete mode 100644 vendor/github.com/Shopify/sarama/go.mod delete mode 100644 vendor/github.com/Shopify/sarama/go.sum delete mode 100644 vendor/github.com/Shopify/sarama/gssapi_kerberos.go delete mode 100644 vendor/github.com/Shopify/sarama/heartbeat_request.go delete mode 100644 vendor/github.com/Shopify/sarama/heartbeat_response.go delete mode 100644 vendor/github.com/Shopify/sarama/init_producer_id_request.go delete mode 100644 vendor/github.com/Shopify/sarama/init_producer_id_response.go delete mode 100644 vendor/github.com/Shopify/sarama/interceptors.go delete mode 100644 vendor/github.com/Shopify/sarama/join_group_request.go delete mode 100644 vendor/github.com/Shopify/sarama/join_group_response.go delete mode 100644 vendor/github.com/Shopify/sarama/kerberos_client.go delete mode 100644 vendor/github.com/Shopify/sarama/leave_group_request.go delete mode 100644 vendor/github.com/Shopify/sarama/leave_group_response.go delete mode 100644 vendor/github.com/Shopify/sarama/length_field.go delete mode 100644 vendor/github.com/Shopify/sarama/list_groups_request.go delete mode 100644 vendor/github.com/Shopify/sarama/list_groups_response.go delete mode 100644 vendor/github.com/Shopify/sarama/list_partition_reassignments_request.go delete mode 100644 vendor/github.com/Shopify/sarama/list_partition_reassignments_response.go delete mode 100644 vendor/github.com/Shopify/sarama/message.go delete mode 100644 vendor/github.com/Shopify/sarama/message_set.go delete mode 100644 vendor/github.com/Shopify/sarama/metadata_request.go delete mode 100644 vendor/github.com/Shopify/sarama/metadata_response.go delete mode 100644 vendor/github.com/Shopify/sarama/metrics.go delete mode 100644 vendor/github.com/Shopify/sarama/mockbroker.go delete mode 100644 vendor/github.com/Shopify/sarama/mockkerberos.go delete mode 100644 vendor/github.com/Shopify/sarama/mockresponses.go delete mode 100644 vendor/github.com/Shopify/sarama/offset_commit_request.go delete mode 100644 vendor/github.com/Shopify/sarama/offset_commit_response.go delete mode 100644 vendor/github.com/Shopify/sarama/offset_fetch_request.go delete mode 100644 vendor/github.com/Shopify/sarama/offset_fetch_response.go delete mode 100644 vendor/github.com/Shopify/sarama/offset_manager.go delete mode 100644 vendor/github.com/Shopify/sarama/offset_request.go delete mode 100644 vendor/github.com/Shopify/sarama/offset_response.go delete mode 100644 vendor/github.com/Shopify/sarama/packet_decoder.go delete mode 100644 vendor/github.com/Shopify/sarama/packet_encoder.go delete mode 100644 vendor/github.com/Shopify/sarama/partitioner.go delete mode 100644 vendor/github.com/Shopify/sarama/prep_encoder.go delete mode 100644 vendor/github.com/Shopify/sarama/produce_request.go delete mode 100644 vendor/github.com/Shopify/sarama/produce_response.go delete mode 100644 vendor/github.com/Shopify/sarama/produce_set.go delete mode 100644 vendor/github.com/Shopify/sarama/real_decoder.go delete mode 100644 vendor/github.com/Shopify/sarama/real_encoder.go delete mode 100644 vendor/github.com/Shopify/sarama/record.go delete mode 100644 vendor/github.com/Shopify/sarama/record_batch.go delete mode 100644 vendor/github.com/Shopify/sarama/records.go delete mode 100644 vendor/github.com/Shopify/sarama/request.go delete mode 100644 vendor/github.com/Shopify/sarama/response_header.go delete mode 100644 vendor/github.com/Shopify/sarama/sarama.go delete mode 100644 vendor/github.com/Shopify/sarama/sasl_authenticate_request.go delete mode 100644 vendor/github.com/Shopify/sarama/sasl_authenticate_response.go delete mode 100644 vendor/github.com/Shopify/sarama/sasl_handshake_request.go delete mode 100644 vendor/github.com/Shopify/sarama/sasl_handshake_response.go delete mode 100644 vendor/github.com/Shopify/sarama/sticky_assignor_user_data.go delete mode 100644 vendor/github.com/Shopify/sarama/sync_group_request.go delete mode 100644 vendor/github.com/Shopify/sarama/sync_group_response.go delete mode 100644 vendor/github.com/Shopify/sarama/sync_producer.go delete mode 100644 vendor/github.com/Shopify/sarama/timestamp.go delete mode 100644 vendor/github.com/Shopify/sarama/txn_offset_commit_request.go delete mode 100644 vendor/github.com/Shopify/sarama/txn_offset_commit_response.go delete mode 100644 vendor/github.com/Shopify/sarama/utils.go delete mode 100644 vendor/github.com/Shopify/sarama/zstd.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/protocol/kafka_sarama/v2/LICENSE delete mode 100644 vendor/github.com/cloudevents/sdk-go/protocol/kafka_sarama/v2/doc.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/protocol/kafka_sarama/v2/go.mod delete mode 100644 vendor/github.com/cloudevents/sdk-go/protocol/kafka_sarama/v2/go.sum delete mode 100644 vendor/github.com/cloudevents/sdk-go/protocol/kafka_sarama/v2/message.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/protocol/kafka_sarama/v2/option.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/protocol/kafka_sarama/v2/protocol.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/protocol/kafka_sarama/v2/receiver.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/protocol/kafka_sarama/v2/sender.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/protocol/kafka_sarama/v2/write_producer_message.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/v2/LICENSE delete mode 100644 vendor/github.com/cloudevents/sdk-go/v2/alias.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/v2/binding/binary_writer.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/v2/binding/doc.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/v2/binding/encoding.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/v2/binding/event_message.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/v2/binding/finish_message.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/v2/binding/format/doc.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/v2/binding/format/format.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/v2/binding/message.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/v2/binding/spec/attributes.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/v2/binding/spec/doc.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/v2/binding/spec/match_exact_version.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/v2/binding/spec/spec.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/v2/binding/structured_writer.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/v2/binding/to_event.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/v2/binding/transformer.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/v2/binding/write.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/v2/client/client.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/v2/client/client_default.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/v2/client/client_observed.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/v2/client/defaulters.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/v2/client/doc.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/v2/client/http_receiver.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/v2/client/invoker.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/v2/client/observability.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/v2/client/options.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/v2/client/receiver.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/v2/context/context.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/v2/context/doc.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/v2/context/logger.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/v2/context/retry.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/v2/event/content_type.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/v2/event/data_content_encoding.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/v2/event/datacodec/codec.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/v2/event/datacodec/codec_observed.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/v2/event/datacodec/doc.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/v2/event/datacodec/json/data.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/v2/event/datacodec/json/data_observed.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/v2/event/datacodec/json/doc.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/v2/event/datacodec/json/observability.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/v2/event/datacodec/observability.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/v2/event/datacodec/text/data.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/v2/event/datacodec/text/data_observed.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/v2/event/datacodec/text/doc.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/v2/event/datacodec/text/observability.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/v2/event/datacodec/xml/data.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/v2/event/datacodec/xml/data_observed.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/v2/event/datacodec/xml/doc.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/v2/event/datacodec/xml/observability.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/v2/event/doc.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/v2/event/event.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/v2/event/event_data.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/v2/event/event_interface.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/v2/event/event_marshal.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/v2/event/event_observability.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/v2/event/event_reader.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/v2/event/event_validation.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/v2/event/event_writer.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/v2/event/eventcontext.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/v2/event/eventcontext_v03.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/v2/event/eventcontext_v03_reader.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/v2/event/eventcontext_v03_writer.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/v2/event/eventcontext_v1.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/v2/event/eventcontext_v1_reader.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/v2/event/eventcontext_v1_writer.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/v2/event/extensions.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/v2/extensions/distributed_tracing_extension.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/v2/extensions/doc.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/v2/go.mod delete mode 100644 vendor/github.com/cloudevents/sdk-go/v2/go.sum delete mode 100644 vendor/github.com/cloudevents/sdk-go/v2/observability/doc.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/v2/observability/keys.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/v2/observability/observer.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/v2/protocol/doc.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/v2/protocol/error.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/v2/protocol/http/abuse_protection.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/v2/protocol/http/doc.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/v2/protocol/http/headers.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/v2/protocol/http/message.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/v2/protocol/http/options.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/v2/protocol/http/protocol.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/v2/protocol/http/protocol_lifecycle.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/v2/protocol/http/protocol_retry.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/v2/protocol/http/result.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/v2/protocol/http/retries_result.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/v2/protocol/http/write_request.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/v2/protocol/http/write_responsewriter.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/v2/protocol/inbound.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/v2/protocol/lifecycle.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/v2/protocol/outbound.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/v2/protocol/result.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/v2/staticcheck.conf delete mode 100644 vendor/github.com/cloudevents/sdk-go/v2/types/allocate.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/v2/types/doc.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/v2/types/timestamp.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/v2/types/uri.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/v2/types/uriref.go delete mode 100644 vendor/github.com/cloudevents/sdk-go/v2/types/value.go delete mode 100644 vendor/github.com/davecgh/go-spew/LICENSE delete mode 100644 vendor/github.com/davecgh/go-spew/spew/bypass.go delete mode 100644 vendor/github.com/davecgh/go-spew/spew/bypasssafe.go delete mode 100644 vendor/github.com/davecgh/go-spew/spew/common.go delete mode 100644 vendor/github.com/davecgh/go-spew/spew/config.go delete mode 100644 vendor/github.com/davecgh/go-spew/spew/doc.go delete mode 100644 vendor/github.com/davecgh/go-spew/spew/dump.go delete mode 100644 vendor/github.com/davecgh/go-spew/spew/format.go delete mode 100644 vendor/github.com/davecgh/go-spew/spew/spew.go delete mode 100644 vendor/github.com/eapache/go-resiliency/LICENSE delete mode 100644 vendor/github.com/eapache/go-resiliency/breaker/README.md delete mode 100644 vendor/github.com/eapache/go-resiliency/breaker/breaker.go delete mode 100644 vendor/github.com/eapache/go-xerial-snappy/.gitignore delete mode 100644 vendor/github.com/eapache/go-xerial-snappy/.travis.yml delete mode 100644 vendor/github.com/eapache/go-xerial-snappy/LICENSE delete mode 100644 vendor/github.com/eapache/go-xerial-snappy/README.md delete mode 100644 vendor/github.com/eapache/go-xerial-snappy/fuzz.go delete mode 100644 vendor/github.com/eapache/go-xerial-snappy/snappy.go delete mode 100644 vendor/github.com/eapache/queue/.gitignore delete mode 100644 vendor/github.com/eapache/queue/.travis.yml delete mode 100644 vendor/github.com/eapache/queue/LICENSE delete mode 100644 vendor/github.com/eapache/queue/README.md delete mode 100644 vendor/github.com/eapache/queue/queue.go delete mode 100644 vendor/github.com/globalsign/mgo/.gitignore delete mode 100644 vendor/github.com/globalsign/mgo/.travis.yml delete mode 100644 vendor/github.com/globalsign/mgo/CONTRIBUTING.md delete mode 100644 vendor/github.com/globalsign/mgo/LICENSE delete mode 100644 vendor/github.com/globalsign/mgo/Makefile delete mode 100644 vendor/github.com/globalsign/mgo/README.md delete mode 100644 vendor/github.com/globalsign/mgo/auth.go delete mode 100644 vendor/github.com/globalsign/mgo/bson/LICENSE delete mode 100644 vendor/github.com/globalsign/mgo/bson/README.md delete mode 100644 vendor/github.com/globalsign/mgo/bson/bson.go delete mode 100644 vendor/github.com/globalsign/mgo/bson/compatibility.go delete mode 100644 vendor/github.com/globalsign/mgo/bson/decimal.go delete mode 100644 vendor/github.com/globalsign/mgo/bson/decode.go delete mode 100644 vendor/github.com/globalsign/mgo/bson/encode.go delete mode 100644 vendor/github.com/globalsign/mgo/bson/json.go delete mode 100644 vendor/github.com/globalsign/mgo/bson/stream.go delete mode 100644 vendor/github.com/globalsign/mgo/bulk.go delete mode 100644 vendor/github.com/globalsign/mgo/changestreams.go delete mode 100644 vendor/github.com/globalsign/mgo/cluster.go delete mode 100644 vendor/github.com/globalsign/mgo/coarse_time.go delete mode 100644 vendor/github.com/globalsign/mgo/doc.go delete mode 100644 vendor/github.com/globalsign/mgo/gridfs.go delete mode 100644 vendor/github.com/globalsign/mgo/internal/json/LICENSE delete mode 100644 vendor/github.com/globalsign/mgo/internal/json/decode.go delete mode 100644 vendor/github.com/globalsign/mgo/internal/json/encode.go delete mode 100644 vendor/github.com/globalsign/mgo/internal/json/extension.go delete mode 100644 vendor/github.com/globalsign/mgo/internal/json/fold.go delete mode 100644 vendor/github.com/globalsign/mgo/internal/json/indent.go delete mode 100644 vendor/github.com/globalsign/mgo/internal/json/scanner.go delete mode 100644 vendor/github.com/globalsign/mgo/internal/json/stream.go delete mode 100644 vendor/github.com/globalsign/mgo/internal/json/tags.go delete mode 100644 vendor/github.com/globalsign/mgo/internal/sasl/sasl.c delete mode 100644 vendor/github.com/globalsign/mgo/internal/sasl/sasl.go delete mode 100644 vendor/github.com/globalsign/mgo/internal/sasl/sasl_windows.c delete mode 100644 vendor/github.com/globalsign/mgo/internal/sasl/sasl_windows.go delete mode 100644 vendor/github.com/globalsign/mgo/internal/sasl/sasl_windows.h delete mode 100644 vendor/github.com/globalsign/mgo/internal/sasl/sspi_windows.c delete mode 100644 vendor/github.com/globalsign/mgo/internal/sasl/sspi_windows.h delete mode 100644 vendor/github.com/globalsign/mgo/internal/scram/scram.go delete mode 100644 vendor/github.com/globalsign/mgo/log.go delete mode 100644 vendor/github.com/globalsign/mgo/queue.go delete mode 100644 vendor/github.com/globalsign/mgo/raceoff.go delete mode 100644 vendor/github.com/globalsign/mgo/raceon.go delete mode 100644 vendor/github.com/globalsign/mgo/saslimpl.go delete mode 100644 vendor/github.com/globalsign/mgo/saslstub.go delete mode 100644 vendor/github.com/globalsign/mgo/server.go delete mode 100644 vendor/github.com/globalsign/mgo/session.go delete mode 100644 vendor/github.com/globalsign/mgo/socket.go delete mode 100644 vendor/github.com/globalsign/mgo/stats.go delete mode 100644 vendor/github.com/golang/snappy/.gitignore delete mode 100644 vendor/github.com/golang/snappy/AUTHORS delete mode 100644 vendor/github.com/golang/snappy/CONTRIBUTORS delete mode 100644 vendor/github.com/golang/snappy/LICENSE delete mode 100644 vendor/github.com/golang/snappy/README delete mode 100644 vendor/github.com/golang/snappy/decode.go delete mode 100644 vendor/github.com/golang/snappy/decode_amd64.go delete mode 100644 vendor/github.com/golang/snappy/decode_amd64.s delete mode 100644 vendor/github.com/golang/snappy/decode_other.go delete mode 100644 vendor/github.com/golang/snappy/encode.go delete mode 100644 vendor/github.com/golang/snappy/encode_amd64.go delete mode 100644 vendor/github.com/golang/snappy/encode_amd64.s delete mode 100644 vendor/github.com/golang/snappy/encode_other.go delete mode 100644 vendor/github.com/golang/snappy/go.mod delete mode 100644 vendor/github.com/golang/snappy/snappy.go delete mode 100644 vendor/github.com/google/uuid/.travis.yml delete mode 100644 vendor/github.com/google/uuid/CONTRIBUTING.md delete mode 100644 vendor/github.com/google/uuid/CONTRIBUTORS delete mode 100644 vendor/github.com/google/uuid/LICENSE delete mode 100644 vendor/github.com/google/uuid/README.md delete mode 100644 vendor/github.com/google/uuid/dce.go delete mode 100644 vendor/github.com/google/uuid/doc.go delete mode 100644 vendor/github.com/google/uuid/go.mod delete mode 100644 vendor/github.com/google/uuid/hash.go delete mode 100644 vendor/github.com/google/uuid/marshal.go delete mode 100644 vendor/github.com/google/uuid/node.go delete mode 100644 vendor/github.com/google/uuid/node_js.go delete mode 100644 vendor/github.com/google/uuid/node_net.go delete mode 100644 vendor/github.com/google/uuid/sql.go delete mode 100644 vendor/github.com/google/uuid/time.go delete mode 100644 vendor/github.com/google/uuid/util.go delete mode 100644 vendor/github.com/google/uuid/uuid.go delete mode 100644 vendor/github.com/google/uuid/version1.go delete mode 100644 vendor/github.com/google/uuid/version4.go delete mode 100644 vendor/github.com/hashicorp/go-uuid/.travis.yml delete mode 100644 vendor/github.com/hashicorp/go-uuid/LICENSE delete mode 100644 vendor/github.com/hashicorp/go-uuid/README.md delete mode 100644 vendor/github.com/hashicorp/go-uuid/go.mod delete mode 100644 vendor/github.com/hashicorp/go-uuid/uuid.go delete mode 100644 vendor/github.com/hashicorp/golang-lru/LICENSE delete mode 100644 vendor/github.com/hashicorp/golang-lru/simplelru/lru.go delete mode 100644 vendor/github.com/hashicorp/golang-lru/simplelru/lru_interface.go delete mode 100644 vendor/github.com/jcmturner/gofork/LICENSE delete mode 100644 vendor/github.com/jcmturner/gofork/encoding/asn1/README.md delete mode 100644 vendor/github.com/jcmturner/gofork/encoding/asn1/asn1.go delete mode 100644 vendor/github.com/jcmturner/gofork/encoding/asn1/common.go delete mode 100644 vendor/github.com/jcmturner/gofork/encoding/asn1/marshal.go delete mode 100644 vendor/github.com/jcmturner/gofork/x/crypto/pbkdf2/pbkdf2.go delete mode 100644 vendor/github.com/kelseyhightower/envconfig/.travis.yml delete mode 100644 vendor/github.com/kelseyhightower/envconfig/LICENSE delete mode 100644 vendor/github.com/kelseyhightower/envconfig/MAINTAINERS delete mode 100644 vendor/github.com/kelseyhightower/envconfig/README.md delete mode 100644 vendor/github.com/kelseyhightower/envconfig/doc.go delete mode 100644 vendor/github.com/kelseyhightower/envconfig/env_os.go delete mode 100644 vendor/github.com/kelseyhightower/envconfig/env_syscall.go delete mode 100644 vendor/github.com/kelseyhightower/envconfig/envconfig.go delete mode 100644 vendor/github.com/kelseyhightower/envconfig/go.mod delete mode 100644 vendor/github.com/kelseyhightower/envconfig/usage.go delete mode 100644 vendor/github.com/klauspost/compress/LICENSE delete mode 100644 vendor/github.com/klauspost/compress/fse/README.md delete mode 100644 vendor/github.com/klauspost/compress/fse/bitreader.go delete mode 100644 vendor/github.com/klauspost/compress/fse/bitwriter.go delete mode 100644 vendor/github.com/klauspost/compress/fse/bytereader.go delete mode 100644 vendor/github.com/klauspost/compress/fse/compress.go delete mode 100644 vendor/github.com/klauspost/compress/fse/decompress.go delete mode 100644 vendor/github.com/klauspost/compress/fse/fse.go delete mode 100644 vendor/github.com/klauspost/compress/huff0/.gitignore delete mode 100644 vendor/github.com/klauspost/compress/huff0/README.md delete mode 100644 vendor/github.com/klauspost/compress/huff0/bitreader.go delete mode 100644 vendor/github.com/klauspost/compress/huff0/bitwriter.go delete mode 100644 vendor/github.com/klauspost/compress/huff0/bytereader.go delete mode 100644 vendor/github.com/klauspost/compress/huff0/compress.go delete mode 100644 vendor/github.com/klauspost/compress/huff0/decompress.go delete mode 100644 vendor/github.com/klauspost/compress/huff0/huff0.go delete mode 100644 vendor/github.com/klauspost/compress/snappy/.gitignore delete mode 100644 vendor/github.com/klauspost/compress/snappy/AUTHORS delete mode 100644 vendor/github.com/klauspost/compress/snappy/CONTRIBUTORS delete mode 100644 vendor/github.com/klauspost/compress/snappy/LICENSE delete mode 100644 vendor/github.com/klauspost/compress/snappy/README delete mode 100644 vendor/github.com/klauspost/compress/snappy/decode.go delete mode 100644 vendor/github.com/klauspost/compress/snappy/decode_amd64.go delete mode 100644 vendor/github.com/klauspost/compress/snappy/decode_amd64.s delete mode 100644 vendor/github.com/klauspost/compress/snappy/decode_other.go delete mode 100644 vendor/github.com/klauspost/compress/snappy/encode.go delete mode 100644 vendor/github.com/klauspost/compress/snappy/encode_amd64.go delete mode 100644 vendor/github.com/klauspost/compress/snappy/encode_amd64.s delete mode 100644 vendor/github.com/klauspost/compress/snappy/encode_other.go delete mode 100644 vendor/github.com/klauspost/compress/snappy/runbench.cmd delete mode 100644 vendor/github.com/klauspost/compress/snappy/snappy.go delete mode 100644 vendor/github.com/klauspost/compress/zstd/README.md delete mode 100644 vendor/github.com/klauspost/compress/zstd/bitreader.go delete mode 100644 vendor/github.com/klauspost/compress/zstd/bitwriter.go delete mode 100644 vendor/github.com/klauspost/compress/zstd/blockdec.go delete mode 100644 vendor/github.com/klauspost/compress/zstd/blockenc.go delete mode 100644 vendor/github.com/klauspost/compress/zstd/blocktype_string.go delete mode 100644 vendor/github.com/klauspost/compress/zstd/bytebuf.go delete mode 100644 vendor/github.com/klauspost/compress/zstd/bytereader.go delete mode 100644 vendor/github.com/klauspost/compress/zstd/decoder.go delete mode 100644 vendor/github.com/klauspost/compress/zstd/decoder_options.go delete mode 100644 vendor/github.com/klauspost/compress/zstd/dict.go delete mode 100644 vendor/github.com/klauspost/compress/zstd/enc_better.go delete mode 100644 vendor/github.com/klauspost/compress/zstd/enc_dfast.go delete mode 100644 vendor/github.com/klauspost/compress/zstd/enc_fast.go delete mode 100644 vendor/github.com/klauspost/compress/zstd/enc_params.go delete mode 100644 vendor/github.com/klauspost/compress/zstd/encoder.go delete mode 100644 vendor/github.com/klauspost/compress/zstd/encoder_options.go delete mode 100644 vendor/github.com/klauspost/compress/zstd/framedec.go delete mode 100644 vendor/github.com/klauspost/compress/zstd/frameenc.go delete mode 100644 vendor/github.com/klauspost/compress/zstd/fse_decoder.go delete mode 100644 vendor/github.com/klauspost/compress/zstd/fse_encoder.go delete mode 100644 vendor/github.com/klauspost/compress/zstd/fse_predefined.go delete mode 100644 vendor/github.com/klauspost/compress/zstd/hash.go delete mode 100644 vendor/github.com/klauspost/compress/zstd/history.go delete mode 100644 vendor/github.com/klauspost/compress/zstd/internal/xxhash/LICENSE.txt delete mode 100644 vendor/github.com/klauspost/compress/zstd/internal/xxhash/README.md delete mode 100644 vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash.go delete mode 100644 vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_amd64.go delete mode 100644 vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_amd64.s delete mode 100644 vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_other.go delete mode 100644 vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_safe.go delete mode 100644 vendor/github.com/klauspost/compress/zstd/seqdec.go delete mode 100644 vendor/github.com/klauspost/compress/zstd/seqenc.go delete mode 100644 vendor/github.com/klauspost/compress/zstd/snappy.go delete mode 100644 vendor/github.com/klauspost/compress/zstd/zstd.go delete mode 100644 vendor/github.com/lightstep/tracecontext.go/LICENSE delete mode 100644 vendor/github.com/lightstep/tracecontext.go/traceparent/package.go delete mode 100644 vendor/github.com/lightstep/tracecontext.go/tracestate/package.go delete mode 100644 vendor/github.com/pierrec/lz4/.gitignore delete mode 100644 vendor/github.com/pierrec/lz4/.travis.yml delete mode 100644 vendor/github.com/pierrec/lz4/LICENSE delete mode 100644 vendor/github.com/pierrec/lz4/README.md delete mode 100644 vendor/github.com/pierrec/lz4/block.go delete mode 100644 vendor/github.com/pierrec/lz4/debug.go delete mode 100644 vendor/github.com/pierrec/lz4/debug_stub.go delete mode 100644 vendor/github.com/pierrec/lz4/decode_amd64.go delete mode 100644 vendor/github.com/pierrec/lz4/decode_amd64.s delete mode 100644 vendor/github.com/pierrec/lz4/decode_other.go delete mode 100644 vendor/github.com/pierrec/lz4/errors.go delete mode 100644 vendor/github.com/pierrec/lz4/internal/xxh32/xxh32zero.go delete mode 100644 vendor/github.com/pierrec/lz4/lz4.go delete mode 100644 vendor/github.com/pierrec/lz4/lz4_go1.10.go delete mode 100644 vendor/github.com/pierrec/lz4/lz4_notgo1.10.go delete mode 100644 vendor/github.com/pierrec/lz4/reader.go delete mode 100644 vendor/github.com/pierrec/lz4/writer.go delete mode 100644 vendor/github.com/rcrowley/go-metrics/.gitignore delete mode 100644 vendor/github.com/rcrowley/go-metrics/.travis.yml delete mode 100644 vendor/github.com/rcrowley/go-metrics/LICENSE delete mode 100644 vendor/github.com/rcrowley/go-metrics/README.md delete mode 100644 vendor/github.com/rcrowley/go-metrics/counter.go delete mode 100644 vendor/github.com/rcrowley/go-metrics/debug.go delete mode 100644 vendor/github.com/rcrowley/go-metrics/ewma.go delete mode 100644 vendor/github.com/rcrowley/go-metrics/gauge.go delete mode 100644 vendor/github.com/rcrowley/go-metrics/gauge_float64.go delete mode 100644 vendor/github.com/rcrowley/go-metrics/graphite.go delete mode 100644 vendor/github.com/rcrowley/go-metrics/healthcheck.go delete mode 100644 vendor/github.com/rcrowley/go-metrics/histogram.go delete mode 100644 vendor/github.com/rcrowley/go-metrics/json.go delete mode 100644 vendor/github.com/rcrowley/go-metrics/log.go delete mode 100644 vendor/github.com/rcrowley/go-metrics/memory.md delete mode 100644 vendor/github.com/rcrowley/go-metrics/meter.go delete mode 100644 vendor/github.com/rcrowley/go-metrics/metrics.go delete mode 100644 vendor/github.com/rcrowley/go-metrics/opentsdb.go delete mode 100644 vendor/github.com/rcrowley/go-metrics/registry.go delete mode 100644 vendor/github.com/rcrowley/go-metrics/runtime.go delete mode 100644 vendor/github.com/rcrowley/go-metrics/runtime_cgo.go delete mode 100644 vendor/github.com/rcrowley/go-metrics/runtime_gccpufraction.go delete mode 100644 vendor/github.com/rcrowley/go-metrics/runtime_no_cgo.go delete mode 100644 vendor/github.com/rcrowley/go-metrics/runtime_no_gccpufraction.go delete mode 100644 vendor/github.com/rcrowley/go-metrics/sample.go delete mode 100644 vendor/github.com/rcrowley/go-metrics/syslog.go delete mode 100644 vendor/github.com/rcrowley/go-metrics/timer.go delete mode 100644 vendor/github.com/rcrowley/go-metrics/validate.sh delete mode 100644 vendor/github.com/rcrowley/go-metrics/writer.go delete mode 100644 vendor/github.com/xdg/scram/.gitignore delete mode 100644 vendor/github.com/xdg/scram/.travis.yml delete mode 100644 vendor/github.com/xdg/scram/LICENSE delete mode 100644 vendor/github.com/xdg/scram/README.md delete mode 100644 vendor/github.com/xdg/scram/client.go delete mode 100644 vendor/github.com/xdg/scram/client_conv.go delete mode 100644 vendor/github.com/xdg/scram/common.go delete mode 100644 vendor/github.com/xdg/scram/doc.go delete mode 100644 vendor/github.com/xdg/scram/parse.go delete mode 100644 vendor/github.com/xdg/scram/scram.go delete mode 100644 vendor/github.com/xdg/scram/server.go delete mode 100644 vendor/github.com/xdg/scram/server_conv.go delete mode 100644 vendor/github.com/xdg/stringprep/.gitignore delete mode 100644 vendor/github.com/xdg/stringprep/.travis.yml delete mode 100644 vendor/github.com/xdg/stringprep/LICENSE delete mode 100644 vendor/github.com/xdg/stringprep/README.md delete mode 100644 vendor/github.com/xdg/stringprep/bidi.go delete mode 100644 vendor/github.com/xdg/stringprep/doc.go delete mode 100644 vendor/github.com/xdg/stringprep/error.go delete mode 100644 vendor/github.com/xdg/stringprep/map.go delete mode 100644 vendor/github.com/xdg/stringprep/profile.go delete mode 100644 vendor/github.com/xdg/stringprep/saslprep.go delete mode 100644 vendor/github.com/xdg/stringprep/set.go delete mode 100644 vendor/github.com/xdg/stringprep/tables.go delete mode 100644 vendor/go.opencensus.io/.gitignore delete mode 100644 vendor/go.opencensus.io/.travis.yml delete mode 100644 vendor/go.opencensus.io/AUTHORS delete mode 100644 vendor/go.opencensus.io/CONTRIBUTING.md delete mode 100644 vendor/go.opencensus.io/Gopkg.lock delete mode 100644 vendor/go.opencensus.io/Gopkg.toml delete mode 100644 vendor/go.opencensus.io/LICENSE delete mode 100644 vendor/go.opencensus.io/Makefile delete mode 100644 vendor/go.opencensus.io/README.md delete mode 100644 vendor/go.opencensus.io/appveyor.yml delete mode 100644 vendor/go.opencensus.io/go.mod delete mode 100644 vendor/go.opencensus.io/go.sum delete mode 100644 vendor/go.opencensus.io/internal/internal.go delete mode 100644 vendor/go.opencensus.io/internal/sanitize.go delete mode 100644 vendor/go.opencensus.io/internal/tagencoding/tagencoding.go delete mode 100644 vendor/go.opencensus.io/internal/traceinternals.go delete mode 100644 vendor/go.opencensus.io/metric/metricdata/doc.go delete mode 100644 vendor/go.opencensus.io/metric/metricdata/exemplar.go delete mode 100644 vendor/go.opencensus.io/metric/metricdata/label.go delete mode 100644 vendor/go.opencensus.io/metric/metricdata/metric.go delete mode 100644 vendor/go.opencensus.io/metric/metricdata/point.go delete mode 100644 vendor/go.opencensus.io/metric/metricdata/type_string.go delete mode 100644 vendor/go.opencensus.io/metric/metricdata/unit.go delete mode 100644 vendor/go.opencensus.io/metric/metricproducer/manager.go delete mode 100644 vendor/go.opencensus.io/metric/metricproducer/producer.go delete mode 100644 vendor/go.opencensus.io/opencensus.go delete mode 100644 vendor/go.opencensus.io/plugin/ochttp/client.go delete mode 100644 vendor/go.opencensus.io/plugin/ochttp/client_stats.go delete mode 100644 vendor/go.opencensus.io/plugin/ochttp/doc.go delete mode 100644 vendor/go.opencensus.io/plugin/ochttp/propagation/b3/b3.go delete mode 100644 vendor/go.opencensus.io/plugin/ochttp/propagation/tracecontext/propagation.go delete mode 100644 vendor/go.opencensus.io/plugin/ochttp/route.go delete mode 100644 vendor/go.opencensus.io/plugin/ochttp/server.go delete mode 100644 vendor/go.opencensus.io/plugin/ochttp/span_annotating_client_trace.go delete mode 100644 vendor/go.opencensus.io/plugin/ochttp/stats.go delete mode 100644 vendor/go.opencensus.io/plugin/ochttp/trace.go delete mode 100644 vendor/go.opencensus.io/plugin/ochttp/wrapped_body.go delete mode 100644 vendor/go.opencensus.io/resource/resource.go delete mode 100644 vendor/go.opencensus.io/stats/doc.go delete mode 100644 vendor/go.opencensus.io/stats/internal/record.go delete mode 100644 vendor/go.opencensus.io/stats/measure.go delete mode 100644 vendor/go.opencensus.io/stats/measure_float64.go delete mode 100644 vendor/go.opencensus.io/stats/measure_int64.go delete mode 100644 vendor/go.opencensus.io/stats/record.go delete mode 100644 vendor/go.opencensus.io/stats/units.go delete mode 100644 vendor/go.opencensus.io/stats/view/aggregation.go delete mode 100644 vendor/go.opencensus.io/stats/view/aggregation_data.go delete mode 100644 vendor/go.opencensus.io/stats/view/collector.go delete mode 100644 vendor/go.opencensus.io/stats/view/doc.go delete mode 100644 vendor/go.opencensus.io/stats/view/export.go delete mode 100644 vendor/go.opencensus.io/stats/view/view.go delete mode 100644 vendor/go.opencensus.io/stats/view/view_to_metric.go delete mode 100644 vendor/go.opencensus.io/stats/view/worker.go delete mode 100644 vendor/go.opencensus.io/stats/view/worker_commands.go delete mode 100644 vendor/go.opencensus.io/tag/context.go delete mode 100644 vendor/go.opencensus.io/tag/doc.go delete mode 100644 vendor/go.opencensus.io/tag/key.go delete mode 100644 vendor/go.opencensus.io/tag/map.go delete mode 100644 vendor/go.opencensus.io/tag/map_codec.go delete mode 100644 vendor/go.opencensus.io/tag/metadata.go delete mode 100644 vendor/go.opencensus.io/tag/profile_19.go delete mode 100644 vendor/go.opencensus.io/tag/profile_not19.go delete mode 100644 vendor/go.opencensus.io/tag/validate.go delete mode 100644 vendor/go.opencensus.io/trace/basetypes.go delete mode 100644 vendor/go.opencensus.io/trace/config.go delete mode 100644 vendor/go.opencensus.io/trace/doc.go delete mode 100644 vendor/go.opencensus.io/trace/evictedqueue.go delete mode 100644 vendor/go.opencensus.io/trace/export.go delete mode 100644 vendor/go.opencensus.io/trace/internal/internal.go delete mode 100644 vendor/go.opencensus.io/trace/lrumap.go delete mode 100644 vendor/go.opencensus.io/trace/propagation/propagation.go delete mode 100644 vendor/go.opencensus.io/trace/sampling.go delete mode 100644 vendor/go.opencensus.io/trace/spanbucket.go delete mode 100644 vendor/go.opencensus.io/trace/spanstore.go delete mode 100644 vendor/go.opencensus.io/trace/status_codes.go delete mode 100644 vendor/go.opencensus.io/trace/trace.go delete mode 100644 vendor/go.opencensus.io/trace/trace_go11.go delete mode 100644 vendor/go.opencensus.io/trace/trace_nongo11.go delete mode 100644 vendor/go.opencensus.io/trace/tracestate/tracestate.go delete mode 100644 vendor/go.uber.org/atomic/.codecov.yml delete mode 100644 vendor/go.uber.org/atomic/.gitignore delete mode 100644 vendor/go.uber.org/atomic/.travis.yml delete mode 100644 vendor/go.uber.org/atomic/LICENSE.txt delete mode 100644 vendor/go.uber.org/atomic/Makefile delete mode 100644 vendor/go.uber.org/atomic/README.md delete mode 100644 vendor/go.uber.org/atomic/atomic.go delete mode 100644 vendor/go.uber.org/atomic/error.go delete mode 100644 vendor/go.uber.org/atomic/glide.lock delete mode 100644 vendor/go.uber.org/atomic/glide.yaml delete mode 100644 vendor/go.uber.org/atomic/string.go delete mode 100644 vendor/go.uber.org/multierr/.codecov.yml delete mode 100644 vendor/go.uber.org/multierr/.gitignore delete mode 100644 vendor/go.uber.org/multierr/.travis.yml delete mode 100644 vendor/go.uber.org/multierr/CHANGELOG.md delete mode 100644 vendor/go.uber.org/multierr/LICENSE.txt delete mode 100644 vendor/go.uber.org/multierr/Makefile delete mode 100644 vendor/go.uber.org/multierr/README.md delete mode 100644 vendor/go.uber.org/multierr/error.go delete mode 100644 vendor/go.uber.org/multierr/glide.lock delete mode 100644 vendor/go.uber.org/multierr/glide.yaml delete mode 100644 vendor/go.uber.org/zap/.codecov.yml delete mode 100644 vendor/go.uber.org/zap/.gitignore delete mode 100644 vendor/go.uber.org/zap/.readme.tmpl delete mode 100644 vendor/go.uber.org/zap/.travis.yml delete mode 100644 vendor/go.uber.org/zap/CHANGELOG.md delete mode 100644 vendor/go.uber.org/zap/CODE_OF_CONDUCT.md delete mode 100644 vendor/go.uber.org/zap/CONTRIBUTING.md delete mode 100644 vendor/go.uber.org/zap/FAQ.md delete mode 100644 vendor/go.uber.org/zap/LICENSE.txt delete mode 100644 vendor/go.uber.org/zap/Makefile delete mode 100644 vendor/go.uber.org/zap/README.md delete mode 100644 vendor/go.uber.org/zap/array.go delete mode 100644 vendor/go.uber.org/zap/buffer/buffer.go delete mode 100644 vendor/go.uber.org/zap/buffer/pool.go delete mode 100644 vendor/go.uber.org/zap/check_license.sh delete mode 100644 vendor/go.uber.org/zap/config.go delete mode 100644 vendor/go.uber.org/zap/doc.go delete mode 100644 vendor/go.uber.org/zap/encoder.go delete mode 100644 vendor/go.uber.org/zap/error.go delete mode 100644 vendor/go.uber.org/zap/field.go delete mode 100644 vendor/go.uber.org/zap/flag.go delete mode 100644 vendor/go.uber.org/zap/glide.lock delete mode 100644 vendor/go.uber.org/zap/glide.yaml delete mode 100644 vendor/go.uber.org/zap/global.go delete mode 100644 vendor/go.uber.org/zap/global_go112.go delete mode 100644 vendor/go.uber.org/zap/global_prego112.go delete mode 100644 vendor/go.uber.org/zap/http_handler.go delete mode 100644 vendor/go.uber.org/zap/internal/bufferpool/bufferpool.go delete mode 100644 vendor/go.uber.org/zap/internal/color/color.go delete mode 100644 vendor/go.uber.org/zap/internal/exit/exit.go delete mode 100644 vendor/go.uber.org/zap/level.go delete mode 100644 vendor/go.uber.org/zap/logger.go delete mode 100644 vendor/go.uber.org/zap/options.go delete mode 100644 vendor/go.uber.org/zap/sink.go delete mode 100644 vendor/go.uber.org/zap/stacktrace.go delete mode 100644 vendor/go.uber.org/zap/sugar.go delete mode 100644 vendor/go.uber.org/zap/time.go delete mode 100644 vendor/go.uber.org/zap/writer.go delete mode 100644 vendor/go.uber.org/zap/zapcore/console_encoder.go delete mode 100644 vendor/go.uber.org/zap/zapcore/core.go delete mode 100644 vendor/go.uber.org/zap/zapcore/doc.go delete mode 100644 vendor/go.uber.org/zap/zapcore/encoder.go delete mode 100644 vendor/go.uber.org/zap/zapcore/entry.go delete mode 100644 vendor/go.uber.org/zap/zapcore/error.go delete mode 100644 vendor/go.uber.org/zap/zapcore/field.go delete mode 100644 vendor/go.uber.org/zap/zapcore/hook.go delete mode 100644 vendor/go.uber.org/zap/zapcore/json_encoder.go delete mode 100644 vendor/go.uber.org/zap/zapcore/level.go delete mode 100644 vendor/go.uber.org/zap/zapcore/level_strings.go delete mode 100644 vendor/go.uber.org/zap/zapcore/marshaler.go delete mode 100644 vendor/go.uber.org/zap/zapcore/memory_encoder.go delete mode 100644 vendor/go.uber.org/zap/zapcore/sampler.go delete mode 100644 vendor/go.uber.org/zap/zapcore/tee.go delete mode 100644 vendor/go.uber.org/zap/zapcore/write_syncer.go delete mode 100644 vendor/golang.org/x/crypto/AUTHORS delete mode 100644 vendor/golang.org/x/crypto/CONTRIBUTORS delete mode 100644 vendor/golang.org/x/crypto/LICENSE delete mode 100644 vendor/golang.org/x/crypto/PATENTS delete mode 100644 vendor/golang.org/x/crypto/md4/md4.go delete mode 100644 vendor/golang.org/x/crypto/md4/md4block.go delete mode 100644 vendor/golang.org/x/crypto/pbkdf2/pbkdf2.go delete mode 100644 vendor/golang.org/x/net/AUTHORS delete mode 100644 vendor/golang.org/x/net/CONTRIBUTORS delete mode 100644 vendor/golang.org/x/net/LICENSE delete mode 100644 vendor/golang.org/x/net/PATENTS delete mode 100644 vendor/golang.org/x/net/internal/socks/client.go delete mode 100644 vendor/golang.org/x/net/internal/socks/socks.go delete mode 100644 vendor/golang.org/x/net/proxy/dial.go delete mode 100644 vendor/golang.org/x/net/proxy/direct.go delete mode 100644 vendor/golang.org/x/net/proxy/per_host.go delete mode 100644 vendor/golang.org/x/net/proxy/proxy.go delete mode 100644 vendor/golang.org/x/net/proxy/socks5.go delete mode 100644 vendor/golang.org/x/text/AUTHORS delete mode 100644 vendor/golang.org/x/text/CONTRIBUTORS delete mode 100644 vendor/golang.org/x/text/LICENSE delete mode 100644 vendor/golang.org/x/text/PATENTS delete mode 100644 vendor/golang.org/x/text/transform/transform.go delete mode 100644 vendor/golang.org/x/text/unicode/norm/composition.go delete mode 100644 vendor/golang.org/x/text/unicode/norm/forminfo.go delete mode 100644 vendor/golang.org/x/text/unicode/norm/input.go delete mode 100644 vendor/golang.org/x/text/unicode/norm/iter.go delete mode 100644 vendor/golang.org/x/text/unicode/norm/normalize.go delete mode 100644 vendor/golang.org/x/text/unicode/norm/readwriter.go delete mode 100644 vendor/golang.org/x/text/unicode/norm/tables10.0.0.go delete mode 100644 vendor/golang.org/x/text/unicode/norm/tables11.0.0.go delete mode 100644 vendor/golang.org/x/text/unicode/norm/tables9.0.0.go delete mode 100644 vendor/golang.org/x/text/unicode/norm/transform.go delete mode 100644 vendor/golang.org/x/text/unicode/norm/trie.go delete mode 100644 vendor/gopkg.in/jcmturner/aescts.v1/.gitignore delete mode 100644 vendor/gopkg.in/jcmturner/aescts.v1/LICENSE delete mode 100644 vendor/gopkg.in/jcmturner/aescts.v1/README.md delete mode 100644 vendor/gopkg.in/jcmturner/aescts.v1/aescts.go delete mode 100644 vendor/gopkg.in/jcmturner/dnsutils.v1/.gitignore delete mode 100644 vendor/gopkg.in/jcmturner/dnsutils.v1/.travis.yml delete mode 100644 vendor/gopkg.in/jcmturner/dnsutils.v1/LICENSE delete mode 100644 vendor/gopkg.in/jcmturner/dnsutils.v1/srv.go delete mode 100644 vendor/gopkg.in/jcmturner/gokrb5.v7/LICENSE delete mode 100644 vendor/gopkg.in/jcmturner/gokrb5.v7/asn1tools/tools.go delete mode 100644 vendor/gopkg.in/jcmturner/gokrb5.v7/client/ASExchange.go delete mode 100644 vendor/gopkg.in/jcmturner/gokrb5.v7/client/TGSExchange.go delete mode 100644 vendor/gopkg.in/jcmturner/gokrb5.v7/client/cache.go delete mode 100644 vendor/gopkg.in/jcmturner/gokrb5.v7/client/client.go delete mode 100644 vendor/gopkg.in/jcmturner/gokrb5.v7/client/network.go delete mode 100644 vendor/gopkg.in/jcmturner/gokrb5.v7/client/passwd.go delete mode 100644 vendor/gopkg.in/jcmturner/gokrb5.v7/client/session.go delete mode 100644 vendor/gopkg.in/jcmturner/gokrb5.v7/client/settings.go delete mode 100644 vendor/gopkg.in/jcmturner/gokrb5.v7/config/error.go delete mode 100644 vendor/gopkg.in/jcmturner/gokrb5.v7/config/hosts.go delete mode 100644 vendor/gopkg.in/jcmturner/gokrb5.v7/config/krb5conf.go delete mode 100644 vendor/gopkg.in/jcmturner/gokrb5.v7/credentials/ccache.go delete mode 100644 vendor/gopkg.in/jcmturner/gokrb5.v7/credentials/credentials.go delete mode 100644 vendor/gopkg.in/jcmturner/gokrb5.v7/crypto/aes128-cts-hmac-sha1-96.go delete mode 100644 vendor/gopkg.in/jcmturner/gokrb5.v7/crypto/aes128-cts-hmac-sha256-128.go delete mode 100644 vendor/gopkg.in/jcmturner/gokrb5.v7/crypto/aes256-cts-hmac-sha1-96.go delete mode 100644 vendor/gopkg.in/jcmturner/gokrb5.v7/crypto/aes256-cts-hmac-sha384-192.go delete mode 100644 vendor/gopkg.in/jcmturner/gokrb5.v7/crypto/common/common.go delete mode 100644 vendor/gopkg.in/jcmturner/gokrb5.v7/crypto/crypto.go delete mode 100644 vendor/gopkg.in/jcmturner/gokrb5.v7/crypto/des3-cbc-sha1-kd.go delete mode 100644 vendor/gopkg.in/jcmturner/gokrb5.v7/crypto/etype/etype.go delete mode 100644 vendor/gopkg.in/jcmturner/gokrb5.v7/crypto/rc4-hmac.go delete mode 100644 vendor/gopkg.in/jcmturner/gokrb5.v7/crypto/rfc3961/encryption.go delete mode 100644 vendor/gopkg.in/jcmturner/gokrb5.v7/crypto/rfc3961/keyDerivation.go delete mode 100644 vendor/gopkg.in/jcmturner/gokrb5.v7/crypto/rfc3961/nfold.go delete mode 100644 vendor/gopkg.in/jcmturner/gokrb5.v7/crypto/rfc3962/encryption.go delete mode 100644 vendor/gopkg.in/jcmturner/gokrb5.v7/crypto/rfc3962/keyDerivation.go delete mode 100644 vendor/gopkg.in/jcmturner/gokrb5.v7/crypto/rfc4757/checksum.go delete mode 100644 vendor/gopkg.in/jcmturner/gokrb5.v7/crypto/rfc4757/encryption.go delete mode 100644 vendor/gopkg.in/jcmturner/gokrb5.v7/crypto/rfc4757/keyDerivation.go delete mode 100644 vendor/gopkg.in/jcmturner/gokrb5.v7/crypto/rfc4757/msgtype.go delete mode 100644 vendor/gopkg.in/jcmturner/gokrb5.v7/crypto/rfc8009/encryption.go delete mode 100644 vendor/gopkg.in/jcmturner/gokrb5.v7/crypto/rfc8009/keyDerivation.go delete mode 100644 vendor/gopkg.in/jcmturner/gokrb5.v7/gssapi/MICToken.go delete mode 100644 vendor/gopkg.in/jcmturner/gokrb5.v7/gssapi/README.md delete mode 100644 vendor/gopkg.in/jcmturner/gokrb5.v7/gssapi/contextFlags.go delete mode 100644 vendor/gopkg.in/jcmturner/gokrb5.v7/gssapi/gssapi.go delete mode 100644 vendor/gopkg.in/jcmturner/gokrb5.v7/gssapi/wrapToken.go delete mode 100644 vendor/gopkg.in/jcmturner/gokrb5.v7/iana/addrtype/constants.go delete mode 100644 vendor/gopkg.in/jcmturner/gokrb5.v7/iana/adtype/constants.go delete mode 100644 vendor/gopkg.in/jcmturner/gokrb5.v7/iana/asnAppTag/constants.go delete mode 100644 vendor/gopkg.in/jcmturner/gokrb5.v7/iana/chksumtype/constants.go delete mode 100644 vendor/gopkg.in/jcmturner/gokrb5.v7/iana/constants.go delete mode 100644 vendor/gopkg.in/jcmturner/gokrb5.v7/iana/errorcode/constants.go delete mode 100644 vendor/gopkg.in/jcmturner/gokrb5.v7/iana/etypeID/constants.go delete mode 100644 vendor/gopkg.in/jcmturner/gokrb5.v7/iana/flags/constants.go delete mode 100644 vendor/gopkg.in/jcmturner/gokrb5.v7/iana/keyusage/constants.go delete mode 100644 vendor/gopkg.in/jcmturner/gokrb5.v7/iana/msgtype/constants.go delete mode 100644 vendor/gopkg.in/jcmturner/gokrb5.v7/iana/nametype/constants.go delete mode 100644 vendor/gopkg.in/jcmturner/gokrb5.v7/iana/patype/constants.go delete mode 100644 vendor/gopkg.in/jcmturner/gokrb5.v7/kadmin/changepasswddata.go delete mode 100644 vendor/gopkg.in/jcmturner/gokrb5.v7/kadmin/message.go delete mode 100644 vendor/gopkg.in/jcmturner/gokrb5.v7/kadmin/passwd.go delete mode 100644 vendor/gopkg.in/jcmturner/gokrb5.v7/keytab/keytab.go delete mode 100644 vendor/gopkg.in/jcmturner/gokrb5.v7/krberror/error.go delete mode 100644 vendor/gopkg.in/jcmturner/gokrb5.v7/messages/APRep.go delete mode 100644 vendor/gopkg.in/jcmturner/gokrb5.v7/messages/APReq.go delete mode 100644 vendor/gopkg.in/jcmturner/gokrb5.v7/messages/KDCRep.go delete mode 100644 vendor/gopkg.in/jcmturner/gokrb5.v7/messages/KDCReq.go delete mode 100644 vendor/gopkg.in/jcmturner/gokrb5.v7/messages/KRBCred.go delete mode 100644 vendor/gopkg.in/jcmturner/gokrb5.v7/messages/KRBError.go delete mode 100644 vendor/gopkg.in/jcmturner/gokrb5.v7/messages/KRBPriv.go delete mode 100644 vendor/gopkg.in/jcmturner/gokrb5.v7/messages/KRBSafe.go delete mode 100644 vendor/gopkg.in/jcmturner/gokrb5.v7/messages/Ticket.go delete mode 100644 vendor/gopkg.in/jcmturner/gokrb5.v7/pac/client_claims.go delete mode 100644 vendor/gopkg.in/jcmturner/gokrb5.v7/pac/client_info.go delete mode 100644 vendor/gopkg.in/jcmturner/gokrb5.v7/pac/credentials_info.go delete mode 100644 vendor/gopkg.in/jcmturner/gokrb5.v7/pac/device_claims.go delete mode 100644 vendor/gopkg.in/jcmturner/gokrb5.v7/pac/device_info.go delete mode 100644 vendor/gopkg.in/jcmturner/gokrb5.v7/pac/kerb_validation_info.go delete mode 100644 vendor/gopkg.in/jcmturner/gokrb5.v7/pac/pac_type.go delete mode 100644 vendor/gopkg.in/jcmturner/gokrb5.v7/pac/s4u_delegation_info.go delete mode 100644 vendor/gopkg.in/jcmturner/gokrb5.v7/pac/signature_data.go delete mode 100644 vendor/gopkg.in/jcmturner/gokrb5.v7/pac/supplemental_cred.go delete mode 100644 vendor/gopkg.in/jcmturner/gokrb5.v7/pac/upn_dns_info.go delete mode 100644 vendor/gopkg.in/jcmturner/gokrb5.v7/types/Authenticator.go delete mode 100644 vendor/gopkg.in/jcmturner/gokrb5.v7/types/AuthorizationData.go delete mode 100644 vendor/gopkg.in/jcmturner/gokrb5.v7/types/Cryptosystem.go delete mode 100644 vendor/gopkg.in/jcmturner/gokrb5.v7/types/HostAddress.go delete mode 100644 vendor/gopkg.in/jcmturner/gokrb5.v7/types/KerberosFlags.go delete mode 100644 vendor/gopkg.in/jcmturner/gokrb5.v7/types/PAData.go delete mode 100644 vendor/gopkg.in/jcmturner/gokrb5.v7/types/PrincipalName.go delete mode 100644 vendor/gopkg.in/jcmturner/gokrb5.v7/types/TypedData.go delete mode 100644 vendor/gopkg.in/jcmturner/rpc.v1/LICENSE delete mode 100644 vendor/gopkg.in/jcmturner/rpc.v1/mstypes/claims.go delete mode 100644 vendor/gopkg.in/jcmturner/rpc.v1/mstypes/common.go delete mode 100644 vendor/gopkg.in/jcmturner/rpc.v1/mstypes/filetime.go delete mode 100644 vendor/gopkg.in/jcmturner/rpc.v1/mstypes/group_membership.go delete mode 100644 vendor/gopkg.in/jcmturner/rpc.v1/mstypes/kerb_sid_and_attributes.go delete mode 100644 vendor/gopkg.in/jcmturner/rpc.v1/mstypes/reader.go delete mode 100644 vendor/gopkg.in/jcmturner/rpc.v1/mstypes/rpc_unicode_string.go delete mode 100644 vendor/gopkg.in/jcmturner/rpc.v1/mstypes/sid.go delete mode 100644 vendor/gopkg.in/jcmturner/rpc.v1/mstypes/user_session_key.go delete mode 100644 vendor/gopkg.in/jcmturner/rpc.v1/ndr/arrays.go delete mode 100644 vendor/gopkg.in/jcmturner/rpc.v1/ndr/decoder.go delete mode 100644 vendor/gopkg.in/jcmturner/rpc.v1/ndr/error.go delete mode 100644 vendor/gopkg.in/jcmturner/rpc.v1/ndr/header.go delete mode 100644 vendor/gopkg.in/jcmturner/rpc.v1/ndr/pipe.go delete mode 100644 vendor/gopkg.in/jcmturner/rpc.v1/ndr/primitives.go delete mode 100644 vendor/gopkg.in/jcmturner/rpc.v1/ndr/rawbytes.go delete mode 100644 vendor/gopkg.in/jcmturner/rpc.v1/ndr/strings.go delete mode 100644 vendor/gopkg.in/jcmturner/rpc.v1/ndr/tags.go delete mode 100644 vendor/gopkg.in/jcmturner/rpc.v1/ndr/union.go delete mode 100644 vendor/modules.txt diff --git a/go.mod b/go.mod index b7aaf5f7..b662bcd2 100644 --- a/go.mod +++ b/go.mod @@ -9,4 +9,7 @@ require ( github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8 github.com/kelseyhightower/envconfig v1.4.0 github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c + go.opentelemetry.io/otel v0.13.0 + go.opentelemetry.io/otel/exporters/stdout v0.13.0 + go.opentelemetry.io/otel/sdk v0.13.0 ) diff --git a/go.sum b/go.sum index e4319b4c..4e06601e 100644 --- a/go.sum +++ b/go.sum @@ -1,10 +1,14 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/DataDog/sketches-go v0.0.1 h1:RtG+76WKgZuz6FIaGsjoPePmadDBkuD/KC6+ZWu78b8= +github.com/DataDog/sketches-go v0.0.1/go.mod h1:Q5DbzQ+3AkgGwymQO7aZFNP7ns2lZKGtvRBzRXfdi60= github.com/Shopify/sarama v1.25.0/go.mod h1:y/CFFTO9eaMTNriwu/Q+W4eioLqiDMGkA1W+gmdfj8w= github.com/Shopify/sarama v1.27.0 h1:tqo2zmyzPf1+gwTTwhI6W+EXDw4PVSczynpHKFtVAmo= github.com/Shopify/sarama v1.27.0/go.mod h1:aCdj6ymI8uyPEux1JJ9gcaDT6cinjGhNCAhs54taSUo= github.com/Shopify/toxiproxy v2.1.4+incompatible h1:TKdv8HiTLgE5wdJuEML90aBgNWsokNbMijUGhmcoBJc= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= +github.com/benbjohnson/clock v1.0.3 h1:vkLuvpK4fmtSCuo60+yC63p7y0BmQ8gm5ZXGuBCJyXg= +github.com/benbjohnson/clock v1.0.3/go.mod h1:bGMdMPoPVvcYyt1gHDf4J2KE153Yf9BuiUKYMaxlTDM= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudevents/sdk-go/protocol/kafka_sarama/v2 v2.2.0 h1:PB+tnuauZWE2RnfDJnyv8wuLc1tkWk/p8mYiy2xzdvQ= github.com/cloudevents/sdk-go/protocol/kafka_sarama/v2 v2.2.0/go.mod h1:XbBXL/a5TGNGs5N4UreDCIK7F71MNrXURJeVXury+XY= @@ -43,6 +47,10 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.4.1 h1:/exdXoGamhu5ONeUJH0deniYLWYvQwW66yvlfiiKTu0= github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= +github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= @@ -94,6 +102,8 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.0 h1:jlIyCplCJFULU/01vCkhKuTyc3OorI3bJFuw6obfgho= github.com/stretchr/testify v1.6.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c h1:u40Z8hqBAAQyv+vATcGgV0YCnDjqSL7/q/JyPhhJSPk= @@ -102,6 +112,12 @@ github.com/xdg/stringprep v1.0.0 h1:d9X0esnoa3dFsV0FG35rAT0RIhYFlPq7MiP+DW89La0= github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= go.opencensus.io v0.22.0 h1:C9hSCOW830chIVkdja34wa6Ky+IzWllkUinR+BtRZd4= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opentelemetry.io/otel v0.13.0 h1:2isEnyzjjJZq6r2EKMsFj4TxiQiexsM04AVhwbR/oBA= +go.opentelemetry.io/otel v0.13.0/go.mod h1:dlSNewoRYikTkotEnxdmuBHgzT+k/idJSfDv/FxEnOY= +go.opentelemetry.io/otel/exporters/stdout v0.13.0 h1:A+XiGIPQbGoJoBOJfKAKnZyiUSjSWvL3XWETUvtom5k= +go.opentelemetry.io/otel/exporters/stdout v0.13.0/go.mod h1:JJt8RpNY6K+ft9ir3iKpceCvT/rhzJXEExGrWFCbv1o= +go.opentelemetry.io/otel/sdk v0.13.0 h1:4VCfpKamZ8GtnepXxMRurSpHpMKkcxhtO33z1S4rGDQ= +go.opentelemetry.io/otel/sdk v0.13.0/go.mod h1:dKvLH8Uu8LcEPlSAUsfW7kMGaJBhk/1NYvpPZ6wIMbU= go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI= diff --git a/tracing/init.go b/tracing/init.go new file mode 100644 index 00000000..96a4cb9e --- /dev/null +++ b/tracing/init.go @@ -0,0 +1,30 @@ +package tracing + +import ( + "log" + + "go.opentelemetry.io/otel" + otelglobal "go.opentelemetry.io/otel/api/global" + "go.opentelemetry.io/otel/exporters/stdout" + "go.opentelemetry.io/otel/propagators" + sdktrace "go.opentelemetry.io/otel/sdk/trace" +) + +func InitTracer() { + exporter, err := stdout.NewExporter(stdout.WithPrettyPrint()) + if err != nil { + log.Fatal(err) + } + cfg := sdktrace.Config{ + DefaultSampler: sdktrace.AlwaysSample(), + } + tp := sdktrace.NewTracerProvider( + sdktrace.WithConfig(cfg), + sdktrace.WithSyncer(exporter), + ) + if err != nil { + log.Fatal(err) + } + otelglobal.SetTracerProvider(tp) + otelglobal.SetTextMapPropagator(otel.NewCompositeTextMapPropagator(propagators.TraceContext{})) +} diff --git a/vendor/github.com/Shopify/sarama/.gitignore b/vendor/github.com/Shopify/sarama/.gitignore deleted file mode 100644 index 2c9adc20..00000000 --- a/vendor/github.com/Shopify/sarama/.gitignore +++ /dev/null @@ -1,29 +0,0 @@ -# Compiled Object files, Static and Dynamic libs (Shared Objects) -*.o -*.a -*.so -*.test - -# Folders -_obj -_test -.vagrant - -# Architecture specific extensions/prefixes -*.[568vq] -[568vq].out - -*.cgo1.go -*.cgo2.c -_cgo_defun.c -_cgo_gotypes.go -_cgo_export.* - -_testmain.go - -*.exe - -coverage.txt -profile.out - -simplest-uncommitted-msg-0.1-jar-with-dependencies.jar diff --git a/vendor/github.com/Shopify/sarama/.golangci.yml b/vendor/github.com/Shopify/sarama/.golangci.yml deleted file mode 100644 index ce2b5230..00000000 --- a/vendor/github.com/Shopify/sarama/.golangci.yml +++ /dev/null @@ -1,77 +0,0 @@ -run: - timeout: 5m - deadline: 10m - -linters-settings: - govet: - check-shadowing: false - golint: - min-confidence: 0 - gocyclo: - min-complexity: 99 - maligned: - suggest-new: true - dupl: - threshold: 100 - goconst: - min-len: 2 - min-occurrences: 3 - misspell: - locale: US - goimports: - local-prefixes: github.com/Shopify/sarama - gocritic: - enabled-tags: - - diagnostic - - experimental - - opinionated - - performance - - style - disabled-checks: - - wrapperFunc - - ifElseChain - funlen: - lines: 300 - statements: 300 - -linters: - disable-all: true - enable: - - bodyclose - - deadcode - - depguard - - dogsled - # - dupl - - errcheck - - funlen - # - gocritic - - gocyclo - - gofmt - - goimports - # - golint - - gosec - # - gosimple - - govet - # - ineffassign - - interfacer - # - misspell - # - nakedret - # - scopelint - # - staticcheck - - structcheck - # - stylecheck - - typecheck - - unconvert - - unused - - varcheck - - whitespace - # - goconst - - gochecknoinits - -issues: - exclude: - - consider giving a name to these results - - include an explanation for nolint directive - - Potential Integer overflow made by strconv.Atoi result conversion to int16/32 - - Use of weak random number generator - - TLS MinVersion too low diff --git a/vendor/github.com/Shopify/sarama/CHANGELOG.md b/vendor/github.com/Shopify/sarama/CHANGELOG.md deleted file mode 100644 index 53204b9d..00000000 --- a/vendor/github.com/Shopify/sarama/CHANGELOG.md +++ /dev/null @@ -1,979 +0,0 @@ -# Changelog - -#### Unreleased - -#### Version 1.27.0 (2020-08-11) - -# Improvements - -#1466 - @rubenvp8510 - Expose kerberos fast negotiation configuration -#1695 - @KJTsanaktsidis - Use docker-compose to run the functional tests -#1699 - @wclaeys - Consumer group support for manually comitting offsets -#1714 - @bai - Bump Go to version 1.14.3, golangci-lint to 1.27.0 -#1726 - @d1egoaz - Include zstd on the functional tests -#1730 - @d1egoaz - KIP-42 Add producer and consumer interceptors -#1738 - @varun06 - fixed variable names that are named same as some std lib package names -#1741 - @varun06 - updated zstd dependency to latest v1.10.10 -#1743 - @varun06 - Fixed declaration dependencies and other lint issues in code base -#1763 - @alrs - remove deprecated tls options from test -#1769 - @bai - Add support for Kafka 2.6.0 - -# Fixes - -#1697 - @kvch - Use gofork for encoding/asn1 to fix ASN errors during Kerberos authentication -#1744 - @alrs - Fix isBalanced Function Signature - -#### Version 1.26.4 (2020-05-19) - -# Fixes - -- #1701 - @d1egoaz - Set server name only for the current broker -- #1694 - @dnwe - testfix: set KAFKA_HEAP_OPTS for zk and kafka - -#### Version 1.26.3 (2020-05-07) - -# Fixes - -- #1692 - @d1egoaz - Set tls ServerName to fix issue: either ServerName or InsecureSkipVerify must be specified in the tls.Config - -#### Version 1.26.2 (2020-05-06) - -# ⚠️ Known Issues - -This release has been marked as not ready for production and may be unstable, please use v1.26.4. - -# Improvements - -- #1560 - @iyacontrol - add sync pool for gzip 1-9 -- #1605 - @dnwe - feat: protocol support for V11 fetch w/ rackID -- #1617 - @sladkoff / @dwi-di / @random-dwi - Add support for alter/list partition reassignements APIs -- #1632 - @bai - Add support for Go 1.14 -- #1640 - @random-dwi - Feature/fix list partition reassignments -- #1646 - @mimaison - Add DescribeLogDirs to admin client -- #1667 - @bai - Add support for kafka 2.5.0 - -# Fixes - -- #1594 - @sladkoff - Sets ConfigEntry.Default flag in addition to the ConfigEntry.Source for Kafka versions > V1_1_0_0 -- #1601 - @alrs - fix: remove use of testing.T.FailNow() inside goroutine -- #1602 - @d1egoaz - adds a note about consumer groups Consume method -- #1607 - @darklore - Fix memory leak when Broker.Open and Broker.Close called repeatedly -- #1613 - @wblakecaldwell - Updated "retrying" log message when BackoffFunc implemented -- #1614 - @alrs - produce_response.go: Remove Unused Functions -- #1619 - @alrs - tools/kafka-producer-performance: prune unused flag variables -- #1639 - @agriffaut - Handle errors with no message but error code -- #1643 - @kzinglzy - fix `config.net.keepalive` -- #1644 - @KJTsanaktsidis - Fix brokers continually allocating new Session IDs -- #1645 - @Stephan14 - Remove broker(s) which no longer exist in metadata -- #1650 - @lavoiesl - Return the response error in heartbeatLoop -- #1661 - @KJTsanaktsidis - Fix "broker received out of order sequence" when brokers die -- #1666 - @KevinJCross - Bugfix: Allow TLS connections to work over socks proxy. - -#### Version 1.26.1 (2020-02-04) - -Improvements: -- Add requests-in-flight metric ([1539](https://github.com/Shopify/sarama/pull/1539)) -- Fix misleading example for cluster admin ([1595](https://github.com/Shopify/sarama/pull/1595)) -- Replace Travis with GitHub Actions, linters housekeeping ([1573](https://github.com/Shopify/sarama/pull/1573)) -- Allow BalanceStrategy to provide custom assignment data ([1592](https://github.com/Shopify/sarama/pull/1592)) - -Bug Fixes: -- Adds back Consumer.Offsets.CommitInterval to fix API ([1590](https://github.com/Shopify/sarama/pull/1590)) -- Fix error message s/CommitInterval/AutoCommit.Interval ([1589](https://github.com/Shopify/sarama/pull/1589)) - -#### Version 1.26.0 (2020-01-24) - -New Features: -- Enable zstd compression - ([1574](https://github.com/Shopify/sarama/pull/1574), - [1582](https://github.com/Shopify/sarama/pull/1582)) -- Support headers in tools kafka-console-producer - ([1549](https://github.com/Shopify/sarama/pull/1549)) - -Improvements: -- Add SASL AuthIdentity to SASL frames (authzid) - ([1585](https://github.com/Shopify/sarama/pull/1585)). - -Bug Fixes: -- Sending messages with ZStd compression enabled fails in multiple ways - ([1252](https://github.com/Shopify/sarama/issues/1252)). -- Use the broker for any admin on BrokerConfig - ([1571](https://github.com/Shopify/sarama/pull/1571)). -- Set DescribeConfigRequest Version field - ([1576](https://github.com/Shopify/sarama/pull/1576)). -- ConsumerGroup flooding logs with client/metadata update req - ([1578](https://github.com/Shopify/sarama/pull/1578)). -- MetadataRequest version in DescribeCluster - ([1580](https://github.com/Shopify/sarama/pull/1580)). -- Fix deadlock in consumer group handleError - ([1581](https://github.com/Shopify/sarama/pull/1581)) -- Fill in the Fetch{Request,Response} protocol - ([1582](https://github.com/Shopify/sarama/pull/1582)). -- Retry topic request on ControllerNotAvailable - ([1586](https://github.com/Shopify/sarama/pull/1586)). - -#### Version 1.25.0 (2020-01-13) - -New Features: -- Support TLS protocol in kafka-producer-performance - ([1538](https://github.com/Shopify/sarama/pull/1538)). -- Add support for kafka 2.4.0 - ([1552](https://github.com/Shopify/sarama/pull/1552)). - -Improvements: -- Allow the Consumer to disable auto-commit offsets - ([1164](https://github.com/Shopify/sarama/pull/1164)). -- Produce records with consistent timestamps - ([1455](https://github.com/Shopify/sarama/pull/1455)). - -Bug Fixes: -- Fix incorrect SetTopicMetadata name mentions - ([1534](https://github.com/Shopify/sarama/pull/1534)). -- Fix client.tryRefreshMetadata Println - ([1535](https://github.com/Shopify/sarama/pull/1535)). -- Fix panic on calling updateMetadata on closed client - ([1531](https://github.com/Shopify/sarama/pull/1531)). -- Fix possible faulty metrics in TestFuncProducing - ([1545](https://github.com/Shopify/sarama/pull/1545)). - -#### Version 1.24.1 (2019-10-31) - -New Features: -- Add DescribeLogDirs Request/Response pair - ([1520](https://github.com/Shopify/sarama/pull/1520)). - -Bug Fixes: -- Fix ClusterAdmin returning invalid controller ID on DescribeCluster - ([1518](https://github.com/Shopify/sarama/pull/1518)). -- Fix issue with consumergroup not rebalancing when new partition is added - ([1525](https://github.com/Shopify/sarama/pull/1525)). -- Ensure consistent use of read/write deadlines - ([1529](https://github.com/Shopify/sarama/pull/1529)). - -#### Version 1.24.0 (2019-10-09) - -New Features: -- Add sticky partition assignor - ([1416](https://github.com/Shopify/sarama/pull/1416)). -- Switch from cgo zstd package to pure Go implementation - ([1477](https://github.com/Shopify/sarama/pull/1477)). - -Improvements: -- Allow creating ClusterAdmin from client - ([1415](https://github.com/Shopify/sarama/pull/1415)). -- Set KafkaVersion in ListAcls method - ([1452](https://github.com/Shopify/sarama/pull/1452)). -- Set request version in CreateACL ClusterAdmin method - ([1458](https://github.com/Shopify/sarama/pull/1458)). -- Set request version in DeleteACL ClusterAdmin method - ([1461](https://github.com/Shopify/sarama/pull/1461)). -- Handle missed error codes on TopicMetaDataRequest and GroupCoordinatorRequest - ([1464](https://github.com/Shopify/sarama/pull/1464)). -- Remove direct usage of gofork - ([1465](https://github.com/Shopify/sarama/pull/1465)). -- Add support for Go 1.13 - ([1478](https://github.com/Shopify/sarama/pull/1478)). -- Improve behavior of NewMockListAclsResponse - ([1481](https://github.com/Shopify/sarama/pull/1481)). - -Bug Fixes: -- Fix race condition in consumergroup example - ([1434](https://github.com/Shopify/sarama/pull/1434)). -- Fix brokerProducer goroutine leak - ([1442](https://github.com/Shopify/sarama/pull/1442)). -- Use released version of lz4 library - ([1469](https://github.com/Shopify/sarama/pull/1469)). -- Set correct version in MockDeleteTopicsResponse - ([1484](https://github.com/Shopify/sarama/pull/1484)). -- Fix CLI help message typo - ([1494](https://github.com/Shopify/sarama/pull/1494)). - -Known Issues: -- Please **don't** use Zstd, as it doesn't work right now. - See https://github.com/Shopify/sarama/issues/1252 - -#### Version 1.23.1 (2019-07-22) - -Bug Fixes: -- Fix fetch delete bug record - ([1425](https://github.com/Shopify/sarama/pull/1425)). -- Handle SASL/OAUTHBEARER token rejection - ([1428](https://github.com/Shopify/sarama/pull/1428)). - -#### Version 1.23.0 (2019-07-02) - -New Features: -- Add support for Kafka 2.3.0 - ([1418](https://github.com/Shopify/sarama/pull/1418)). -- Add support for ListConsumerGroupOffsets v2 - ([1374](https://github.com/Shopify/sarama/pull/1374)). -- Add support for DeleteConsumerGroup - ([1417](https://github.com/Shopify/sarama/pull/1417)). -- Add support for SASLVersion configuration - ([1410](https://github.com/Shopify/sarama/pull/1410)). -- Add kerberos support - ([1366](https://github.com/Shopify/sarama/pull/1366)). - -Improvements: -- Improve sasl_scram_client example - ([1406](https://github.com/Shopify/sarama/pull/1406)). -- Fix shutdown and race-condition in consumer-group example - ([1404](https://github.com/Shopify/sarama/pull/1404)). -- Add support for error codes 77—81 - ([1397](https://github.com/Shopify/sarama/pull/1397)). -- Pool internal objects allocated per message - ([1385](https://github.com/Shopify/sarama/pull/1385)). -- Reduce packet decoder allocations - ([1373](https://github.com/Shopify/sarama/pull/1373)). -- Support timeout when fetching metadata - ([1359](https://github.com/Shopify/sarama/pull/1359)). - -Bug Fixes: -- Fix fetch size integer overflow - ([1376](https://github.com/Shopify/sarama/pull/1376)). -- Handle and log throttled FetchResponses - ([1383](https://github.com/Shopify/sarama/pull/1383)). -- Refactor misspelled word Resouce to Resource - ([1368](https://github.com/Shopify/sarama/pull/1368)). - -#### Version 1.22.1 (2019-04-29) - -Improvements: -- Use zstd 1.3.8 - ([1350](https://github.com/Shopify/sarama/pull/1350)). -- Add support for SaslHandshakeRequest v1 - ([1354](https://github.com/Shopify/sarama/pull/1354)). - -Bug Fixes: -- Fix V5 MetadataRequest nullable topics array - ([1353](https://github.com/Shopify/sarama/pull/1353)). -- Use a different SCRAM client for each broker connection - ([1349](https://github.com/Shopify/sarama/pull/1349)). -- Fix AllowAutoTopicCreation for MetadataRequest greater than v3 - ([1344](https://github.com/Shopify/sarama/pull/1344)). - -#### Version 1.22.0 (2019-04-09) - -New Features: -- Add Offline Replicas Operation to Client - ([1318](https://github.com/Shopify/sarama/pull/1318)). -- Allow using proxy when connecting to broker - ([1326](https://github.com/Shopify/sarama/pull/1326)). -- Implement ReadCommitted - ([1307](https://github.com/Shopify/sarama/pull/1307)). -- Add support for Kafka 2.2.0 - ([1331](https://github.com/Shopify/sarama/pull/1331)). -- Add SASL SCRAM-SHA-512 and SCRAM-SHA-256 mechanismes - ([1331](https://github.com/Shopify/sarama/pull/1295)). - -Improvements: -- Unregister all broker metrics on broker stop - ([1232](https://github.com/Shopify/sarama/pull/1232)). -- Add SCRAM authentication example - ([1303](https://github.com/Shopify/sarama/pull/1303)). -- Add consumergroup examples - ([1304](https://github.com/Shopify/sarama/pull/1304)). -- Expose consumer batch size metric - ([1296](https://github.com/Shopify/sarama/pull/1296)). -- Add TLS options to console producer and consumer - ([1300](https://github.com/Shopify/sarama/pull/1300)). -- Reduce client close bookkeeping - ([1297](https://github.com/Shopify/sarama/pull/1297)). -- Satisfy error interface in create responses - ([1154](https://github.com/Shopify/sarama/pull/1154)). -- Please lint gods - ([1346](https://github.com/Shopify/sarama/pull/1346)). - -Bug Fixes: -- Fix multi consumer group instance crash - ([1338](https://github.com/Shopify/sarama/pull/1338)). -- Update lz4 to latest version - ([1347](https://github.com/Shopify/sarama/pull/1347)). -- Retry ErrNotCoordinatorForConsumer in new consumergroup session - ([1231](https://github.com/Shopify/sarama/pull/1231)). -- Fix cleanup error handler - ([1332](https://github.com/Shopify/sarama/pull/1332)). -- Fix rate condition in PartitionConsumer - ([1156](https://github.com/Shopify/sarama/pull/1156)). - -#### Version 1.21.0 (2019-02-24) - -New Features: -- Add CreateAclRequest, DescribeAclRequest, DeleteAclRequest - ([1236](https://github.com/Shopify/sarama/pull/1236)). -- Add DescribeTopic, DescribeConsumerGroup, ListConsumerGroups, ListConsumerGroupOffsets admin requests - ([1178](https://github.com/Shopify/sarama/pull/1178)). -- Implement SASL/OAUTHBEARER - ([1240](https://github.com/Shopify/sarama/pull/1240)). - -Improvements: -- Add Go mod support - ([1282](https://github.com/Shopify/sarama/pull/1282)). -- Add error codes 73—76 - ([1239](https://github.com/Shopify/sarama/pull/1239)). -- Add retry backoff function - ([1160](https://github.com/Shopify/sarama/pull/1160)). -- Maintain metadata in the producer even when retries are disabled - ([1189](https://github.com/Shopify/sarama/pull/1189)). -- Include ReplicaAssignment in ListTopics - ([1274](https://github.com/Shopify/sarama/pull/1274)). -- Add producer performance tool - ([1222](https://github.com/Shopify/sarama/pull/1222)). -- Add support LogAppend timestamps - ([1258](https://github.com/Shopify/sarama/pull/1258)). - -Bug Fixes: -- Fix potential deadlock when a heartbeat request fails - ([1286](https://github.com/Shopify/sarama/pull/1286)). -- Fix consuming compacted topic - ([1227](https://github.com/Shopify/sarama/pull/1227)). -- Set correct Kafka version for DescribeConfigsRequest v1 - ([1277](https://github.com/Shopify/sarama/pull/1277)). -- Update kafka test version - ([1273](https://github.com/Shopify/sarama/pull/1273)). - -#### Version 1.20.1 (2019-01-10) - -New Features: -- Add optional replica id in offset request - ([1100](https://github.com/Shopify/sarama/pull/1100)). - -Improvements: -- Implement DescribeConfigs Request + Response v1 & v2 - ([1230](https://github.com/Shopify/sarama/pull/1230)). -- Reuse compression objects - ([1185](https://github.com/Shopify/sarama/pull/1185)). -- Switch from png to svg for GoDoc link in README - ([1243](https://github.com/Shopify/sarama/pull/1243)). -- Fix typo in deprecation notice for FetchResponseBlock.Records - ([1242](https://github.com/Shopify/sarama/pull/1242)). -- Fix typos in consumer metadata response file - ([1244](https://github.com/Shopify/sarama/pull/1244)). - -Bug Fixes: -- Revert to individual msg retries for non-idempotent - ([1203](https://github.com/Shopify/sarama/pull/1203)). -- Respect MaxMessageBytes limit for uncompressed messages - ([1141](https://github.com/Shopify/sarama/pull/1141)). - -#### Version 1.20.0 (2018-12-10) - -New Features: - - Add support for zstd compression - ([#1170](https://github.com/Shopify/sarama/pull/1170)). - - Add support for Idempotent Producer - ([#1152](https://github.com/Shopify/sarama/pull/1152)). - - Add support support for Kafka 2.1.0 - ([#1229](https://github.com/Shopify/sarama/pull/1229)). - - Add support support for OffsetCommit request/response pairs versions v1 to v5 - ([#1201](https://github.com/Shopify/sarama/pull/1201)). - - Add support support for OffsetFetch request/response pair up to version v5 - ([#1198](https://github.com/Shopify/sarama/pull/1198)). - -Improvements: - - Export broker's Rack setting - ([#1173](https://github.com/Shopify/sarama/pull/1173)). - - Always use latest patch version of Go on CI - ([#1202](https://github.com/Shopify/sarama/pull/1202)). - - Add error codes 61 to 72 - ([#1195](https://github.com/Shopify/sarama/pull/1195)). - -Bug Fixes: - - Fix build without cgo - ([#1182](https://github.com/Shopify/sarama/pull/1182)). - - Fix go vet suggestion in consumer group file - ([#1209](https://github.com/Shopify/sarama/pull/1209)). - - Fix typos in code and comments - ([#1228](https://github.com/Shopify/sarama/pull/1228)). - -#### Version 1.19.0 (2018-09-27) - -New Features: - - Implement a higher-level consumer group - ([#1099](https://github.com/Shopify/sarama/pull/1099)). - -Improvements: - - Add support for Go 1.11 - ([#1176](https://github.com/Shopify/sarama/pull/1176)). - -Bug Fixes: - - Fix encoding of `MetadataResponse` with version 2 and higher - ([#1174](https://github.com/Shopify/sarama/pull/1174)). - - Fix race condition in mock async producer - ([#1174](https://github.com/Shopify/sarama/pull/1174)). - -#### Version 1.18.0 (2018-09-07) - -New Features: - - Make `Partitioner.RequiresConsistency` vary per-message - ([#1112](https://github.com/Shopify/sarama/pull/1112)). - - Add customizable partitioner - ([#1118](https://github.com/Shopify/sarama/pull/1118)). - - Add `ClusterAdmin` support for `CreateTopic`, `DeleteTopic`, `CreatePartitions`, - `DeleteRecords`, `DescribeConfig`, `AlterConfig`, `CreateACL`, `ListAcls`, `DeleteACL` - ([#1055](https://github.com/Shopify/sarama/pull/1055)). - -Improvements: - - Add support for Kafka 2.0.0 - ([#1149](https://github.com/Shopify/sarama/pull/1149)). - - Allow setting `LocalAddr` when dialing an address to support multi-homed hosts - ([#1123](https://github.com/Shopify/sarama/pull/1123)). - - Simpler offset management - ([#1127](https://github.com/Shopify/sarama/pull/1127)). - -Bug Fixes: - - Fix mutation of `ProducerMessage.MetaData` when producing to Kafka - ([#1110](https://github.com/Shopify/sarama/pull/1110)). - - Fix consumer block when response did not contain all the - expected topic/partition blocks - ([#1086](https://github.com/Shopify/sarama/pull/1086)). - - Fix consumer block when response contains only constrol messages - ([#1115](https://github.com/Shopify/sarama/pull/1115)). - - Add timeout config for ClusterAdmin requests - ([#1142](https://github.com/Shopify/sarama/pull/1142)). - - Add version check when producing message with headers - ([#1117](https://github.com/Shopify/sarama/pull/1117)). - - Fix `MetadataRequest` for empty list of topics - ([#1132](https://github.com/Shopify/sarama/pull/1132)). - - Fix producer topic metadata on-demand fetch when topic error happens in metadata response - ([#1125](https://github.com/Shopify/sarama/pull/1125)). - -#### Version 1.17.0 (2018-05-30) - -New Features: - - Add support for gzip compression levels - ([#1044](https://github.com/Shopify/sarama/pull/1044)). - - Add support for Metadata request/response pairs versions v1 to v5 - ([#1047](https://github.com/Shopify/sarama/pull/1047), - [#1069](https://github.com/Shopify/sarama/pull/1069)). - - Add versioning to JoinGroup request/response pairs - ([#1098](https://github.com/Shopify/sarama/pull/1098)) - - Add support for CreatePartitions, DeleteGroups, DeleteRecords request/response pairs - ([#1065](https://github.com/Shopify/sarama/pull/1065), - [#1096](https://github.com/Shopify/sarama/pull/1096), - [#1027](https://github.com/Shopify/sarama/pull/1027)). - - Add `Controller()` method to Client interface - ([#1063](https://github.com/Shopify/sarama/pull/1063)). - -Improvements: - - ConsumerMetadataReq/Resp has been migrated to FindCoordinatorReq/Resp - ([#1010](https://github.com/Shopify/sarama/pull/1010)). - - Expose missing protocol parts: `msgSet` and `recordBatch` - ([#1049](https://github.com/Shopify/sarama/pull/1049)). - - Add support for v1 DeleteTopics Request - ([#1052](https://github.com/Shopify/sarama/pull/1052)). - - Add support for Go 1.10 - ([#1064](https://github.com/Shopify/sarama/pull/1064)). - - Claim support for Kafka 1.1.0 - ([#1073](https://github.com/Shopify/sarama/pull/1073)). - -Bug Fixes: - - Fix FindCoordinatorResponse.encode to allow nil Coordinator - ([#1050](https://github.com/Shopify/sarama/pull/1050), - [#1051](https://github.com/Shopify/sarama/pull/1051)). - - Clear all metadata when we have the latest topic info - ([#1033](https://github.com/Shopify/sarama/pull/1033)). - - Make `PartitionConsumer.Close` idempotent - ([#1092](https://github.com/Shopify/sarama/pull/1092)). - -#### Version 1.16.0 (2018-02-12) - -New Features: - - Add support for the Create/Delete Topics request/response pairs - ([#1007](https://github.com/Shopify/sarama/pull/1007), - [#1008](https://github.com/Shopify/sarama/pull/1008)). - - Add support for the Describe/Create/Delete ACL request/response pairs - ([#1009](https://github.com/Shopify/sarama/pull/1009)). - - Add support for the five transaction-related request/response pairs - ([#1016](https://github.com/Shopify/sarama/pull/1016)). - -Improvements: - - Permit setting version on mock producer responses - ([#999](https://github.com/Shopify/sarama/pull/999)). - - Add `NewMockBrokerListener` helper for testing TLS connections - ([#1019](https://github.com/Shopify/sarama/pull/1019)). - - Changed the default value for `Consumer.Fetch.Default` from 32KiB to 1MiB - which results in much higher throughput in most cases - ([#1024](https://github.com/Shopify/sarama/pull/1024)). - - Reuse the `time.Ticker` across fetch requests in the PartitionConsumer to - reduce CPU and memory usage when processing many partitions - ([#1028](https://github.com/Shopify/sarama/pull/1028)). - - Assign relative offsets to messages in the producer to save the brokers a - recompression pass - ([#1002](https://github.com/Shopify/sarama/pull/1002), - [#1015](https://github.com/Shopify/sarama/pull/1015)). - -Bug Fixes: - - Fix producing uncompressed batches with the new protocol format - ([#1032](https://github.com/Shopify/sarama/issues/1032)). - - Fix consuming compacted topics with the new protocol format - ([#1005](https://github.com/Shopify/sarama/issues/1005)). - - Fix consuming topics with a mix of protocol formats - ([#1021](https://github.com/Shopify/sarama/issues/1021)). - - Fix consuming when the broker includes multiple batches in a single response - ([#1022](https://github.com/Shopify/sarama/issues/1022)). - - Fix detection of `PartialTrailingMessage` when the partial message was - truncated before the magic value indicating its version - ([#1030](https://github.com/Shopify/sarama/pull/1030)). - - Fix expectation-checking in the mock of `SyncProducer.SendMessages` - ([#1035](https://github.com/Shopify/sarama/pull/1035)). - -#### Version 1.15.0 (2017-12-08) - -New Features: - - Claim official support for Kafka 1.0, though it did already work - ([#984](https://github.com/Shopify/sarama/pull/984)). - - Helper methods for Kafka version numbers to/from strings - ([#989](https://github.com/Shopify/sarama/pull/989)). - - Implement CreatePartitions request/response - ([#985](https://github.com/Shopify/sarama/pull/985)). - -Improvements: - - Add error codes 45-60 - ([#986](https://github.com/Shopify/sarama/issues/986)). - -Bug Fixes: - - Fix slow consuming for certain Kafka 0.11/1.0 configurations - ([#982](https://github.com/Shopify/sarama/pull/982)). - - Correctly determine when a FetchResponse contains the new message format - ([#990](https://github.com/Shopify/sarama/pull/990)). - - Fix producing with multiple headers - ([#996](https://github.com/Shopify/sarama/pull/996)). - - Fix handling of truncated record batches - ([#998](https://github.com/Shopify/sarama/pull/998)). - - Fix leaking metrics when closing brokers - ([#991](https://github.com/Shopify/sarama/pull/991)). - -#### Version 1.14.0 (2017-11-13) - -New Features: - - Add support for the new Kafka 0.11 record-batch format, including the wire - protocol and the necessary behavioural changes in the producer and consumer. - Transactions and idempotency are not yet supported, but producing and - consuming should work with all the existing bells and whistles (batching, - compression, etc) as well as the new custom headers. Thanks to Vlad Hanciuta - of Arista Networks for this work. Part of - ([#901](https://github.com/Shopify/sarama/issues/901)). - -Bug Fixes: - - Fix encoding of ProduceResponse versions in test - ([#970](https://github.com/Shopify/sarama/pull/970)). - - Return partial replicas list when we have it - ([#975](https://github.com/Shopify/sarama/pull/975)). - -#### Version 1.13.0 (2017-10-04) - -New Features: - - Support for FetchRequest version 3 - ([#905](https://github.com/Shopify/sarama/pull/905)). - - Permit setting version on mock FetchResponses - ([#939](https://github.com/Shopify/sarama/pull/939)). - - Add a configuration option to support storing only minimal metadata for - extremely large clusters - ([#937](https://github.com/Shopify/sarama/pull/937)). - - Add `PartitionOffsetManager.ResetOffset` for backtracking tracked offsets - ([#932](https://github.com/Shopify/sarama/pull/932)). - -Improvements: - - Provide the block-level timestamp when consuming compressed messages - ([#885](https://github.com/Shopify/sarama/issues/885)). - - `Client.Replicas` and `Client.InSyncReplicas` now respect the order returned - by the broker, which can be meaningful - ([#930](https://github.com/Shopify/sarama/pull/930)). - - Use a `Ticker` to reduce consumer timer overhead at the cost of higher - variance in the actual timeout - ([#933](https://github.com/Shopify/sarama/pull/933)). - -Bug Fixes: - - Gracefully handle messages with negative timestamps - ([#907](https://github.com/Shopify/sarama/pull/907)). - - Raise a proper error when encountering an unknown message version - ([#940](https://github.com/Shopify/sarama/pull/940)). - -#### Version 1.12.0 (2017-05-08) - -New Features: - - Added support for the `ApiVersions` request and response pair, and Kafka - version 0.10.2 ([#867](https://github.com/Shopify/sarama/pull/867)). Note - that you still need to specify the Kafka version in the Sarama configuration - for the time being. - - Added a `Brokers` method to the Client which returns the complete set of - active brokers ([#813](https://github.com/Shopify/sarama/pull/813)). - - Added an `InSyncReplicas` method to the Client which returns the set of all - in-sync broker IDs for the given partition, now that the Kafka versions for - which this was misleading are no longer in our supported set - ([#872](https://github.com/Shopify/sarama/pull/872)). - - Added a `NewCustomHashPartitioner` method which allows constructing a hash - partitioner with a custom hash method in case the default (FNV-1a) is not - suitable - ([#837](https://github.com/Shopify/sarama/pull/837), - [#841](https://github.com/Shopify/sarama/pull/841)). - -Improvements: - - Recognize more Kafka error codes - ([#859](https://github.com/Shopify/sarama/pull/859)). - -Bug Fixes: - - Fix an issue where decoding a malformed FetchRequest would not return the - correct error ([#818](https://github.com/Shopify/sarama/pull/818)). - - Respect ordering of group protocols in JoinGroupRequests. This fix is - transparent if you're using the `AddGroupProtocol` or - `AddGroupProtocolMetadata` helpers; otherwise you will need to switch from - the `GroupProtocols` field (now deprecated) to use `OrderedGroupProtocols` - ([#812](https://github.com/Shopify/sarama/issues/812)). - - Fix an alignment-related issue with atomics on 32-bit architectures - ([#859](https://github.com/Shopify/sarama/pull/859)). - -#### Version 1.11.0 (2016-12-20) - -_Important:_ As of Sarama 1.11 it is necessary to set the config value of -`Producer.Return.Successes` to true in order to use the SyncProducer. Previous -versions would silently override this value when instantiating a SyncProducer -which led to unexpected values and data races. - -New Features: - - Metrics! Thanks to Sébastien Launay for all his work on this feature - ([#701](https://github.com/Shopify/sarama/pull/701), - [#746](https://github.com/Shopify/sarama/pull/746), - [#766](https://github.com/Shopify/sarama/pull/766)). - - Add support for LZ4 compression - ([#786](https://github.com/Shopify/sarama/pull/786)). - - Add support for ListOffsetRequest v1 and Kafka 0.10.1 - ([#775](https://github.com/Shopify/sarama/pull/775)). - - Added a `HighWaterMarks` method to the Consumer which aggregates the - `HighWaterMarkOffset` values of its child topic/partitions - ([#769](https://github.com/Shopify/sarama/pull/769)). - -Bug Fixes: - - Fixed producing when using timestamps, compression and Kafka 0.10 - ([#759](https://github.com/Shopify/sarama/pull/759)). - - Added missing decoder methods to DescribeGroups response - ([#756](https://github.com/Shopify/sarama/pull/756)). - - Fix producer shutdown when `Return.Errors` is disabled - ([#787](https://github.com/Shopify/sarama/pull/787)). - - Don't mutate configuration in SyncProducer - ([#790](https://github.com/Shopify/sarama/pull/790)). - - Fix crash on SASL initialization failure - ([#795](https://github.com/Shopify/sarama/pull/795)). - -#### Version 1.10.1 (2016-08-30) - -Bug Fixes: - - Fix the documentation for `HashPartitioner` which was incorrect - ([#717](https://github.com/Shopify/sarama/pull/717)). - - Permit client creation even when it is limited by ACLs - ([#722](https://github.com/Shopify/sarama/pull/722)). - - Several fixes to the consumer timer optimization code, regressions introduced - in v1.10.0. Go's timers are finicky - ([#730](https://github.com/Shopify/sarama/pull/730), - [#733](https://github.com/Shopify/sarama/pull/733), - [#734](https://github.com/Shopify/sarama/pull/734)). - - Handle consuming compressed relative offsets with Kafka 0.10 - ([#735](https://github.com/Shopify/sarama/pull/735)). - -#### Version 1.10.0 (2016-08-02) - -_Important:_ As of Sarama 1.10 it is necessary to tell Sarama the version of -Kafka you are running against (via the `config.Version` value) in order to use -features that may not be compatible with old Kafka versions. If you don't -specify this value it will default to 0.8.2 (the minimum supported), and trying -to use more recent features (like the offset manager) will fail with an error. - -_Also:_ The offset-manager's behaviour has been changed to match the upstream -java consumer (see [#705](https://github.com/Shopify/sarama/pull/705) and -[#713](https://github.com/Shopify/sarama/pull/713)). If you use the -offset-manager, please ensure that you are committing one *greater* than the -last consumed message offset or else you may end up consuming duplicate -messages. - -New Features: - - Support for Kafka 0.10 - ([#672](https://github.com/Shopify/sarama/pull/672), - [#678](https://github.com/Shopify/sarama/pull/678), - [#681](https://github.com/Shopify/sarama/pull/681), and others). - - Support for configuring the target Kafka version - ([#676](https://github.com/Shopify/sarama/pull/676)). - - Batch producing support in the SyncProducer - ([#677](https://github.com/Shopify/sarama/pull/677)). - - Extend producer mock to allow setting expectations on message contents - ([#667](https://github.com/Shopify/sarama/pull/667)). - -Improvements: - - Support `nil` compressed messages for deleting in compacted topics - ([#634](https://github.com/Shopify/sarama/pull/634)). - - Pre-allocate decoding errors, greatly reducing heap usage and GC time against - misbehaving brokers ([#690](https://github.com/Shopify/sarama/pull/690)). - - Re-use consumer expiry timers, removing one allocation per consumed message - ([#707](https://github.com/Shopify/sarama/pull/707)). - -Bug Fixes: - - Actually default the client ID to "sarama" like we say we do - ([#664](https://github.com/Shopify/sarama/pull/664)). - - Fix a rare issue where `Client.Leader` could return the wrong error - ([#685](https://github.com/Shopify/sarama/pull/685)). - - Fix a possible tight loop in the consumer - ([#693](https://github.com/Shopify/sarama/pull/693)). - - Match upstream's offset-tracking behaviour - ([#705](https://github.com/Shopify/sarama/pull/705)). - - Report UnknownTopicOrPartition errors from the offset manager - ([#706](https://github.com/Shopify/sarama/pull/706)). - - Fix possible negative partition value from the HashPartitioner - ([#709](https://github.com/Shopify/sarama/pull/709)). - -#### Version 1.9.0 (2016-05-16) - -New Features: - - Add support for custom offset manager retention durations - ([#602](https://github.com/Shopify/sarama/pull/602)). - - Publish low-level mocks to enable testing of third-party producer/consumer - implementations ([#570](https://github.com/Shopify/sarama/pull/570)). - - Declare support for Golang 1.6 - ([#611](https://github.com/Shopify/sarama/pull/611)). - - Support for SASL plain-text auth - ([#648](https://github.com/Shopify/sarama/pull/648)). - -Improvements: - - Simplified broker locking scheme slightly - ([#604](https://github.com/Shopify/sarama/pull/604)). - - Documentation cleanup - ([#605](https://github.com/Shopify/sarama/pull/605), - [#621](https://github.com/Shopify/sarama/pull/621), - [#654](https://github.com/Shopify/sarama/pull/654)). - -Bug Fixes: - - Fix race condition shutting down the OffsetManager - ([#658](https://github.com/Shopify/sarama/pull/658)). - -#### Version 1.8.0 (2016-02-01) - -New Features: - - Full support for Kafka 0.9: - - All protocol messages and fields - ([#586](https://github.com/Shopify/sarama/pull/586), - [#588](https://github.com/Shopify/sarama/pull/588), - [#590](https://github.com/Shopify/sarama/pull/590)). - - Verified that TLS support works - ([#581](https://github.com/Shopify/sarama/pull/581)). - - Fixed the OffsetManager compatibility - ([#585](https://github.com/Shopify/sarama/pull/585)). - -Improvements: - - Optimize for fewer system calls when reading from the network - ([#584](https://github.com/Shopify/sarama/pull/584)). - - Automatically retry `InvalidMessage` errors to match upstream behaviour - ([#589](https://github.com/Shopify/sarama/pull/589)). - -#### Version 1.7.0 (2015-12-11) - -New Features: - - Preliminary support for Kafka 0.9 - ([#572](https://github.com/Shopify/sarama/pull/572)). This comes with several - caveats: - - Protocol-layer support is mostly in place - ([#577](https://github.com/Shopify/sarama/pull/577)), however Kafka 0.9 - renamed some messages and fields, which we did not in order to preserve API - compatibility. - - The producer and consumer work against 0.9, but the offset manager does - not ([#573](https://github.com/Shopify/sarama/pull/573)). - - TLS support may or may not work - ([#581](https://github.com/Shopify/sarama/pull/581)). - -Improvements: - - Don't wait for request timeouts on dead brokers, greatly speeding recovery - when the TCP connection is left hanging - ([#548](https://github.com/Shopify/sarama/pull/548)). - - Refactored part of the producer. The new version provides a much more elegant - solution to [#449](https://github.com/Shopify/sarama/pull/449). It is also - slightly more efficient, and much more precise in calculating batch sizes - when compression is used - ([#549](https://github.com/Shopify/sarama/pull/549), - [#550](https://github.com/Shopify/sarama/pull/550), - [#551](https://github.com/Shopify/sarama/pull/551)). - -Bug Fixes: - - Fix race condition in consumer test mock - ([#553](https://github.com/Shopify/sarama/pull/553)). - -#### Version 1.6.1 (2015-09-25) - -Bug Fixes: - - Fix panic that could occur if a user-supplied message value failed to encode - ([#449](https://github.com/Shopify/sarama/pull/449)). - -#### Version 1.6.0 (2015-09-04) - -New Features: - - Implementation of a consumer offset manager using the APIs introduced in - Kafka 0.8.2. The API is designed mainly for integration into a future - high-level consumer, not for direct use, although it is *possible* to use it - directly. - ([#461](https://github.com/Shopify/sarama/pull/461)). - -Improvements: - - CRC32 calculation is much faster on machines with SSE4.2 instructions, - removing a major hotspot from most profiles - ([#255](https://github.com/Shopify/sarama/pull/255)). - -Bug Fixes: - - Make protocol decoding more robust against some malformed packets generated - by go-fuzz ([#523](https://github.com/Shopify/sarama/pull/523), - [#525](https://github.com/Shopify/sarama/pull/525)) or found in other ways - ([#528](https://github.com/Shopify/sarama/pull/528)). - - Fix a potential race condition panic in the consumer on shutdown - ([#529](https://github.com/Shopify/sarama/pull/529)). - -#### Version 1.5.0 (2015-08-17) - -New Features: - - TLS-encrypted network connections are now supported. This feature is subject - to change when Kafka releases built-in TLS support, but for now this is - enough to work with TLS-terminating proxies - ([#154](https://github.com/Shopify/sarama/pull/154)). - -Improvements: - - The consumer will not block if a single partition is not drained by the user; - all other partitions will continue to consume normally - ([#485](https://github.com/Shopify/sarama/pull/485)). - - Formatting of error strings has been much improved - ([#495](https://github.com/Shopify/sarama/pull/495)). - - Internal refactoring of the producer for code cleanliness and to enable - future work ([#300](https://github.com/Shopify/sarama/pull/300)). - -Bug Fixes: - - Fix a potential deadlock in the consumer on shutdown - ([#475](https://github.com/Shopify/sarama/pull/475)). - -#### Version 1.4.3 (2015-07-21) - -Bug Fixes: - - Don't include the partitioner in the producer's "fetch partitions" - circuit-breaker ([#466](https://github.com/Shopify/sarama/pull/466)). - - Don't retry messages until the broker is closed when abandoning a broker in - the producer ([#468](https://github.com/Shopify/sarama/pull/468)). - - Update the import path for snappy-go, it has moved again and the API has - changed slightly ([#486](https://github.com/Shopify/sarama/pull/486)). - -#### Version 1.4.2 (2015-05-27) - -Bug Fixes: - - Update the import path for snappy-go, it has moved from google code to github - ([#456](https://github.com/Shopify/sarama/pull/456)). - -#### Version 1.4.1 (2015-05-25) - -Improvements: - - Optimizations when decoding snappy messages, thanks to John Potocny - ([#446](https://github.com/Shopify/sarama/pull/446)). - -Bug Fixes: - - Fix hypothetical race conditions on producer shutdown - ([#450](https://github.com/Shopify/sarama/pull/450), - [#451](https://github.com/Shopify/sarama/pull/451)). - -#### Version 1.4.0 (2015-05-01) - -New Features: - - The consumer now implements `Topics()` and `Partitions()` methods to enable - users to dynamically choose what topics/partitions to consume without - instantiating a full client - ([#431](https://github.com/Shopify/sarama/pull/431)). - - The partition-consumer now exposes the high water mark offset value returned - by the broker via the `HighWaterMarkOffset()` method ([#339](https://github.com/Shopify/sarama/pull/339)). - - Added a `kafka-console-consumer` tool capable of handling multiple - partitions, and deprecated the now-obsolete `kafka-console-partitionConsumer` - ([#439](https://github.com/Shopify/sarama/pull/439), - [#442](https://github.com/Shopify/sarama/pull/442)). - -Improvements: - - The producer's logging during retry scenarios is more consistent, more - useful, and slightly less verbose - ([#429](https://github.com/Shopify/sarama/pull/429)). - - The client now shuffles its initial list of seed brokers in order to prevent - thundering herd on the first broker in the list - ([#441](https://github.com/Shopify/sarama/pull/441)). - -Bug Fixes: - - The producer now correctly manages its state if retries occur when it is - shutting down, fixing several instances of confusing behaviour and at least - one potential deadlock ([#419](https://github.com/Shopify/sarama/pull/419)). - - The consumer now handles messages for different partitions asynchronously, - making it much more resilient to specific user code ordering - ([#325](https://github.com/Shopify/sarama/pull/325)). - -#### Version 1.3.0 (2015-04-16) - -New Features: - - The client now tracks consumer group coordinators using - ConsumerMetadataRequests similar to how it tracks partition leadership using - regular MetadataRequests ([#411](https://github.com/Shopify/sarama/pull/411)). - This adds two methods to the client API: - - `Coordinator(consumerGroup string) (*Broker, error)` - - `RefreshCoordinator(consumerGroup string) error` - -Improvements: - - ConsumerMetadataResponses now automatically create a Broker object out of the - ID/address/port combination for the Coordinator; accessing the fields - individually has been deprecated - ([#413](https://github.com/Shopify/sarama/pull/413)). - - Much improved handling of `OffsetOutOfRange` errors in the consumer. - Consumers will fail to start if the provided offset is out of range - ([#418](https://github.com/Shopify/sarama/pull/418)) - and they will automatically shut down if the offset falls out of range - ([#424](https://github.com/Shopify/sarama/pull/424)). - - Small performance improvement in encoding and decoding protocol messages - ([#427](https://github.com/Shopify/sarama/pull/427)). - -Bug Fixes: - - Fix a rare race condition in the client's background metadata refresher if - it happens to be activated while the client is being closed - ([#422](https://github.com/Shopify/sarama/pull/422)). - -#### Version 1.2.0 (2015-04-07) - -Improvements: - - The producer's behaviour when `Flush.Frequency` is set is now more intuitive - ([#389](https://github.com/Shopify/sarama/pull/389)). - - The producer is now somewhat more memory-efficient during and after retrying - messages due to an improved queue implementation - ([#396](https://github.com/Shopify/sarama/pull/396)). - - The consumer produces much more useful logging output when leadership - changes ([#385](https://github.com/Shopify/sarama/pull/385)). - - The client's `GetOffset` method will now automatically refresh metadata and - retry once in the event of stale information or similar - ([#394](https://github.com/Shopify/sarama/pull/394)). - - Broker connections now have support for using TCP keepalives - ([#407](https://github.com/Shopify/sarama/issues/407)). - -Bug Fixes: - - The OffsetCommitRequest message now correctly implements all three possible - API versions ([#390](https://github.com/Shopify/sarama/pull/390), - [#400](https://github.com/Shopify/sarama/pull/400)). - -#### Version 1.1.0 (2015-03-20) - -Improvements: - - Wrap the producer's partitioner call in a circuit-breaker so that repeatedly - broken topics don't choke throughput - ([#373](https://github.com/Shopify/sarama/pull/373)). - -Bug Fixes: - - Fix the producer's internal reference counting in certain unusual scenarios - ([#367](https://github.com/Shopify/sarama/pull/367)). - - Fix the consumer's internal reference counting in certain unusual scenarios - ([#369](https://github.com/Shopify/sarama/pull/369)). - - Fix a condition where the producer's internal control messages could have - gotten stuck ([#368](https://github.com/Shopify/sarama/pull/368)). - - Fix an issue where invalid partition lists would be cached when asking for - metadata for a non-existant topic ([#372](https://github.com/Shopify/sarama/pull/372)). - - -#### Version 1.0.0 (2015-03-17) - -Version 1.0.0 is the first tagged version, and is almost a complete rewrite. The primary differences with previous untagged versions are: - -- The producer has been rewritten; there is now a `SyncProducer` with a blocking API, and an `AsyncProducer` that is non-blocking. -- The consumer has been rewritten to only open one connection per broker instead of one connection per partition. -- The main types of Sarama are now interfaces to make depedency injection easy; mock implementations for `Consumer`, `SyncProducer` and `AsyncProducer` are provided in the `github.com/Shopify/sarama/mocks` package. -- For most uses cases, it is no longer necessary to open a `Client`; this will be done for you. -- All the configuration values have been unified in the `Config` struct. -- Much improved test suite. diff --git a/vendor/github.com/Shopify/sarama/LICENSE b/vendor/github.com/Shopify/sarama/LICENSE deleted file mode 100644 index d2bf4352..00000000 --- a/vendor/github.com/Shopify/sarama/LICENSE +++ /dev/null @@ -1,20 +0,0 @@ -Copyright (c) 2013 Shopify - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -"Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: - -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/vendor/github.com/Shopify/sarama/Makefile b/vendor/github.com/Shopify/sarama/Makefile deleted file mode 100644 index a0586348..00000000 --- a/vendor/github.com/Shopify/sarama/Makefile +++ /dev/null @@ -1,31 +0,0 @@ -default: fmt get update test lint - -GO := GO111MODULE=on GOPRIVATE=github.com/linkedin GOSUMDB=off go -GOBUILD := CGO_ENABLED=0 $(GO) build $(BUILD_FLAG) -GOTEST := $(GO) test -gcflags='-l' -p 3 -v -race -timeout 6m -coverprofile=profile.out -covermode=atomic - -FILES := $(shell find . -name '*.go' -type f -not -name '*.pb.go' -not -name '*_generated.go' -not -name '*_test.go') -TESTS := $(shell find . -name '*.go' -type f -not -name '*.pb.go' -not -name '*_generated.go' -name '*_test.go') - -get: - $(GO) get ./... - $(GO) mod verify - $(GO) mod tidy - -update: - $(GO) get -u -v all - $(GO) mod verify - $(GO) mod tidy - -fmt: - gofmt -s -l -w $(FILES) $(TESTS) - -lint: - GOFLAGS="-tags=functional" golangci-lint run - -test: - $(GOTEST) ./... - -.PHONY: test_functional -test_functional: - $(GOTEST) -tags=functional ./... diff --git a/vendor/github.com/Shopify/sarama/README.md b/vendor/github.com/Shopify/sarama/README.md deleted file mode 100644 index 38d39695..00000000 --- a/vendor/github.com/Shopify/sarama/README.md +++ /dev/null @@ -1,36 +0,0 @@ -# sarama - -[![GoDoc](https://godoc.org/github.com/Shopify/sarama?status.svg)](https://godoc.org/github.com/Shopify/sarama) -[![Build Status](https://travis-ci.org/Shopify/sarama.svg?branch=master)](https://travis-ci.org/Shopify/sarama) -[![Coverage](https://codecov.io/gh/Shopify/sarama/branch/master/graph/badge.svg)](https://codecov.io/gh/Shopify/sarama) - -Sarama is an MIT-licensed Go client library for [Apache Kafka](https://kafka.apache.org/) version 0.8 (and later). - -## Getting started - -- API documentation and examples are available via [godoc](https://godoc.org/github.com/Shopify/sarama). -- Mocks for testing are available in the [mocks](./mocks) subpackage. -- The [examples](./examples) directory contains more elaborate example applications. -- The [tools](./tools) directory contains command line tools that can be useful for testing, diagnostics, and instrumentation. - -You might also want to look at the [Frequently Asked Questions](https://github.com/Shopify/sarama/wiki/Frequently-Asked-Questions). - -## Compatibility and API stability - -Sarama provides a "2 releases + 2 months" compatibility guarantee: we support -the two latest stable releases of Kafka and Go, and we provide a two month -grace period for older releases. This means we currently officially support -Go 1.13 through 1.14, and Kafka 2.4 through 2.6, although older releases are -still likely to work. - -Sarama follows semantic versioning and provides API stability via the gopkg.in service. -You can import a version with a guaranteed stable API via http://gopkg.in/Shopify/sarama.v1. -A changelog is available [here](CHANGELOG.md). - -## Contributing - -- Get started by checking our [contribution guidelines](https://github.com/Shopify/sarama/blob/master/.github/CONTRIBUTING.md). -- Read the [Sarama wiki](https://github.com/Shopify/sarama/wiki) for more technical and design details. -- The [Kafka Protocol Specification](https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol) contains a wealth of useful information. -- For more general issues, there is [a google group](https://groups.google.com/forum/#!forum/kafka-clients) for Kafka client developers. -- If you have any questions, just ask! diff --git a/vendor/github.com/Shopify/sarama/Vagrantfile b/vendor/github.com/Shopify/sarama/Vagrantfile deleted file mode 100644 index 07d7ffb8..00000000 --- a/vendor/github.com/Shopify/sarama/Vagrantfile +++ /dev/null @@ -1,14 +0,0 @@ -# We have 5 * 192MB ZK processes and 5 * 320MB Kafka processes => 2560MB -MEMORY = 3072 - -Vagrant.configure("2") do |config| - config.vm.box = "ubuntu/bionic64" - - config.vm.provision :shell, path: "vagrant/provision.sh" - - config.vm.network "private_network", ip: "192.168.100.67" - - config.vm.provider "virtualbox" do |v| - v.memory = MEMORY - end -end diff --git a/vendor/github.com/Shopify/sarama/acl_bindings.go b/vendor/github.com/Shopify/sarama/acl_bindings.go deleted file mode 100644 index 50b689d1..00000000 --- a/vendor/github.com/Shopify/sarama/acl_bindings.go +++ /dev/null @@ -1,138 +0,0 @@ -package sarama - -//Resource holds information about acl resource type -type Resource struct { - ResourceType AclResourceType - ResourceName string - ResourcePatternType AclResourcePatternType -} - -func (r *Resource) encode(pe packetEncoder, version int16) error { - pe.putInt8(int8(r.ResourceType)) - - if err := pe.putString(r.ResourceName); err != nil { - return err - } - - if version == 1 { - if r.ResourcePatternType == AclPatternUnknown { - Logger.Print("Cannot encode an unknown resource pattern type, using Literal instead") - r.ResourcePatternType = AclPatternLiteral - } - pe.putInt8(int8(r.ResourcePatternType)) - } - - return nil -} - -func (r *Resource) decode(pd packetDecoder, version int16) (err error) { - resourceType, err := pd.getInt8() - if err != nil { - return err - } - r.ResourceType = AclResourceType(resourceType) - - if r.ResourceName, err = pd.getString(); err != nil { - return err - } - if version == 1 { - pattern, err := pd.getInt8() - if err != nil { - return err - } - r.ResourcePatternType = AclResourcePatternType(pattern) - } - - return nil -} - -//Acl holds information about acl type -type Acl struct { - Principal string - Host string - Operation AclOperation - PermissionType AclPermissionType -} - -func (a *Acl) encode(pe packetEncoder) error { - if err := pe.putString(a.Principal); err != nil { - return err - } - - if err := pe.putString(a.Host); err != nil { - return err - } - - pe.putInt8(int8(a.Operation)) - pe.putInt8(int8(a.PermissionType)) - - return nil -} - -func (a *Acl) decode(pd packetDecoder, version int16) (err error) { - if a.Principal, err = pd.getString(); err != nil { - return err - } - - if a.Host, err = pd.getString(); err != nil { - return err - } - - operation, err := pd.getInt8() - if err != nil { - return err - } - a.Operation = AclOperation(operation) - - permissionType, err := pd.getInt8() - if err != nil { - return err - } - a.PermissionType = AclPermissionType(permissionType) - - return nil -} - -//ResourceAcls is an acl resource type -type ResourceAcls struct { - Resource - Acls []*Acl -} - -func (r *ResourceAcls) encode(pe packetEncoder, version int16) error { - if err := r.Resource.encode(pe, version); err != nil { - return err - } - - if err := pe.putArrayLength(len(r.Acls)); err != nil { - return err - } - for _, acl := range r.Acls { - if err := acl.encode(pe); err != nil { - return err - } - } - - return nil -} - -func (r *ResourceAcls) decode(pd packetDecoder, version int16) error { - if err := r.Resource.decode(pd, version); err != nil { - return err - } - - n, err := pd.getArrayLength() - if err != nil { - return err - } - - r.Acls = make([]*Acl, n) - for i := 0; i < n; i++ { - r.Acls[i] = new(Acl) - if err := r.Acls[i].decode(pd, version); err != nil { - return err - } - } - - return nil -} diff --git a/vendor/github.com/Shopify/sarama/acl_create_request.go b/vendor/github.com/Shopify/sarama/acl_create_request.go deleted file mode 100644 index 6d8a70e1..00000000 --- a/vendor/github.com/Shopify/sarama/acl_create_request.go +++ /dev/null @@ -1,89 +0,0 @@ -package sarama - -//CreateAclsRequest is an acl creation request -type CreateAclsRequest struct { - Version int16 - AclCreations []*AclCreation -} - -func (c *CreateAclsRequest) encode(pe packetEncoder) error { - if err := pe.putArrayLength(len(c.AclCreations)); err != nil { - return err - } - - for _, aclCreation := range c.AclCreations { - if err := aclCreation.encode(pe, c.Version); err != nil { - return err - } - } - - return nil -} - -func (c *CreateAclsRequest) decode(pd packetDecoder, version int16) (err error) { - c.Version = version - n, err := pd.getArrayLength() - if err != nil { - return err - } - - c.AclCreations = make([]*AclCreation, n) - - for i := 0; i < n; i++ { - c.AclCreations[i] = new(AclCreation) - if err := c.AclCreations[i].decode(pd, version); err != nil { - return err - } - } - - return nil -} - -func (c *CreateAclsRequest) key() int16 { - return 30 -} - -func (c *CreateAclsRequest) version() int16 { - return c.Version -} - -func (c *CreateAclsRequest) headerVersion() int16 { - return 1 -} - -func (c *CreateAclsRequest) requiredVersion() KafkaVersion { - switch c.Version { - case 1: - return V2_0_0_0 - default: - return V0_11_0_0 - } -} - -//AclCreation is a wrapper around Resource and Acl type -type AclCreation struct { - Resource - Acl -} - -func (a *AclCreation) encode(pe packetEncoder, version int16) error { - if err := a.Resource.encode(pe, version); err != nil { - return err - } - if err := a.Acl.encode(pe); err != nil { - return err - } - - return nil -} - -func (a *AclCreation) decode(pd packetDecoder, version int16) (err error) { - if err := a.Resource.decode(pd, version); err != nil { - return err - } - if err := a.Acl.decode(pd, version); err != nil { - return err - } - - return nil -} diff --git a/vendor/github.com/Shopify/sarama/acl_create_response.go b/vendor/github.com/Shopify/sarama/acl_create_response.go deleted file mode 100644 index 14b1b9e1..00000000 --- a/vendor/github.com/Shopify/sarama/acl_create_response.go +++ /dev/null @@ -1,94 +0,0 @@ -package sarama - -import "time" - -//CreateAclsResponse is a an acl response creation type -type CreateAclsResponse struct { - ThrottleTime time.Duration - AclCreationResponses []*AclCreationResponse -} - -func (c *CreateAclsResponse) encode(pe packetEncoder) error { - pe.putInt32(int32(c.ThrottleTime / time.Millisecond)) - - if err := pe.putArrayLength(len(c.AclCreationResponses)); err != nil { - return err - } - - for _, aclCreationResponse := range c.AclCreationResponses { - if err := aclCreationResponse.encode(pe); err != nil { - return err - } - } - - return nil -} - -func (c *CreateAclsResponse) decode(pd packetDecoder, version int16) (err error) { - throttleTime, err := pd.getInt32() - if err != nil { - return err - } - c.ThrottleTime = time.Duration(throttleTime) * time.Millisecond - - n, err := pd.getArrayLength() - if err != nil { - return err - } - - c.AclCreationResponses = make([]*AclCreationResponse, n) - for i := 0; i < n; i++ { - c.AclCreationResponses[i] = new(AclCreationResponse) - if err := c.AclCreationResponses[i].decode(pd, version); err != nil { - return err - } - } - - return nil -} - -func (c *CreateAclsResponse) key() int16 { - return 30 -} - -func (c *CreateAclsResponse) version() int16 { - return 0 -} - -func (c *CreateAclsResponse) headerVersion() int16 { - return 0 -} - -func (c *CreateAclsResponse) requiredVersion() KafkaVersion { - return V0_11_0_0 -} - -//AclCreationResponse is an acl creation response type -type AclCreationResponse struct { - Err KError - ErrMsg *string -} - -func (a *AclCreationResponse) encode(pe packetEncoder) error { - pe.putInt16(int16(a.Err)) - - if err := pe.putNullableString(a.ErrMsg); err != nil { - return err - } - - return nil -} - -func (a *AclCreationResponse) decode(pd packetDecoder, version int16) (err error) { - kerr, err := pd.getInt16() - if err != nil { - return err - } - a.Err = KError(kerr) - - if a.ErrMsg, err = pd.getNullableString(); err != nil { - return err - } - - return nil -} diff --git a/vendor/github.com/Shopify/sarama/acl_delete_request.go b/vendor/github.com/Shopify/sarama/acl_delete_request.go deleted file mode 100644 index 41525225..00000000 --- a/vendor/github.com/Shopify/sarama/acl_delete_request.go +++ /dev/null @@ -1,62 +0,0 @@ -package sarama - -//DeleteAclsRequest is a delete acl request -type DeleteAclsRequest struct { - Version int - Filters []*AclFilter -} - -func (d *DeleteAclsRequest) encode(pe packetEncoder) error { - if err := pe.putArrayLength(len(d.Filters)); err != nil { - return err - } - - for _, filter := range d.Filters { - filter.Version = d.Version - if err := filter.encode(pe); err != nil { - return err - } - } - - return nil -} - -func (d *DeleteAclsRequest) decode(pd packetDecoder, version int16) (err error) { - d.Version = int(version) - n, err := pd.getArrayLength() - if err != nil { - return err - } - - d.Filters = make([]*AclFilter, n) - for i := 0; i < n; i++ { - d.Filters[i] = new(AclFilter) - d.Filters[i].Version = int(version) - if err := d.Filters[i].decode(pd, version); err != nil { - return err - } - } - - return nil -} - -func (d *DeleteAclsRequest) key() int16 { - return 31 -} - -func (d *DeleteAclsRequest) version() int16 { - return int16(d.Version) -} - -func (c *DeleteAclsRequest) headerVersion() int16 { - return 1 -} - -func (d *DeleteAclsRequest) requiredVersion() KafkaVersion { - switch d.Version { - case 1: - return V2_0_0_0 - default: - return V0_11_0_0 - } -} diff --git a/vendor/github.com/Shopify/sarama/acl_delete_response.go b/vendor/github.com/Shopify/sarama/acl_delete_response.go deleted file mode 100644 index cb630882..00000000 --- a/vendor/github.com/Shopify/sarama/acl_delete_response.go +++ /dev/null @@ -1,163 +0,0 @@ -package sarama - -import "time" - -//DeleteAclsResponse is a delete acl response -type DeleteAclsResponse struct { - Version int16 - ThrottleTime time.Duration - FilterResponses []*FilterResponse -} - -func (d *DeleteAclsResponse) encode(pe packetEncoder) error { - pe.putInt32(int32(d.ThrottleTime / time.Millisecond)) - - if err := pe.putArrayLength(len(d.FilterResponses)); err != nil { - return err - } - - for _, filterResponse := range d.FilterResponses { - if err := filterResponse.encode(pe, d.Version); err != nil { - return err - } - } - - return nil -} - -func (d *DeleteAclsResponse) decode(pd packetDecoder, version int16) (err error) { - throttleTime, err := pd.getInt32() - if err != nil { - return err - } - d.ThrottleTime = time.Duration(throttleTime) * time.Millisecond - - n, err := pd.getArrayLength() - if err != nil { - return err - } - d.FilterResponses = make([]*FilterResponse, n) - - for i := 0; i < n; i++ { - d.FilterResponses[i] = new(FilterResponse) - if err := d.FilterResponses[i].decode(pd, version); err != nil { - return err - } - } - - return nil -} - -func (d *DeleteAclsResponse) key() int16 { - return 31 -} - -func (d *DeleteAclsResponse) version() int16 { - return d.Version -} - -func (d *DeleteAclsResponse) headerVersion() int16 { - return 0 -} - -func (d *DeleteAclsResponse) requiredVersion() KafkaVersion { - return V0_11_0_0 -} - -//FilterResponse is a filter response type -type FilterResponse struct { - Err KError - ErrMsg *string - MatchingAcls []*MatchingAcl -} - -func (f *FilterResponse) encode(pe packetEncoder, version int16) error { - pe.putInt16(int16(f.Err)) - if err := pe.putNullableString(f.ErrMsg); err != nil { - return err - } - - if err := pe.putArrayLength(len(f.MatchingAcls)); err != nil { - return err - } - for _, matchingAcl := range f.MatchingAcls { - if err := matchingAcl.encode(pe, version); err != nil { - return err - } - } - - return nil -} - -func (f *FilterResponse) decode(pd packetDecoder, version int16) (err error) { - kerr, err := pd.getInt16() - if err != nil { - return err - } - f.Err = KError(kerr) - - if f.ErrMsg, err = pd.getNullableString(); err != nil { - return err - } - - n, err := pd.getArrayLength() - if err != nil { - return err - } - f.MatchingAcls = make([]*MatchingAcl, n) - for i := 0; i < n; i++ { - f.MatchingAcls[i] = new(MatchingAcl) - if err := f.MatchingAcls[i].decode(pd, version); err != nil { - return err - } - } - - return nil -} - -//MatchingAcl is a matching acl type -type MatchingAcl struct { - Err KError - ErrMsg *string - Resource - Acl -} - -func (m *MatchingAcl) encode(pe packetEncoder, version int16) error { - pe.putInt16(int16(m.Err)) - if err := pe.putNullableString(m.ErrMsg); err != nil { - return err - } - - if err := m.Resource.encode(pe, version); err != nil { - return err - } - - if err := m.Acl.encode(pe); err != nil { - return err - } - - return nil -} - -func (m *MatchingAcl) decode(pd packetDecoder, version int16) (err error) { - kerr, err := pd.getInt16() - if err != nil { - return err - } - m.Err = KError(kerr) - - if m.ErrMsg, err = pd.getNullableString(); err != nil { - return err - } - - if err := m.Resource.decode(pd, version); err != nil { - return err - } - - if err := m.Acl.decode(pd, version); err != nil { - return err - } - - return nil -} diff --git a/vendor/github.com/Shopify/sarama/acl_describe_request.go b/vendor/github.com/Shopify/sarama/acl_describe_request.go deleted file mode 100644 index 29841a5c..00000000 --- a/vendor/github.com/Shopify/sarama/acl_describe_request.go +++ /dev/null @@ -1,39 +0,0 @@ -package sarama - -//DescribeAclsRequest is a secribe acl request type -type DescribeAclsRequest struct { - Version int - AclFilter -} - -func (d *DescribeAclsRequest) encode(pe packetEncoder) error { - d.AclFilter.Version = d.Version - return d.AclFilter.encode(pe) -} - -func (d *DescribeAclsRequest) decode(pd packetDecoder, version int16) (err error) { - d.Version = int(version) - d.AclFilter.Version = int(version) - return d.AclFilter.decode(pd, version) -} - -func (d *DescribeAclsRequest) key() int16 { - return 29 -} - -func (d *DescribeAclsRequest) version() int16 { - return int16(d.Version) -} - -func (d *DescribeAclsRequest) headerVersion() int16 { - return 1 -} - -func (d *DescribeAclsRequest) requiredVersion() KafkaVersion { - switch d.Version { - case 1: - return V2_0_0_0 - default: - return V0_11_0_0 - } -} diff --git a/vendor/github.com/Shopify/sarama/acl_describe_response.go b/vendor/github.com/Shopify/sarama/acl_describe_response.go deleted file mode 100644 index c43408b2..00000000 --- a/vendor/github.com/Shopify/sarama/acl_describe_response.go +++ /dev/null @@ -1,91 +0,0 @@ -package sarama - -import "time" - -//DescribeAclsResponse is a describe acl response type -type DescribeAclsResponse struct { - Version int16 - ThrottleTime time.Duration - Err KError - ErrMsg *string - ResourceAcls []*ResourceAcls -} - -func (d *DescribeAclsResponse) encode(pe packetEncoder) error { - pe.putInt32(int32(d.ThrottleTime / time.Millisecond)) - pe.putInt16(int16(d.Err)) - - if err := pe.putNullableString(d.ErrMsg); err != nil { - return err - } - - if err := pe.putArrayLength(len(d.ResourceAcls)); err != nil { - return err - } - - for _, resourceAcl := range d.ResourceAcls { - if err := resourceAcl.encode(pe, d.Version); err != nil { - return err - } - } - - return nil -} - -func (d *DescribeAclsResponse) decode(pd packetDecoder, version int16) (err error) { - throttleTime, err := pd.getInt32() - if err != nil { - return err - } - d.ThrottleTime = time.Duration(throttleTime) * time.Millisecond - - kerr, err := pd.getInt16() - if err != nil { - return err - } - d.Err = KError(kerr) - - errmsg, err := pd.getString() - if err != nil { - return err - } - if errmsg != "" { - d.ErrMsg = &errmsg - } - - n, err := pd.getArrayLength() - if err != nil { - return err - } - d.ResourceAcls = make([]*ResourceAcls, n) - - for i := 0; i < n; i++ { - d.ResourceAcls[i] = new(ResourceAcls) - if err := d.ResourceAcls[i].decode(pd, version); err != nil { - return err - } - } - - return nil -} - -func (d *DescribeAclsResponse) key() int16 { - return 29 -} - -func (d *DescribeAclsResponse) version() int16 { - return d.Version -} - -func (d *DescribeAclsResponse) headerVersion() int16 { - return 0 -} - -func (d *DescribeAclsResponse) requiredVersion() KafkaVersion { - switch d.Version { - case 1: - return V2_0_0_0 - default: - return V0_11_0_0 - } -} diff --git a/vendor/github.com/Shopify/sarama/acl_filter.go b/vendor/github.com/Shopify/sarama/acl_filter.go deleted file mode 100644 index fad55587..00000000 --- a/vendor/github.com/Shopify/sarama/acl_filter.go +++ /dev/null @@ -1,78 +0,0 @@ -package sarama - -type AclFilter struct { - Version int - ResourceType AclResourceType - ResourceName *string - ResourcePatternTypeFilter AclResourcePatternType - Principal *string - Host *string - Operation AclOperation - PermissionType AclPermissionType -} - -func (a *AclFilter) encode(pe packetEncoder) error { - pe.putInt8(int8(a.ResourceType)) - if err := pe.putNullableString(a.ResourceName); err != nil { - return err - } - - if a.Version == 1 { - pe.putInt8(int8(a.ResourcePatternTypeFilter)) - } - - if err := pe.putNullableString(a.Principal); err != nil { - return err - } - if err := pe.putNullableString(a.Host); err != nil { - return err - } - pe.putInt8(int8(a.Operation)) - pe.putInt8(int8(a.PermissionType)) - - return nil -} - -func (a *AclFilter) decode(pd packetDecoder, version int16) (err error) { - resourceType, err := pd.getInt8() - if err != nil { - return err - } - a.ResourceType = AclResourceType(resourceType) - - if a.ResourceName, err = pd.getNullableString(); err != nil { - return err - } - - if a.Version == 1 { - pattern, err := pd.getInt8() - - if err != nil { - return err - } - - a.ResourcePatternTypeFilter = AclResourcePatternType(pattern) - } - - if a.Principal, err = pd.getNullableString(); err != nil { - return err - } - - if a.Host, err = pd.getNullableString(); err != nil { - return err - } - - operation, err := pd.getInt8() - if err != nil { - return err - } - a.Operation = AclOperation(operation) - - permissionType, err := pd.getInt8() - if err != nil { - return err - } - a.PermissionType = AclPermissionType(permissionType) - - return nil -} diff --git a/vendor/github.com/Shopify/sarama/acl_types.go b/vendor/github.com/Shopify/sarama/acl_types.go deleted file mode 100644 index c10ad7b9..00000000 --- a/vendor/github.com/Shopify/sarama/acl_types.go +++ /dev/null @@ -1,55 +0,0 @@ -package sarama - -type ( - AclOperation int - - AclPermissionType int - - AclResourceType int - - AclResourcePatternType int -) - -// ref: https://github.com/apache/kafka/blob/trunk/clients/src/main/java/org/apache/kafka/common/acl/AclOperation.java -const ( - AclOperationUnknown AclOperation = iota - AclOperationAny - AclOperationAll - AclOperationRead - AclOperationWrite - AclOperationCreate - AclOperationDelete - AclOperationAlter - AclOperationDescribe - AclOperationClusterAction - AclOperationDescribeConfigs - AclOperationAlterConfigs - AclOperationIdempotentWrite -) - -// ref: https://github.com/apache/kafka/blob/trunk/clients/src/main/java/org/apache/kafka/common/acl/AclPermissionType.java -const ( - AclPermissionUnknown AclPermissionType = iota - AclPermissionAny - AclPermissionDeny - AclPermissionAllow -) - -// ref: https://github.com/apache/kafka/blob/trunk/clients/src/main/java/org/apache/kafka/common/resource/ResourceType.java -const ( - AclResourceUnknown AclResourceType = iota - AclResourceAny - AclResourceTopic - AclResourceGroup - AclResourceCluster - AclResourceTransactionalID -) - -// ref: https://github.com/apache/kafka/blob/trunk/clients/src/main/java/org/apache/kafka/common/resource/PatternType.java -const ( - AclPatternUnknown AclResourcePatternType = iota - AclPatternAny - AclPatternMatch - AclPatternLiteral - AclPatternPrefixed -) diff --git a/vendor/github.com/Shopify/sarama/add_offsets_to_txn_request.go b/vendor/github.com/Shopify/sarama/add_offsets_to_txn_request.go deleted file mode 100644 index 95586f9a..00000000 --- a/vendor/github.com/Shopify/sarama/add_offsets_to_txn_request.go +++ /dev/null @@ -1,57 +0,0 @@ -package sarama - -//AddOffsetsToTxnRequest adds offsets to a transaction request -type AddOffsetsToTxnRequest struct { - TransactionalID string - ProducerID int64 - ProducerEpoch int16 - GroupID string -} - -func (a *AddOffsetsToTxnRequest) encode(pe packetEncoder) error { - if err := pe.putString(a.TransactionalID); err != nil { - return err - } - - pe.putInt64(a.ProducerID) - - pe.putInt16(a.ProducerEpoch) - - if err := pe.putString(a.GroupID); err != nil { - return err - } - - return nil -} - -func (a *AddOffsetsToTxnRequest) decode(pd packetDecoder, version int16) (err error) { - if a.TransactionalID, err = pd.getString(); err != nil { - return err - } - if a.ProducerID, err = pd.getInt64(); err != nil { - return err - } - if a.ProducerEpoch, err = pd.getInt16(); err != nil { - return err - } - if a.GroupID, err = pd.getString(); err != nil { - return err - } - return nil -} - -func (a *AddOffsetsToTxnRequest) key() int16 { - return 25 -} - -func (a *AddOffsetsToTxnRequest) version() int16 { - return 0 -} - -func (a *AddOffsetsToTxnRequest) headerVersion() int16 { - return 1 -} - -func (a *AddOffsetsToTxnRequest) requiredVersion() KafkaVersion { - return V0_11_0_0 -} diff --git a/vendor/github.com/Shopify/sarama/add_offsets_to_txn_response.go b/vendor/github.com/Shopify/sarama/add_offsets_to_txn_response.go deleted file mode 100644 index bdb18441..00000000 --- a/vendor/github.com/Shopify/sarama/add_offsets_to_txn_response.go +++ /dev/null @@ -1,49 +0,0 @@ -package sarama - -import ( - "time" -) - -//AddOffsetsToTxnResponse is a response type for adding offsets to txns -type AddOffsetsToTxnResponse struct { - ThrottleTime time.Duration - Err KError -} - -func (a *AddOffsetsToTxnResponse) encode(pe packetEncoder) error { - pe.putInt32(int32(a.ThrottleTime / time.Millisecond)) - pe.putInt16(int16(a.Err)) - return nil -} - -func (a *AddOffsetsToTxnResponse) decode(pd packetDecoder, version int16) (err error) { - throttleTime, err := pd.getInt32() - if err != nil { - return err - } - a.ThrottleTime = time.Duration(throttleTime) * time.Millisecond - - kerr, err := pd.getInt16() - if err != nil { - return err - } - a.Err = KError(kerr) - - return nil -} - -func (a *AddOffsetsToTxnResponse) key() int16 { - return 25 -} - -func (a *AddOffsetsToTxnResponse) version() int16 { - return 0 -} - -func (a *AddOffsetsToTxnResponse) headerVersion() int16 { - return 0 -} - -func (a *AddOffsetsToTxnResponse) requiredVersion() KafkaVersion { - return V0_11_0_0 -} diff --git a/vendor/github.com/Shopify/sarama/add_partitions_to_txn_request.go b/vendor/github.com/Shopify/sarama/add_partitions_to_txn_request.go deleted file mode 100644 index 6289f451..00000000 --- a/vendor/github.com/Shopify/sarama/add_partitions_to_txn_request.go +++ /dev/null @@ -1,81 +0,0 @@ -package sarama - -//AddPartitionsToTxnRequest is a add paartition request -type AddPartitionsToTxnRequest struct { - TransactionalID string - ProducerID int64 - ProducerEpoch int16 - TopicPartitions map[string][]int32 -} - -func (a *AddPartitionsToTxnRequest) encode(pe packetEncoder) error { - if err := pe.putString(a.TransactionalID); err != nil { - return err - } - pe.putInt64(a.ProducerID) - pe.putInt16(a.ProducerEpoch) - - if err := pe.putArrayLength(len(a.TopicPartitions)); err != nil { - return err - } - for topic, partitions := range a.TopicPartitions { - if err := pe.putString(topic); err != nil { - return err - } - if err := pe.putInt32Array(partitions); err != nil { - return err - } - } - - return nil -} - -func (a *AddPartitionsToTxnRequest) decode(pd packetDecoder, version int16) (err error) { - if a.TransactionalID, err = pd.getString(); err != nil { - return err - } - if a.ProducerID, err = pd.getInt64(); err != nil { - return err - } - if a.ProducerEpoch, err = pd.getInt16(); err != nil { - return err - } - - n, err := pd.getArrayLength() - if err != nil { - return err - } - - a.TopicPartitions = make(map[string][]int32) - for i := 0; i < n; i++ { - topic, err := pd.getString() - if err != nil { - return err - } - - partitions, err := pd.getInt32Array() - if err != nil { - return err - } - - a.TopicPartitions[topic] = partitions - } - - return nil -} - -func (a *AddPartitionsToTxnRequest) key() int16 { - return 24 -} - -func (a *AddPartitionsToTxnRequest) version() int16 { - return 0 -} - -func (a *AddPartitionsToTxnRequest) headerVersion() int16 { - return 1 -} - -func (a *AddPartitionsToTxnRequest) requiredVersion() KafkaVersion { - return V0_11_0_0 -} diff --git a/vendor/github.com/Shopify/sarama/add_partitions_to_txn_response.go b/vendor/github.com/Shopify/sarama/add_partitions_to_txn_response.go deleted file mode 100644 index 73b73b07..00000000 --- a/vendor/github.com/Shopify/sarama/add_partitions_to_txn_response.go +++ /dev/null @@ -1,114 +0,0 @@ -package sarama - -import ( - "time" -) - -//AddPartitionsToTxnResponse is a partition errors to transaction type -type AddPartitionsToTxnResponse struct { - ThrottleTime time.Duration - Errors map[string][]*PartitionError -} - -func (a *AddPartitionsToTxnResponse) encode(pe packetEncoder) error { - pe.putInt32(int32(a.ThrottleTime / time.Millisecond)) - if err := pe.putArrayLength(len(a.Errors)); err != nil { - return err - } - - for topic, e := range a.Errors { - if err := pe.putString(topic); err != nil { - return err - } - if err := pe.putArrayLength(len(e)); err != nil { - return err - } - for _, partitionError := range e { - if err := partitionError.encode(pe); err != nil { - return err - } - } - } - - return nil -} - -func (a *AddPartitionsToTxnResponse) decode(pd packetDecoder, version int16) (err error) { - throttleTime, err := pd.getInt32() - if err != nil { - return err - } - a.ThrottleTime = time.Duration(throttleTime) * time.Millisecond - - n, err := pd.getArrayLength() - if err != nil { - return err - } - - a.Errors = make(map[string][]*PartitionError) - - for i := 0; i < n; i++ { - topic, err := pd.getString() - if err != nil { - return err - } - - m, err := pd.getArrayLength() - if err != nil { - return err - } - - a.Errors[topic] = make([]*PartitionError, m) - - for j := 0; j < m; j++ { - a.Errors[topic][j] = new(PartitionError) - if err := a.Errors[topic][j].decode(pd, version); err != nil { - return err - } - } - } - - return nil -} - -func (a *AddPartitionsToTxnResponse) key() int16 { - return 24 -} - -func (a *AddPartitionsToTxnResponse) version() int16 { - return 0 -} - -func (a *AddPartitionsToTxnResponse) headerVersion() int16 { - return 0 -} - -func (a *AddPartitionsToTxnResponse) requiredVersion() KafkaVersion { - return V0_11_0_0 -} - -//PartitionError is a partition error type -type PartitionError struct { - Partition int32 - Err KError -} - -func (p *PartitionError) encode(pe packetEncoder) error { - pe.putInt32(p.Partition) - pe.putInt16(int16(p.Err)) - return nil -} - -func (p *PartitionError) decode(pd packetDecoder, version int16) (err error) { - if p.Partition, err = pd.getInt32(); err != nil { - return err - } - - kerr, err := pd.getInt16() - if err != nil { - return err - } - p.Err = KError(kerr) - - return nil -} diff --git a/vendor/github.com/Shopify/sarama/admin.go b/vendor/github.com/Shopify/sarama/admin.go deleted file mode 100644 index 9dea0255..00000000 --- a/vendor/github.com/Shopify/sarama/admin.go +++ /dev/null @@ -1,934 +0,0 @@ -package sarama - -import ( - "errors" - "fmt" - "math/rand" - "strconv" - "sync" - "time" -) - -// ClusterAdmin is the administrative client for Kafka, which supports managing and inspecting topics, -// brokers, configurations and ACLs. The minimum broker version required is 0.10.0.0. -// Methods with stricter requirements will specify the minimum broker version required. -// You MUST call Close() on a client to avoid leaks -type ClusterAdmin interface { - // Creates a new topic. This operation is supported by brokers with version 0.10.1.0 or higher. - // It may take several seconds after CreateTopic returns success for all the brokers - // to become aware that the topic has been created. During this time, listTopics - // may not return information about the new topic.The validateOnly option is supported from version 0.10.2.0. - CreateTopic(topic string, detail *TopicDetail, validateOnly bool) error - - // List the topics available in the cluster with the default options. - ListTopics() (map[string]TopicDetail, error) - - // Describe some topics in the cluster. - DescribeTopics(topics []string) (metadata []*TopicMetadata, err error) - - // Delete a topic. It may take several seconds after the DeleteTopic to returns success - // and for all the brokers to become aware that the topics are gone. - // During this time, listTopics may continue to return information about the deleted topic. - // If delete.topic.enable is false on the brokers, deleteTopic will mark - // the topic for deletion, but not actually delete them. - // This operation is supported by brokers with version 0.10.1.0 or higher. - DeleteTopic(topic string) error - - // Increase the number of partitions of the topics according to the corresponding values. - // If partitions are increased for a topic that has a key, the partition logic or ordering of - // the messages will be affected. It may take several seconds after this method returns - // success for all the brokers to become aware that the partitions have been created. - // During this time, ClusterAdmin#describeTopics may not return information about the - // new partitions. This operation is supported by brokers with version 1.0.0 or higher. - CreatePartitions(topic string, count int32, assignment [][]int32, validateOnly bool) error - - // Alter the replica assignment for partitions. - // This operation is supported by brokers with version 2.4.0.0 or higher. - AlterPartitionReassignments(topic string, assignment [][]int32) error - - // Provides info on ongoing partitions replica reassignments. - // This operation is supported by brokers with version 2.4.0.0 or higher. - ListPartitionReassignments(topics string, partitions []int32) (topicStatus map[string]map[int32]*PartitionReplicaReassignmentsStatus, err error) - - // Delete records whose offset is smaller than the given offset of the corresponding partition. - // This operation is supported by brokers with version 0.11.0.0 or higher. - DeleteRecords(topic string, partitionOffsets map[int32]int64) error - - // Get the configuration for the specified resources. - // The returned configuration includes default values and the Default is true - // can be used to distinguish them from user supplied values. - // Config entries where ReadOnly is true cannot be updated. - // The value of config entries where Sensitive is true is always nil so - // sensitive information is not disclosed. - // This operation is supported by brokers with version 0.11.0.0 or higher. - DescribeConfig(resource ConfigResource) ([]ConfigEntry, error) - - // Update the configuration for the specified resources with the default options. - // This operation is supported by brokers with version 0.11.0.0 or higher. - // The resources with their configs (topic is the only resource type with configs - // that can be updated currently Updates are not transactional so they may succeed - // for some resources while fail for others. The configs for a particular resource are updated automatically. - AlterConfig(resourceType ConfigResourceType, name string, entries map[string]*string, validateOnly bool) error - - // Creates access control lists (ACLs) which are bound to specific resources. - // This operation is not transactional so it may succeed for some ACLs while fail for others. - // If you attempt to add an ACL that duplicates an existing ACL, no error will be raised, but - // no changes will be made. This operation is supported by brokers with version 0.11.0.0 or higher. - CreateACL(resource Resource, acl Acl) error - - // Lists access control lists (ACLs) according to the supplied filter. - // it may take some time for changes made by createAcls or deleteAcls to be reflected in the output of ListAcls - // This operation is supported by brokers with version 0.11.0.0 or higher. - ListAcls(filter AclFilter) ([]ResourceAcls, error) - - // Deletes access control lists (ACLs) according to the supplied filters. - // This operation is not transactional so it may succeed for some ACLs while fail for others. - // This operation is supported by brokers with version 0.11.0.0 or higher. - DeleteACL(filter AclFilter, validateOnly bool) ([]MatchingAcl, error) - - // List the consumer groups available in the cluster. - ListConsumerGroups() (map[string]string, error) - - // Describe the given consumer groups. - DescribeConsumerGroups(groups []string) ([]*GroupDescription, error) - - // List the consumer group offsets available in the cluster. - ListConsumerGroupOffsets(group string, topicPartitions map[string][]int32) (*OffsetFetchResponse, error) - - // Delete a consumer group. - DeleteConsumerGroup(group string) error - - // Get information about the nodes in the cluster - DescribeCluster() (brokers []*Broker, controllerID int32, err error) - - // Get information about all log directories on the given set of brokers - DescribeLogDirs(brokers []int32) (map[int32][]DescribeLogDirsResponseDirMetadata, error) - - // Close shuts down the admin and closes underlying client. - Close() error -} - -type clusterAdmin struct { - client Client - conf *Config -} - -// NewClusterAdmin creates a new ClusterAdmin using the given broker addresses and configuration. -func NewClusterAdmin(addrs []string, conf *Config) (ClusterAdmin, error) { - client, err := NewClient(addrs, conf) - if err != nil { - return nil, err - } - return NewClusterAdminFromClient(client) -} - -// NewClusterAdminFromClient creates a new ClusterAdmin using the given client. -// Note that underlying client will also be closed on admin's Close() call. -func NewClusterAdminFromClient(client Client) (ClusterAdmin, error) { - //make sure we can retrieve the controller - _, err := client.Controller() - if err != nil { - return nil, err - } - - ca := &clusterAdmin{ - client: client, - conf: client.Config(), - } - return ca, nil -} - -func (ca *clusterAdmin) Close() error { - return ca.client.Close() -} - -func (ca *clusterAdmin) Controller() (*Broker, error) { - return ca.client.Controller() -} - -func (ca *clusterAdmin) refreshController() (*Broker, error) { - return ca.client.RefreshController() -} - -// isErrNoController returns `true` if the given error type unwraps to an -// `ErrNotController` response from Kafka -func isErrNoController(err error) bool { - switch e := err.(type) { - case *TopicError: - return e.Err == ErrNotController - case *TopicPartitionError: - return e.Err == ErrNotController - case KError: - return e == ErrNotController - } - return false -} - -// retryOnError will repeatedly call the given (error-returning) func in the -// case that its response is non-nil and retriable (as determined by the -// provided retriable func) up to the maximum number of tries permitted by -// the admin client configuration -func (ca *clusterAdmin) retryOnError(retriable func(error) bool, fn func() error) error { - var err error - for attempt := 0; attempt < ca.conf.Admin.Retry.Max; attempt++ { - err = fn() - if err == nil || !retriable(err) { - return err - } - Logger.Printf( - "admin/request retrying after %dms... (%d attempts remaining)\n", - ca.conf.Admin.Retry.Backoff/time.Millisecond, ca.conf.Admin.Retry.Max-attempt) - time.Sleep(ca.conf.Admin.Retry.Backoff) - continue - } - return err -} - -func (ca *clusterAdmin) CreateTopic(topic string, detail *TopicDetail, validateOnly bool) error { - if topic == "" { - return ErrInvalidTopic - } - - if detail == nil { - return errors.New("you must specify topic details") - } - - topicDetails := make(map[string]*TopicDetail) - topicDetails[topic] = detail - - request := &CreateTopicsRequest{ - TopicDetails: topicDetails, - ValidateOnly: validateOnly, - Timeout: ca.conf.Admin.Timeout, - } - - if ca.conf.Version.IsAtLeast(V0_11_0_0) { - request.Version = 1 - } - if ca.conf.Version.IsAtLeast(V1_0_0_0) { - request.Version = 2 - } - - return ca.retryOnError(isErrNoController, func() error { - b, err := ca.Controller() - if err != nil { - return err - } - - rsp, err := b.CreateTopics(request) - if err != nil { - return err - } - - topicErr, ok := rsp.TopicErrors[topic] - if !ok { - return ErrIncompleteResponse - } - - if topicErr.Err != ErrNoError { - if topicErr.Err == ErrNotController { - _, _ = ca.refreshController() - } - return topicErr - } - - return nil - }) -} - -func (ca *clusterAdmin) DescribeTopics(topics []string) (metadata []*TopicMetadata, err error) { - controller, err := ca.Controller() - if err != nil { - return nil, err - } - - request := &MetadataRequest{ - Topics: topics, - AllowAutoTopicCreation: false, - } - - if ca.conf.Version.IsAtLeast(V1_0_0_0) { - request.Version = 5 - } else if ca.conf.Version.IsAtLeast(V0_11_0_0) { - request.Version = 4 - } - - response, err := controller.GetMetadata(request) - if err != nil { - return nil, err - } - return response.Topics, nil -} - -func (ca *clusterAdmin) DescribeCluster() (brokers []*Broker, controllerID int32, err error) { - controller, err := ca.Controller() - if err != nil { - return nil, int32(0), err - } - - request := &MetadataRequest{ - Topics: []string{}, - } - - if ca.conf.Version.IsAtLeast(V0_10_0_0) { - request.Version = 1 - } - - response, err := controller.GetMetadata(request) - if err != nil { - return nil, int32(0), err - } - - return response.Brokers, response.ControllerID, nil -} - -func (ca *clusterAdmin) findBroker(id int32) (*Broker, error) { - brokers := ca.client.Brokers() - for _, b := range brokers { - if b.ID() == id { - return b, nil - } - } - return nil, fmt.Errorf("could not find broker id %d", id) -} - -func (ca *clusterAdmin) findAnyBroker() (*Broker, error) { - brokers := ca.client.Brokers() - if len(brokers) > 0 { - index := rand.Intn(len(brokers)) - return brokers[index], nil - } - return nil, errors.New("no available broker") -} - -func (ca *clusterAdmin) ListTopics() (map[string]TopicDetail, error) { - // In order to build TopicDetails we need to first get the list of all - // topics using a MetadataRequest and then get their configs using a - // DescribeConfigsRequest request. To avoid sending many requests to the - // broker, we use a single DescribeConfigsRequest. - - // Send the all-topic MetadataRequest - b, err := ca.findAnyBroker() - if err != nil { - return nil, err - } - _ = b.Open(ca.client.Config()) - - metadataReq := &MetadataRequest{} - metadataResp, err := b.GetMetadata(metadataReq) - if err != nil { - return nil, err - } - - topicsDetailsMap := make(map[string]TopicDetail) - - var describeConfigsResources []*ConfigResource - - for _, topic := range metadataResp.Topics { - topicDetails := TopicDetail{ - NumPartitions: int32(len(topic.Partitions)), - } - if len(topic.Partitions) > 0 { - topicDetails.ReplicaAssignment = map[int32][]int32{} - for _, partition := range topic.Partitions { - topicDetails.ReplicaAssignment[partition.ID] = partition.Replicas - } - topicDetails.ReplicationFactor = int16(len(topic.Partitions[0].Replicas)) - } - topicsDetailsMap[topic.Name] = topicDetails - - // we populate the resources we want to describe from the MetadataResponse - topicResource := ConfigResource{ - Type: TopicResource, - Name: topic.Name, - } - describeConfigsResources = append(describeConfigsResources, &topicResource) - } - - // Send the DescribeConfigsRequest - describeConfigsReq := &DescribeConfigsRequest{ - Resources: describeConfigsResources, - } - - if ca.conf.Version.IsAtLeast(V1_1_0_0) { - describeConfigsReq.Version = 1 - } - - if ca.conf.Version.IsAtLeast(V2_0_0_0) { - describeConfigsReq.Version = 2 - } - - describeConfigsResp, err := b.DescribeConfigs(describeConfigsReq) - if err != nil { - return nil, err - } - - for _, resource := range describeConfigsResp.Resources { - topicDetails := topicsDetailsMap[resource.Name] - topicDetails.ConfigEntries = make(map[string]*string) - - for _, entry := range resource.Configs { - // only include non-default non-sensitive config - // (don't actually think topic config will ever be sensitive) - if entry.Default || entry.Sensitive { - continue - } - topicDetails.ConfigEntries[entry.Name] = &entry.Value - } - - topicsDetailsMap[resource.Name] = topicDetails - } - - return topicsDetailsMap, nil -} - -func (ca *clusterAdmin) DeleteTopic(topic string) error { - if topic == "" { - return ErrInvalidTopic - } - - request := &DeleteTopicsRequest{ - Topics: []string{topic}, - Timeout: ca.conf.Admin.Timeout, - } - - if ca.conf.Version.IsAtLeast(V0_11_0_0) { - request.Version = 1 - } - - return ca.retryOnError(isErrNoController, func() error { - b, err := ca.Controller() - if err != nil { - return err - } - - rsp, err := b.DeleteTopics(request) - if err != nil { - return err - } - - topicErr, ok := rsp.TopicErrorCodes[topic] - if !ok { - return ErrIncompleteResponse - } - - if topicErr != ErrNoError { - if topicErr == ErrNotController { - _, _ = ca.refreshController() - } - return topicErr - } - - return nil - }) -} - -func (ca *clusterAdmin) CreatePartitions(topic string, count int32, assignment [][]int32, validateOnly bool) error { - if topic == "" { - return ErrInvalidTopic - } - - topicPartitions := make(map[string]*TopicPartition) - topicPartitions[topic] = &TopicPartition{Count: count, Assignment: assignment} - - request := &CreatePartitionsRequest{ - TopicPartitions: topicPartitions, - Timeout: ca.conf.Admin.Timeout, - } - - return ca.retryOnError(isErrNoController, func() error { - b, err := ca.Controller() - if err != nil { - return err - } - - rsp, err := b.CreatePartitions(request) - if err != nil { - return err - } - - topicErr, ok := rsp.TopicPartitionErrors[topic] - if !ok { - return ErrIncompleteResponse - } - - if topicErr.Err != ErrNoError { - if topicErr.Err == ErrNotController { - _, _ = ca.refreshController() - } - return topicErr - } - - return nil - }) -} - -func (ca *clusterAdmin) AlterPartitionReassignments(topic string, assignment [][]int32) error { - if topic == "" { - return ErrInvalidTopic - } - - request := &AlterPartitionReassignmentsRequest{ - TimeoutMs: int32(60000), - Version: int16(0), - } - - for i := 0; i < len(assignment); i++ { - request.AddBlock(topic, int32(i), assignment[i]) - } - - return ca.retryOnError(isErrNoController, func() error { - b, err := ca.Controller() - if err != nil { - return err - } - - errs := make([]error, 0) - - rsp, err := b.AlterPartitionReassignments(request) - - if err != nil { - errs = append(errs, err) - } else { - if rsp.ErrorCode > 0 { - errs = append(errs, errors.New(rsp.ErrorCode.Error())) - } - - for topic, topicErrors := range rsp.Errors { - for partition, partitionError := range topicErrors { - if partitionError.errorCode != ErrNoError { - errStr := fmt.Sprintf("[%s-%d]: %s", topic, partition, partitionError.errorCode.Error()) - errs = append(errs, errors.New(errStr)) - } - } - } - } - - if len(errs) > 0 { - return ErrReassignPartitions{MultiError{&errs}} - } - - return nil - }) -} - -func (ca *clusterAdmin) ListPartitionReassignments(topic string, partitions []int32) (topicStatus map[string]map[int32]*PartitionReplicaReassignmentsStatus, err error) { - if topic == "" { - return nil, ErrInvalidTopic - } - - request := &ListPartitionReassignmentsRequest{ - TimeoutMs: int32(60000), - Version: int16(0), - } - - request.AddBlock(topic, partitions) - - b, err := ca.Controller() - if err != nil { - return nil, err - } - _ = b.Open(ca.client.Config()) - - rsp, err := b.ListPartitionReassignments(request) - - if err == nil && rsp != nil { - return rsp.TopicStatus, nil - } else { - return nil, err - } -} - -func (ca *clusterAdmin) DeleteRecords(topic string, partitionOffsets map[int32]int64) error { - if topic == "" { - return ErrInvalidTopic - } - partitionPerBroker := make(map[*Broker][]int32) - for partition := range partitionOffsets { - broker, err := ca.client.Leader(topic, partition) - if err != nil { - return err - } - if _, ok := partitionPerBroker[broker]; ok { - partitionPerBroker[broker] = append(partitionPerBroker[broker], partition) - } else { - partitionPerBroker[broker] = []int32{partition} - } - } - errs := make([]error, 0) - for broker, partitions := range partitionPerBroker { - topics := make(map[string]*DeleteRecordsRequestTopic) - recordsToDelete := make(map[int32]int64) - for _, p := range partitions { - recordsToDelete[p] = partitionOffsets[p] - } - topics[topic] = &DeleteRecordsRequestTopic{PartitionOffsets: recordsToDelete} - request := &DeleteRecordsRequest{ - Topics: topics, - Timeout: ca.conf.Admin.Timeout, - } - - rsp, err := broker.DeleteRecords(request) - if err != nil { - errs = append(errs, err) - } else { - deleteRecordsResponseTopic, ok := rsp.Topics[topic] - if !ok { - errs = append(errs, ErrIncompleteResponse) - } else { - for _, deleteRecordsResponsePartition := range deleteRecordsResponseTopic.Partitions { - if deleteRecordsResponsePartition.Err != ErrNoError { - errs = append(errs, errors.New(deleteRecordsResponsePartition.Err.Error())) - } - } - } - } - } - if len(errs) > 0 { - return ErrDeleteRecords{MultiError{&errs}} - } - //todo since we are dealing with couple of partitions it would be good if we return slice of errors - //for each partition instead of one error - return nil -} - -// Returns a bool indicating whether the resource request needs to go to a -// specific broker -func dependsOnSpecificNode(resource ConfigResource) bool { - return (resource.Type == BrokerResource && resource.Name != "") || - resource.Type == BrokerLoggerResource -} - -func (ca *clusterAdmin) DescribeConfig(resource ConfigResource) ([]ConfigEntry, error) { - var entries []ConfigEntry - var resources []*ConfigResource - resources = append(resources, &resource) - - request := &DescribeConfigsRequest{ - Resources: resources, - } - - if ca.conf.Version.IsAtLeast(V1_1_0_0) { - request.Version = 1 - } - - if ca.conf.Version.IsAtLeast(V2_0_0_0) { - request.Version = 2 - } - - var ( - b *Broker - err error - ) - - // DescribeConfig of broker/broker logger must be sent to the broker in question - if dependsOnSpecificNode(resource) { - id, _ := strconv.Atoi(resource.Name) - b, err = ca.findBroker(int32(id)) - } else { - b, err = ca.findAnyBroker() - } - if err != nil { - return nil, err - } - - _ = b.Open(ca.client.Config()) - rsp, err := b.DescribeConfigs(request) - if err != nil { - return nil, err - } - - for _, rspResource := range rsp.Resources { - if rspResource.Name == resource.Name { - if rspResource.ErrorMsg != "" { - return nil, errors.New(rspResource.ErrorMsg) - } - if rspResource.ErrorCode != 0 { - return nil, KError(rspResource.ErrorCode) - } - for _, cfgEntry := range rspResource.Configs { - entries = append(entries, *cfgEntry) - } - } - } - return entries, nil -} - -func (ca *clusterAdmin) AlterConfig(resourceType ConfigResourceType, name string, entries map[string]*string, validateOnly bool) error { - var resources []*AlterConfigsResource - resources = append(resources, &AlterConfigsResource{ - Type: resourceType, - Name: name, - ConfigEntries: entries, - }) - - request := &AlterConfigsRequest{ - Resources: resources, - ValidateOnly: validateOnly, - } - - var ( - b *Broker - err error - ) - - // AlterConfig of broker/broker logger must be sent to the broker in question - if dependsOnSpecificNode(ConfigResource{Name: name, Type: resourceType}) { - id, _ := strconv.Atoi(name) - b, err = ca.findBroker(int32(id)) - } else { - b, err = ca.findAnyBroker() - } - if err != nil { - return err - } - - _ = b.Open(ca.client.Config()) - rsp, err := b.AlterConfigs(request) - if err != nil { - return err - } - - for _, rspResource := range rsp.Resources { - if rspResource.Name == name { - if rspResource.ErrorMsg != "" { - return errors.New(rspResource.ErrorMsg) - } - if rspResource.ErrorCode != 0 { - return KError(rspResource.ErrorCode) - } - } - } - return nil -} - -func (ca *clusterAdmin) CreateACL(resource Resource, acl Acl) error { - var acls []*AclCreation - acls = append(acls, &AclCreation{resource, acl}) - request := &CreateAclsRequest{AclCreations: acls} - - if ca.conf.Version.IsAtLeast(V2_0_0_0) { - request.Version = 1 - } - - b, err := ca.Controller() - if err != nil { - return err - } - - _, err = b.CreateAcls(request) - return err -} - -func (ca *clusterAdmin) ListAcls(filter AclFilter) ([]ResourceAcls, error) { - request := &DescribeAclsRequest{AclFilter: filter} - - if ca.conf.Version.IsAtLeast(V2_0_0_0) { - request.Version = 1 - } - - b, err := ca.Controller() - if err != nil { - return nil, err - } - - rsp, err := b.DescribeAcls(request) - if err != nil { - return nil, err - } - - var lAcls []ResourceAcls - for _, rAcl := range rsp.ResourceAcls { - lAcls = append(lAcls, *rAcl) - } - return lAcls, nil -} - -func (ca *clusterAdmin) DeleteACL(filter AclFilter, validateOnly bool) ([]MatchingAcl, error) { - var filters []*AclFilter - filters = append(filters, &filter) - request := &DeleteAclsRequest{Filters: filters} - - if ca.conf.Version.IsAtLeast(V2_0_0_0) { - request.Version = 1 - } - - b, err := ca.Controller() - if err != nil { - return nil, err - } - - rsp, err := b.DeleteAcls(request) - if err != nil { - return nil, err - } - - var mAcls []MatchingAcl - for _, fr := range rsp.FilterResponses { - for _, mACL := range fr.MatchingAcls { - mAcls = append(mAcls, *mACL) - } - } - return mAcls, nil -} - -func (ca *clusterAdmin) DescribeConsumerGroups(groups []string) (result []*GroupDescription, err error) { - groupsPerBroker := make(map[*Broker][]string) - - for _, group := range groups { - controller, err := ca.client.Coordinator(group) - if err != nil { - return nil, err - } - groupsPerBroker[controller] = append(groupsPerBroker[controller], group) - } - - for broker, brokerGroups := range groupsPerBroker { - response, err := broker.DescribeGroups(&DescribeGroupsRequest{ - Groups: brokerGroups, - }) - if err != nil { - return nil, err - } - - result = append(result, response.Groups...) - } - return result, nil -} - -func (ca *clusterAdmin) ListConsumerGroups() (allGroups map[string]string, err error) { - allGroups = make(map[string]string) - - // Query brokers in parallel, since we have to query *all* brokers - brokers := ca.client.Brokers() - groupMaps := make(chan map[string]string, len(brokers)) - errChan := make(chan error, len(brokers)) - wg := sync.WaitGroup{} - - for _, b := range brokers { - wg.Add(1) - go func(b *Broker, conf *Config) { - defer wg.Done() - _ = b.Open(conf) // Ensure that broker is opened - - response, err := b.ListGroups(&ListGroupsRequest{}) - if err != nil { - errChan <- err - return - } - - groups := make(map[string]string) - for group, typ := range response.Groups { - groups[group] = typ - } - - groupMaps <- groups - }(b, ca.conf) - } - - wg.Wait() - close(groupMaps) - close(errChan) - - for groupMap := range groupMaps { - for group, protocolType := range groupMap { - allGroups[group] = protocolType - } - } - - // Intentionally return only the first error for simplicity - err = <-errChan - return -} - -func (ca *clusterAdmin) ListConsumerGroupOffsets(group string, topicPartitions map[string][]int32) (*OffsetFetchResponse, error) { - coordinator, err := ca.client.Coordinator(group) - if err != nil { - return nil, err - } - - request := &OffsetFetchRequest{ - ConsumerGroup: group, - partitions: topicPartitions, - } - - if ca.conf.Version.IsAtLeast(V0_10_2_0) { - request.Version = 2 - } else if ca.conf.Version.IsAtLeast(V0_8_2_2) { - request.Version = 1 - } - - return coordinator.FetchOffset(request) -} - -func (ca *clusterAdmin) DeleteConsumerGroup(group string) error { - coordinator, err := ca.client.Coordinator(group) - if err != nil { - return err - } - - request := &DeleteGroupsRequest{ - Groups: []string{group}, - } - - resp, err := coordinator.DeleteGroups(request) - if err != nil { - return err - } - - groupErr, ok := resp.GroupErrorCodes[group] - if !ok { - return ErrIncompleteResponse - } - - if groupErr != ErrNoError { - return groupErr - } - - return nil -} - -func (ca *clusterAdmin) DescribeLogDirs(brokerIds []int32) (allLogDirs map[int32][]DescribeLogDirsResponseDirMetadata, err error) { - allLogDirs = make(map[int32][]DescribeLogDirsResponseDirMetadata) - - // Query brokers in parallel, since we may have to query multiple brokers - logDirsMaps := make(chan map[int32][]DescribeLogDirsResponseDirMetadata, len(brokerIds)) - errChan := make(chan error, len(brokerIds)) - wg := sync.WaitGroup{} - - for _, b := range brokerIds { - wg.Add(1) - broker, err := ca.findBroker(b) - if err != nil { - Logger.Printf("Unable to find broker with ID = %v\n", b) - continue - } - go func(b *Broker, conf *Config) { - defer wg.Done() - _ = b.Open(conf) // Ensure that broker is opened - - response, err := b.DescribeLogDirs(&DescribeLogDirsRequest{}) - if err != nil { - errChan <- err - return - } - logDirs := make(map[int32][]DescribeLogDirsResponseDirMetadata) - logDirs[b.ID()] = response.LogDirs - logDirsMaps <- logDirs - }(broker, ca.conf) - } - - wg.Wait() - close(logDirsMaps) - close(errChan) - - for logDirsMap := range logDirsMaps { - for id, logDirs := range logDirsMap { - allLogDirs[id] = logDirs - } - } - - // Intentionally return only the first error for simplicity - err = <-errChan - return -} diff --git a/vendor/github.com/Shopify/sarama/alter_configs_request.go b/vendor/github.com/Shopify/sarama/alter_configs_request.go deleted file mode 100644 index c88bb604..00000000 --- a/vendor/github.com/Shopify/sarama/alter_configs_request.go +++ /dev/null @@ -1,126 +0,0 @@ -package sarama - -//AlterConfigsRequest is an alter config request type -type AlterConfigsRequest struct { - Resources []*AlterConfigsResource - ValidateOnly bool -} - -//AlterConfigsResource is an alter config resource type -type AlterConfigsResource struct { - Type ConfigResourceType - Name string - ConfigEntries map[string]*string -} - -func (a *AlterConfigsRequest) encode(pe packetEncoder) error { - if err := pe.putArrayLength(len(a.Resources)); err != nil { - return err - } - - for _, r := range a.Resources { - if err := r.encode(pe); err != nil { - return err - } - } - - pe.putBool(a.ValidateOnly) - return nil -} - -func (a *AlterConfigsRequest) decode(pd packetDecoder, version int16) error { - resourceCount, err := pd.getArrayLength() - if err != nil { - return err - } - - a.Resources = make([]*AlterConfigsResource, resourceCount) - for i := range a.Resources { - r := &AlterConfigsResource{} - err = r.decode(pd, version) - if err != nil { - return err - } - a.Resources[i] = r - } - - validateOnly, err := pd.getBool() - if err != nil { - return err - } - - a.ValidateOnly = validateOnly - - return nil -} - -func (a *AlterConfigsResource) encode(pe packetEncoder) error { - pe.putInt8(int8(a.Type)) - - if err := pe.putString(a.Name); err != nil { - return err - } - - if err := pe.putArrayLength(len(a.ConfigEntries)); err != nil { - return err - } - for configKey, configValue := range a.ConfigEntries { - if err := pe.putString(configKey); err != nil { - return err - } - if err := pe.putNullableString(configValue); err != nil { - return err - } - } - - return nil -} - -func (a *AlterConfigsResource) decode(pd packetDecoder, version int16) error { - t, err := pd.getInt8() - if err != nil { - return err - } - a.Type = ConfigResourceType(t) - - name, err := pd.getString() - if err != nil { - return err - } - a.Name = name - - n, err := pd.getArrayLength() - if err != nil { - return err - } - - if n > 0 { - a.ConfigEntries = make(map[string]*string, n) - for i := 0; i < n; i++ { - configKey, err := pd.getString() - if err != nil { - return err - } - if a.ConfigEntries[configKey], err = pd.getNullableString(); err != nil { - return err - } - } - } - return err -} - -func (a *AlterConfigsRequest) key() int16 { - return 33 -} - -func (a *AlterConfigsRequest) version() int16 { - return 0 -} - -func (a *AlterConfigsRequest) headerVersion() int16 { - return 1 -} - -func (a *AlterConfigsRequest) requiredVersion() KafkaVersion { - return V0_11_0_0 -} diff --git a/vendor/github.com/Shopify/sarama/alter_configs_response.go b/vendor/github.com/Shopify/sarama/alter_configs_response.go deleted file mode 100644 index 3266f927..00000000 --- a/vendor/github.com/Shopify/sarama/alter_configs_response.go +++ /dev/null @@ -1,101 +0,0 @@ -package sarama - -import "time" - -//AlterConfigsResponse is a response type for alter config -type AlterConfigsResponse struct { - ThrottleTime time.Duration - Resources []*AlterConfigsResourceResponse -} - -//AlterConfigsResourceResponse is a response type for alter config resource -type AlterConfigsResourceResponse struct { - ErrorCode int16 - ErrorMsg string - Type ConfigResourceType - Name string -} - -func (a *AlterConfigsResponse) encode(pe packetEncoder) error { - pe.putInt32(int32(a.ThrottleTime / time.Millisecond)) - - if err := pe.putArrayLength(len(a.Resources)); err != nil { - return err - } - - for i := range a.Resources { - pe.putInt16(a.Resources[i].ErrorCode) - err := pe.putString(a.Resources[i].ErrorMsg) - if err != nil { - return nil - } - pe.putInt8(int8(a.Resources[i].Type)) - err = pe.putString(a.Resources[i].Name) - if err != nil { - return nil - } - } - - return nil -} - -func (a *AlterConfigsResponse) decode(pd packetDecoder, version int16) error { - throttleTime, err := pd.getInt32() - if err != nil { - return err - } - a.ThrottleTime = time.Duration(throttleTime) * time.Millisecond - - responseCount, err := pd.getArrayLength() - if err != nil { - return err - } - - a.Resources = make([]*AlterConfigsResourceResponse, responseCount) - - for i := range a.Resources { - a.Resources[i] = new(AlterConfigsResourceResponse) - - errCode, err := pd.getInt16() - if err != nil { - return err - } - a.Resources[i].ErrorCode = errCode - - e, err := pd.getString() - if err != nil { - return err - } - a.Resources[i].ErrorMsg = e - - t, err := pd.getInt8() - if err != nil { - return err - } - a.Resources[i].Type = ConfigResourceType(t) - - name, err := pd.getString() - if err != nil { - return err - } - a.Resources[i].Name = name - } - - return nil -} - -func (a *AlterConfigsResponse) key() int16 { - return 32 -} - -func (a *AlterConfigsResponse) version() int16 { - return 0 -} - -func (a *AlterConfigsResponse) headerVersion() int16 { - return 0 -} - -func (a *AlterConfigsResponse) requiredVersion() KafkaVersion { - return V0_11_0_0 -} diff --git a/vendor/github.com/Shopify/sarama/alter_partition_reassignments_request.go b/vendor/github.com/Shopify/sarama/alter_partition_reassignments_request.go deleted file mode 100644 index f0a2f9dd..00000000 --- a/vendor/github.com/Shopify/sarama/alter_partition_reassignments_request.go +++ /dev/null @@ -1,130 +0,0 @@ -package sarama - -type alterPartitionReassignmentsBlock struct { - replicas []int32 -} - -func (b *alterPartitionReassignmentsBlock) encode(pe packetEncoder) error { - if err := pe.putNullableCompactInt32Array(b.replicas); err != nil { - return err - } - - pe.putEmptyTaggedFieldArray() - return nil -} - -func (b *alterPartitionReassignmentsBlock) decode(pd packetDecoder) (err error) { - if b.replicas, err = pd.getCompactInt32Array(); err != nil { - return err - } - return nil -} - -type AlterPartitionReassignmentsRequest struct { - TimeoutMs int32 - blocks map[string]map[int32]*alterPartitionReassignmentsBlock - Version int16 -} - -func (r *AlterPartitionReassignmentsRequest) encode(pe packetEncoder) error { - pe.putInt32(r.TimeoutMs) - - pe.putCompactArrayLength(len(r.blocks)) - - for topic, partitions := range r.blocks { - if err := pe.putCompactString(topic); err != nil { - return err - } - pe.putCompactArrayLength(len(partitions)) - for partition, block := range partitions { - pe.putInt32(partition) - if err := block.encode(pe); err != nil { - return err - } - } - pe.putEmptyTaggedFieldArray() - } - - pe.putEmptyTaggedFieldArray() - - return nil -} - -func (r *AlterPartitionReassignmentsRequest) decode(pd packetDecoder, version int16) (err error) { - r.Version = version - - if r.TimeoutMs, err = pd.getInt32(); err != nil { - return err - } - - topicCount, err := pd.getCompactArrayLength() - if err != nil { - return err - } - if topicCount > 0 { - r.blocks = make(map[string]map[int32]*alterPartitionReassignmentsBlock) - for i := 0; i < topicCount; i++ { - topic, err := pd.getCompactString() - if err != nil { - return err - } - partitionCount, err := pd.getCompactArrayLength() - if err != nil { - return err - } - r.blocks[topic] = make(map[int32]*alterPartitionReassignmentsBlock) - for j := 0; j < partitionCount; j++ { - partition, err := pd.getInt32() - if err != nil { - return err - } - block := &alterPartitionReassignmentsBlock{} - if err := block.decode(pd); err != nil { - return err - } - r.blocks[topic][partition] = block - - if _, err := pd.getEmptyTaggedFieldArray(); err != nil { - return err - } - } - if _, err := pd.getEmptyTaggedFieldArray(); err != nil { - return err - } - } - } - - if _, err := pd.getEmptyTaggedFieldArray(); err != nil { - return err - } - - return -} - -func (r *AlterPartitionReassignmentsRequest) key() int16 { - return 45 -} - -func (r *AlterPartitionReassignmentsRequest) version() int16 { - return r.Version -} - -func (r *AlterPartitionReassignmentsRequest) headerVersion() int16 { - return 2 -} - -func (r *AlterPartitionReassignmentsRequest) requiredVersion() KafkaVersion { - return V2_4_0_0 -} - -func (r *AlterPartitionReassignmentsRequest) AddBlock(topic string, partitionID int32, replicas []int32) { - if r.blocks == nil { - r.blocks = make(map[string]map[int32]*alterPartitionReassignmentsBlock) - } - - if r.blocks[topic] == nil { - r.blocks[topic] = make(map[int32]*alterPartitionReassignmentsBlock) - } - - r.blocks[topic][partitionID] = &alterPartitionReassignmentsBlock{replicas} -} diff --git a/vendor/github.com/Shopify/sarama/alter_partition_reassignments_response.go b/vendor/github.com/Shopify/sarama/alter_partition_reassignments_response.go deleted file mode 100644 index b3f9a15f..00000000 --- a/vendor/github.com/Shopify/sarama/alter_partition_reassignments_response.go +++ /dev/null @@ -1,157 +0,0 @@ -package sarama - -type alterPartitionReassignmentsErrorBlock struct { - errorCode KError - errorMessage *string -} - -func (b *alterPartitionReassignmentsErrorBlock) encode(pe packetEncoder) error { - pe.putInt16(int16(b.errorCode)) - if err := pe.putNullableCompactString(b.errorMessage); err != nil { - return err - } - pe.putEmptyTaggedFieldArray() - - return nil -} - -func (b *alterPartitionReassignmentsErrorBlock) decode(pd packetDecoder) (err error) { - errorCode, err := pd.getInt16() - if err != nil { - return err - } - b.errorCode = KError(errorCode) - b.errorMessage, err = pd.getCompactNullableString() - - if _, err := pd.getEmptyTaggedFieldArray(); err != nil { - return err - } - return err -} - -type AlterPartitionReassignmentsResponse struct { - Version int16 - ThrottleTimeMs int32 - ErrorCode KError - ErrorMessage *string - Errors map[string]map[int32]*alterPartitionReassignmentsErrorBlock -} - -func (r *AlterPartitionReassignmentsResponse) AddError(topic string, partition int32, kerror KError, message *string) { - if r.Errors == nil { - r.Errors = make(map[string]map[int32]*alterPartitionReassignmentsErrorBlock) - } - partitions := r.Errors[topic] - if partitions == nil { - partitions = make(map[int32]*alterPartitionReassignmentsErrorBlock) - r.Errors[topic] = partitions - } - - partitions[partition] = &alterPartitionReassignmentsErrorBlock{errorCode: kerror, errorMessage: message} -} - -func (r *AlterPartitionReassignmentsResponse) encode(pe packetEncoder) error { - pe.putInt32(r.ThrottleTimeMs) - pe.putInt16(int16(r.ErrorCode)) - if err := pe.putNullableCompactString(r.ErrorMessage); err != nil { - return err - } - - pe.putCompactArrayLength(len(r.Errors)) - for topic, partitions := range r.Errors { - if err := pe.putCompactString(topic); err != nil { - return err - } - pe.putCompactArrayLength(len(partitions)) - for partition, block := range partitions { - pe.putInt32(partition) - - if err := block.encode(pe); err != nil { - return err - } - } - pe.putEmptyTaggedFieldArray() - } - - pe.putEmptyTaggedFieldArray() - return nil -} - -func (r *AlterPartitionReassignmentsResponse) decode(pd packetDecoder, version int16) (err error) { - r.Version = version - - if r.ThrottleTimeMs, err = pd.getInt32(); err != nil { - return err - } - - kerr, err := pd.getInt16() - if err != nil { - return err - } - - r.ErrorCode = KError(kerr) - - if r.ErrorMessage, err = pd.getCompactNullableString(); err != nil { - return err - } - - numTopics, err := pd.getCompactArrayLength() - if err != nil { - return err - } - - if numTopics > 0 { - r.Errors = make(map[string]map[int32]*alterPartitionReassignmentsErrorBlock, numTopics) - for i := 0; i < numTopics; i++ { - topic, err := pd.getCompactString() - if err != nil { - return err - } - - ongoingPartitionReassignments, err := pd.getCompactArrayLength() - if err != nil { - return err - } - - r.Errors[topic] = make(map[int32]*alterPartitionReassignmentsErrorBlock, ongoingPartitionReassignments) - - for j := 0; j < ongoingPartitionReassignments; j++ { - partition, err := pd.getInt32() - if err != nil { - return err - } - block := &alterPartitionReassignmentsErrorBlock{} - if err := block.decode(pd); err != nil { - return err - } - - r.Errors[topic][partition] = block - } - if _, err = pd.getEmptyTaggedFieldArray(); err != nil { - return err - } - } - } - - if _, err = pd.getEmptyTaggedFieldArray(); err != nil { - return err - } - - return nil -} - -func (r *AlterPartitionReassignmentsResponse) key() int16 { - return 45 -} - -func (r *AlterPartitionReassignmentsResponse) version() int16 { - return r.Version -} - -func (r *AlterPartitionReassignmentsResponse) headerVersion() int16 { - return 1 -} - -func (r *AlterPartitionReassignmentsResponse) requiredVersion() KafkaVersion { - return V2_4_0_0 -} diff --git a/vendor/github.com/Shopify/sarama/api_versions_request.go b/vendor/github.com/Shopify/sarama/api_versions_request.go deleted file mode 100644 index d67c5e1e..00000000 --- a/vendor/github.com/Shopify/sarama/api_versions_request.go +++ /dev/null @@ -1,29 +0,0 @@ -package sarama - -//ApiVersionsRequest ... -type ApiVersionsRequest struct { -} - -func (a *ApiVersionsRequest) encode(pe packetEncoder) error { - return nil -} - -func (a *ApiVersionsRequest) decode(pd packetDecoder, version int16) (err error) { - return nil -} - -func (a *ApiVersionsRequest) key() int16 { - return 18 -} - -func (a *ApiVersionsRequest) version() int16 { - return 0 -} - -func (a *ApiVersionsRequest) headerVersion() int16 { - return 1 -} - -func (a *ApiVersionsRequest) requiredVersion() KafkaVersion { - return V0_10_0_0 -} diff --git a/vendor/github.com/Shopify/sarama/api_versions_response.go b/vendor/github.com/Shopify/sarama/api_versions_response.go deleted file mode 100644 index d09e8d9e..00000000 --- a/vendor/github.com/Shopify/sarama/api_versions_response.go +++ /dev/null @@ -1,93 +0,0 @@ -package sarama - -//ApiVersionsResponseBlock is an api version response block type -type ApiVersionsResponseBlock struct { - ApiKey int16 - MinVersion int16 - MaxVersion int16 -} - -func (b *ApiVersionsResponseBlock) encode(pe packetEncoder) error { - pe.putInt16(b.ApiKey) - pe.putInt16(b.MinVersion) - pe.putInt16(b.MaxVersion) - return nil -} - -func (b *ApiVersionsResponseBlock) decode(pd packetDecoder) error { - var err error - - if b.ApiKey, err = pd.getInt16(); err != nil { - return err - } - - if b.MinVersion, err = pd.getInt16(); err != nil { - return err - } - - if b.MaxVersion, err = pd.getInt16(); err != nil { - return err - } - - return nil -} - -//ApiVersionsResponse is an api version response type -type ApiVersionsResponse struct { - Err KError - ApiVersions []*ApiVersionsResponseBlock -} - -func (r *ApiVersionsResponse) encode(pe packetEncoder) error { - pe.putInt16(int16(r.Err)) - if err := pe.putArrayLength(len(r.ApiVersions)); err != nil { - return err - } - for _, apiVersion := range r.ApiVersions { - if err := apiVersion.encode(pe); err != nil { - return err - } - } - return nil -} - -func (r *ApiVersionsResponse) decode(pd packetDecoder, version int16) error { - kerr, err := pd.getInt16() - if err != nil { - return err - } - - r.Err = KError(kerr) - - numBlocks, err := pd.getArrayLength() - if err != nil { - return err - } - - r.ApiVersions = make([]*ApiVersionsResponseBlock, numBlocks) - for i := 0; i < numBlocks; i++ { - block := new(ApiVersionsResponseBlock) - if err := block.decode(pd); err != nil { - return err - } - r.ApiVersions[i] = block - } - - return nil -} - -func (r *ApiVersionsResponse) key() int16 { - return 18 -} - -func (r *ApiVersionsResponse) version() int16 { - return 0 -} - -func (a *ApiVersionsResponse) headerVersion() int16 { - return 0 -} - -func (r *ApiVersionsResponse) requiredVersion() KafkaVersion { - return V0_10_0_0 -} diff --git a/vendor/github.com/Shopify/sarama/async_producer.go b/vendor/github.com/Shopify/sarama/async_producer.go deleted file mode 100644 index f1ffc8f9..00000000 --- a/vendor/github.com/Shopify/sarama/async_producer.go +++ /dev/null @@ -1,1158 +0,0 @@ -package sarama - -import ( - "encoding/binary" - "fmt" - "sync" - "time" - - "github.com/eapache/go-resiliency/breaker" - "github.com/eapache/queue" -) - -// AsyncProducer publishes Kafka messages using a non-blocking API. It routes messages -// to the correct broker for the provided topic-partition, refreshing metadata as appropriate, -// and parses responses for errors. You must read from the Errors() channel or the -// producer will deadlock. You must call Close() or AsyncClose() on a producer to avoid -// leaks: it will not be garbage-collected automatically when it passes out of -// scope. -type AsyncProducer interface { - - // AsyncClose triggers a shutdown of the producer. The shutdown has completed - // when both the Errors and Successes channels have been closed. When calling - // AsyncClose, you *must* continue to read from those channels in order to - // drain the results of any messages in flight. - AsyncClose() - - // Close shuts down the producer and waits for any buffered messages to be - // flushed. You must call this function before a producer object passes out of - // scope, as it may otherwise leak memory. You must call this before calling - // Close on the underlying client. - Close() error - - // Input is the input channel for the user to write messages to that they - // wish to send. - Input() chan<- *ProducerMessage - - // Successes is the success output channel back to the user when Return.Successes is - // enabled. If Return.Successes is true, you MUST read from this channel or the - // Producer will deadlock. It is suggested that you send and read messages - // together in a single select statement. - Successes() <-chan *ProducerMessage - - // Errors is the error output channel back to the user. You MUST read from this - // channel or the Producer will deadlock when the channel is full. Alternatively, - // you can set Producer.Return.Errors in your config to false, which prevents - // errors to be returned. - Errors() <-chan *ProducerError -} - -// transactionManager keeps the state necessary to ensure idempotent production -type transactionManager struct { - producerID int64 - producerEpoch int16 - sequenceNumbers map[string]int32 - mutex sync.Mutex -} - -const ( - noProducerID = -1 - noProducerEpoch = -1 -) - -func (t *transactionManager) getAndIncrementSequenceNumber(topic string, partition int32) (int32, int16) { - key := fmt.Sprintf("%s-%d", topic, partition) - t.mutex.Lock() - defer t.mutex.Unlock() - sequence := t.sequenceNumbers[key] - t.sequenceNumbers[key] = sequence + 1 - return sequence, t.producerEpoch -} - -func (t *transactionManager) bumpEpoch() { - t.mutex.Lock() - defer t.mutex.Unlock() - t.producerEpoch++ - for k := range t.sequenceNumbers { - t.sequenceNumbers[k] = 0 - } -} - -func (t *transactionManager) getProducerID() (int64, int16) { - t.mutex.Lock() - defer t.mutex.Unlock() - return t.producerID, t.producerEpoch -} - -func newTransactionManager(conf *Config, client Client) (*transactionManager, error) { - txnmgr := &transactionManager{ - producerID: noProducerID, - producerEpoch: noProducerEpoch, - } - - if conf.Producer.Idempotent { - initProducerIDResponse, err := client.InitProducerID() - if err != nil { - return nil, err - } - txnmgr.producerID = initProducerIDResponse.ProducerID - txnmgr.producerEpoch = initProducerIDResponse.ProducerEpoch - txnmgr.sequenceNumbers = make(map[string]int32) - txnmgr.mutex = sync.Mutex{} - - Logger.Printf("Obtained a ProducerId: %d and ProducerEpoch: %d\n", txnmgr.producerID, txnmgr.producerEpoch) - } - - return txnmgr, nil -} - -type asyncProducer struct { - client Client - conf *Config - - errors chan *ProducerError - input, successes, retries chan *ProducerMessage - inFlight sync.WaitGroup - - brokers map[*Broker]*brokerProducer - brokerRefs map[*brokerProducer]int - brokerLock sync.Mutex - - txnmgr *transactionManager -} - -// NewAsyncProducer creates a new AsyncProducer using the given broker addresses and configuration. -func NewAsyncProducer(addrs []string, conf *Config) (AsyncProducer, error) { - client, err := NewClient(addrs, conf) - if err != nil { - return nil, err - } - return newAsyncProducer(client) -} - -// NewAsyncProducerFromClient creates a new Producer using the given client. It is still -// necessary to call Close() on the underlying client when shutting down this producer. -func NewAsyncProducerFromClient(client Client) (AsyncProducer, error) { - // For clients passed in by the client, ensure we don't - // call Close() on it. - cli := &nopCloserClient{client} - return newAsyncProducer(cli) -} - -func newAsyncProducer(client Client) (AsyncProducer, error) { - // Check that we are not dealing with a closed Client before processing any other arguments - if client.Closed() { - return nil, ErrClosedClient - } - - txnmgr, err := newTransactionManager(client.Config(), client) - if err != nil { - return nil, err - } - - p := &asyncProducer{ - client: client, - conf: client.Config(), - errors: make(chan *ProducerError), - input: make(chan *ProducerMessage), - successes: make(chan *ProducerMessage), - retries: make(chan *ProducerMessage), - brokers: make(map[*Broker]*brokerProducer), - brokerRefs: make(map[*brokerProducer]int), - txnmgr: txnmgr, - } - - // launch our singleton dispatchers - go withRecover(p.dispatcher) - go withRecover(p.retryHandler) - - return p, nil -} - -type flagSet int8 - -const ( - syn flagSet = 1 << iota // first message from partitionProducer to brokerProducer - fin // final message from partitionProducer to brokerProducer and back - shutdown // start the shutdown process -) - -// ProducerMessage is the collection of elements passed to the Producer in order to send a message. -type ProducerMessage struct { - Topic string // The Kafka topic for this message. - // The partitioning key for this message. Pre-existing Encoders include - // StringEncoder and ByteEncoder. - Key Encoder - // The actual message to store in Kafka. Pre-existing Encoders include - // StringEncoder and ByteEncoder. - Value Encoder - - // The headers are key-value pairs that are transparently passed - // by Kafka between producers and consumers. - Headers []RecordHeader - - // This field is used to hold arbitrary data you wish to include so it - // will be available when receiving on the Successes and Errors channels. - // Sarama completely ignores this field and is only to be used for - // pass-through data. - Metadata interface{} - - // Below this point are filled in by the producer as the message is processed - - // Offset is the offset of the message stored on the broker. This is only - // guaranteed to be defined if the message was successfully delivered and - // RequiredAcks is not NoResponse. - Offset int64 - // Partition is the partition that the message was sent to. This is only - // guaranteed to be defined if the message was successfully delivered. - Partition int32 - // Timestamp can vary in behaviour depending on broker configuration, being - // in either one of the CreateTime or LogAppendTime modes (default CreateTime), - // and requiring version at least 0.10.0. - // - // When configured to CreateTime, the timestamp is specified by the producer - // either by explicitly setting this field, or when the message is added - // to a produce set. - // - // When configured to LogAppendTime, the timestamp assigned to the message - // by the broker. This is only guaranteed to be defined if the message was - // successfully delivered and RequiredAcks is not NoResponse. - Timestamp time.Time - - retries int - flags flagSet - expectation chan *ProducerError - sequenceNumber int32 - producerEpoch int16 - hasSequence bool -} - -const producerMessageOverhead = 26 // the metadata overhead of CRC, flags, etc. - -func (m *ProducerMessage) byteSize(version int) int { - var size int - if version >= 2 { - size = maximumRecordOverhead - for _, h := range m.Headers { - size += len(h.Key) + len(h.Value) + 2*binary.MaxVarintLen32 - } - } else { - size = producerMessageOverhead - } - if m.Key != nil { - size += m.Key.Length() - } - if m.Value != nil { - size += m.Value.Length() - } - return size -} - -func (m *ProducerMessage) clear() { - m.flags = 0 - m.retries = 0 - m.sequenceNumber = 0 - m.producerEpoch = 0 - m.hasSequence = false -} - -// ProducerError is the type of error generated when the producer fails to deliver a message. -// It contains the original ProducerMessage as well as the actual error value. -type ProducerError struct { - Msg *ProducerMessage - Err error -} - -func (pe ProducerError) Error() string { - return fmt.Sprintf("kafka: Failed to produce message to topic %s: %s", pe.Msg.Topic, pe.Err) -} - -// ProducerErrors is a type that wraps a batch of "ProducerError"s and implements the Error interface. -// It can be returned from the Producer's Close method to avoid the need to manually drain the Errors channel -// when closing a producer. -type ProducerErrors []*ProducerError - -func (pe ProducerErrors) Error() string { - return fmt.Sprintf("kafka: Failed to deliver %d messages.", len(pe)) -} - -func (p *asyncProducer) Errors() <-chan *ProducerError { - return p.errors -} - -func (p *asyncProducer) Successes() <-chan *ProducerMessage { - return p.successes -} - -func (p *asyncProducer) Input() chan<- *ProducerMessage { - return p.input -} - -func (p *asyncProducer) Close() error { - p.AsyncClose() - - if p.conf.Producer.Return.Successes { - go withRecover(func() { - for range p.successes { - } - }) - } - - var errors ProducerErrors - if p.conf.Producer.Return.Errors { - for event := range p.errors { - errors = append(errors, event) - } - } else { - <-p.errors - } - - if len(errors) > 0 { - return errors - } - return nil -} - -func (p *asyncProducer) AsyncClose() { - go withRecover(p.shutdown) -} - -// singleton -// dispatches messages by topic -func (p *asyncProducer) dispatcher() { - handlers := make(map[string]chan<- *ProducerMessage) - shuttingDown := false - - for msg := range p.input { - if msg == nil { - Logger.Println("Something tried to send a nil message, it was ignored.") - continue - } - - if msg.flags&shutdown != 0 { - shuttingDown = true - p.inFlight.Done() - continue - } else if msg.retries == 0 { - if shuttingDown { - // we can't just call returnError here because that decrements the wait group, - // which hasn't been incremented yet for this message, and shouldn't be - pErr := &ProducerError{Msg: msg, Err: ErrShuttingDown} - if p.conf.Producer.Return.Errors { - p.errors <- pErr - } else { - Logger.Println(pErr) - } - continue - } - p.inFlight.Add(1) - } - - for _, interceptor := range p.conf.Producer.Interceptors { - msg.safelyApplyInterceptor(interceptor) - } - - version := 1 - if p.conf.Version.IsAtLeast(V0_11_0_0) { - version = 2 - } else if msg.Headers != nil { - p.returnError(msg, ConfigurationError("Producing headers requires Kafka at least v0.11")) - continue - } - if msg.byteSize(version) > p.conf.Producer.MaxMessageBytes { - p.returnError(msg, ErrMessageSizeTooLarge) - continue - } - - handler := handlers[msg.Topic] - if handler == nil { - handler = p.newTopicProducer(msg.Topic) - handlers[msg.Topic] = handler - } - - handler <- msg - } - - for _, handler := range handlers { - close(handler) - } -} - -// one per topic -// partitions messages, then dispatches them by partition -type topicProducer struct { - parent *asyncProducer - topic string - input <-chan *ProducerMessage - - breaker *breaker.Breaker - handlers map[int32]chan<- *ProducerMessage - partitioner Partitioner -} - -func (p *asyncProducer) newTopicProducer(topic string) chan<- *ProducerMessage { - input := make(chan *ProducerMessage, p.conf.ChannelBufferSize) - tp := &topicProducer{ - parent: p, - topic: topic, - input: input, - breaker: breaker.New(3, 1, 10*time.Second), - handlers: make(map[int32]chan<- *ProducerMessage), - partitioner: p.conf.Producer.Partitioner(topic), - } - go withRecover(tp.dispatch) - return input -} - -func (tp *topicProducer) dispatch() { - for msg := range tp.input { - if msg.retries == 0 { - if err := tp.partitionMessage(msg); err != nil { - tp.parent.returnError(msg, err) - continue - } - } - - handler := tp.handlers[msg.Partition] - if handler == nil { - handler = tp.parent.newPartitionProducer(msg.Topic, msg.Partition) - tp.handlers[msg.Partition] = handler - } - - handler <- msg - } - - for _, handler := range tp.handlers { - close(handler) - } -} - -func (tp *topicProducer) partitionMessage(msg *ProducerMessage) error { - var partitions []int32 - - err := tp.breaker.Run(func() (err error) { - requiresConsistency := false - if ep, ok := tp.partitioner.(DynamicConsistencyPartitioner); ok { - requiresConsistency = ep.MessageRequiresConsistency(msg) - } else { - requiresConsistency = tp.partitioner.RequiresConsistency() - } - - if requiresConsistency { - partitions, err = tp.parent.client.Partitions(msg.Topic) - } else { - partitions, err = tp.parent.client.WritablePartitions(msg.Topic) - } - return - }) - - if err != nil { - return err - } - - numPartitions := int32(len(partitions)) - - if numPartitions == 0 { - return ErrLeaderNotAvailable - } - - choice, err := tp.partitioner.Partition(msg, numPartitions) - - if err != nil { - return err - } else if choice < 0 || choice >= numPartitions { - return ErrInvalidPartition - } - - msg.Partition = partitions[choice] - - return nil -} - -// one per partition per topic -// dispatches messages to the appropriate broker -// also responsible for maintaining message order during retries -type partitionProducer struct { - parent *asyncProducer - topic string - partition int32 - input <-chan *ProducerMessage - - leader *Broker - breaker *breaker.Breaker - brokerProducer *brokerProducer - - // highWatermark tracks the "current" retry level, which is the only one where we actually let messages through, - // all other messages get buffered in retryState[msg.retries].buf to preserve ordering - // retryState[msg.retries].expectChaser simply tracks whether we've seen a fin message for a given level (and - // therefore whether our buffer is complete and safe to flush) - highWatermark int - retryState []partitionRetryState -} - -type partitionRetryState struct { - buf []*ProducerMessage - expectChaser bool -} - -func (p *asyncProducer) newPartitionProducer(topic string, partition int32) chan<- *ProducerMessage { - input := make(chan *ProducerMessage, p.conf.ChannelBufferSize) - pp := &partitionProducer{ - parent: p, - topic: topic, - partition: partition, - input: input, - - breaker: breaker.New(3, 1, 10*time.Second), - retryState: make([]partitionRetryState, p.conf.Producer.Retry.Max+1), - } - go withRecover(pp.dispatch) - return input -} - -func (pp *partitionProducer) backoff(retries int) { - var backoff time.Duration - if pp.parent.conf.Producer.Retry.BackoffFunc != nil { - maxRetries := pp.parent.conf.Producer.Retry.Max - backoff = pp.parent.conf.Producer.Retry.BackoffFunc(retries, maxRetries) - } else { - backoff = pp.parent.conf.Producer.Retry.Backoff - } - if backoff > 0 { - time.Sleep(backoff) - } -} - -func (pp *partitionProducer) dispatch() { - // try to prefetch the leader; if this doesn't work, we'll do a proper call to `updateLeader` - // on the first message - pp.leader, _ = pp.parent.client.Leader(pp.topic, pp.partition) - if pp.leader != nil { - pp.brokerProducer = pp.parent.getBrokerProducer(pp.leader) - pp.parent.inFlight.Add(1) // we're generating a syn message; track it so we don't shut down while it's still inflight - pp.brokerProducer.input <- &ProducerMessage{Topic: pp.topic, Partition: pp.partition, flags: syn} - } - - defer func() { - if pp.brokerProducer != nil { - pp.parent.unrefBrokerProducer(pp.leader, pp.brokerProducer) - } - }() - - for msg := range pp.input { - if pp.brokerProducer != nil && pp.brokerProducer.abandoned != nil { - select { - case <-pp.brokerProducer.abandoned: - // a message on the abandoned channel means that our current broker selection is out of date - Logger.Printf("producer/leader/%s/%d abandoning broker %d\n", pp.topic, pp.partition, pp.leader.ID()) - pp.parent.unrefBrokerProducer(pp.leader, pp.brokerProducer) - pp.brokerProducer = nil - time.Sleep(pp.parent.conf.Producer.Retry.Backoff) - default: - // producer connection is still open. - } - } - - if msg.retries > pp.highWatermark { - // a new, higher, retry level; handle it and then back off - pp.newHighWatermark(msg.retries) - pp.backoff(msg.retries) - } else if pp.highWatermark > 0 { - // we are retrying something (else highWatermark would be 0) but this message is not a *new* retry level - if msg.retries < pp.highWatermark { - // in fact this message is not even the current retry level, so buffer it for now (unless it's a just a fin) - if msg.flags&fin == fin { - pp.retryState[msg.retries].expectChaser = false - pp.parent.inFlight.Done() // this fin is now handled and will be garbage collected - } else { - pp.retryState[msg.retries].buf = append(pp.retryState[msg.retries].buf, msg) - } - continue - } else if msg.flags&fin == fin { - // this message is of the current retry level (msg.retries == highWatermark) and the fin flag is set, - // meaning this retry level is done and we can go down (at least) one level and flush that - pp.retryState[pp.highWatermark].expectChaser = false - pp.flushRetryBuffers() - pp.parent.inFlight.Done() // this fin is now handled and will be garbage collected - continue - } - } - - // if we made it this far then the current msg contains real data, and can be sent to the next goroutine - // without breaking any of our ordering guarantees - - if pp.brokerProducer == nil { - if err := pp.updateLeader(); err != nil { - pp.parent.returnError(msg, err) - pp.backoff(msg.retries) - continue - } - Logger.Printf("producer/leader/%s/%d selected broker %d\n", pp.topic, pp.partition, pp.leader.ID()) - } - - // Now that we know we have a broker to actually try and send this message to, generate the sequence - // number for it. - // All messages being retried (sent or not) have already had their retry count updated - // Also, ignore "special" syn/fin messages used to sync the brokerProducer and the topicProducer. - if pp.parent.conf.Producer.Idempotent && msg.retries == 0 && msg.flags == 0 { - msg.sequenceNumber, msg.producerEpoch = pp.parent.txnmgr.getAndIncrementSequenceNumber(msg.Topic, msg.Partition) - msg.hasSequence = true - } - - pp.brokerProducer.input <- msg - } -} - -func (pp *partitionProducer) newHighWatermark(hwm int) { - Logger.Printf("producer/leader/%s/%d state change to [retrying-%d]\n", pp.topic, pp.partition, hwm) - pp.highWatermark = hwm - - // send off a fin so that we know when everything "in between" has made it - // back to us and we can safely flush the backlog (otherwise we risk re-ordering messages) - pp.retryState[pp.highWatermark].expectChaser = true - pp.parent.inFlight.Add(1) // we're generating a fin message; track it so we don't shut down while it's still inflight - pp.brokerProducer.input <- &ProducerMessage{Topic: pp.topic, Partition: pp.partition, flags: fin, retries: pp.highWatermark - 1} - - // a new HWM means that our current broker selection is out of date - Logger.Printf("producer/leader/%s/%d abandoning broker %d\n", pp.topic, pp.partition, pp.leader.ID()) - pp.parent.unrefBrokerProducer(pp.leader, pp.brokerProducer) - pp.brokerProducer = nil -} - -func (pp *partitionProducer) flushRetryBuffers() { - Logger.Printf("producer/leader/%s/%d state change to [flushing-%d]\n", pp.topic, pp.partition, pp.highWatermark) - for { - pp.highWatermark-- - - if pp.brokerProducer == nil { - if err := pp.updateLeader(); err != nil { - pp.parent.returnErrors(pp.retryState[pp.highWatermark].buf, err) - goto flushDone - } - Logger.Printf("producer/leader/%s/%d selected broker %d\n", pp.topic, pp.partition, pp.leader.ID()) - } - - for _, msg := range pp.retryState[pp.highWatermark].buf { - pp.brokerProducer.input <- msg - } - - flushDone: - pp.retryState[pp.highWatermark].buf = nil - if pp.retryState[pp.highWatermark].expectChaser { - Logger.Printf("producer/leader/%s/%d state change to [retrying-%d]\n", pp.topic, pp.partition, pp.highWatermark) - break - } else if pp.highWatermark == 0 { - Logger.Printf("producer/leader/%s/%d state change to [normal]\n", pp.topic, pp.partition) - break - } - } -} - -func (pp *partitionProducer) updateLeader() error { - return pp.breaker.Run(func() (err error) { - if err = pp.parent.client.RefreshMetadata(pp.topic); err != nil { - return err - } - - if pp.leader, err = pp.parent.client.Leader(pp.topic, pp.partition); err != nil { - return err - } - - pp.brokerProducer = pp.parent.getBrokerProducer(pp.leader) - pp.parent.inFlight.Add(1) // we're generating a syn message; track it so we don't shut down while it's still inflight - pp.brokerProducer.input <- &ProducerMessage{Topic: pp.topic, Partition: pp.partition, flags: syn} - - return nil - }) -} - -// one per broker; also constructs an associated flusher -func (p *asyncProducer) newBrokerProducer(broker *Broker) *brokerProducer { - var ( - input = make(chan *ProducerMessage) - bridge = make(chan *produceSet) - responses = make(chan *brokerProducerResponse) - ) - - bp := &brokerProducer{ - parent: p, - broker: broker, - input: input, - output: bridge, - responses: responses, - stopchan: make(chan struct{}), - buffer: newProduceSet(p), - currentRetries: make(map[string]map[int32]error), - } - go withRecover(bp.run) - - // minimal bridge to make the network response `select`able - go withRecover(func() { - for set := range bridge { - request := set.buildRequest() - - response, err := broker.Produce(request) - - responses <- &brokerProducerResponse{ - set: set, - err: err, - res: response, - } - } - close(responses) - }) - - if p.conf.Producer.Retry.Max <= 0 { - bp.abandoned = make(chan struct{}) - } - - return bp -} - -type brokerProducerResponse struct { - set *produceSet - err error - res *ProduceResponse -} - -// groups messages together into appropriately-sized batches for sending to the broker -// handles state related to retries etc -type brokerProducer struct { - parent *asyncProducer - broker *Broker - - input chan *ProducerMessage - output chan<- *produceSet - responses <-chan *brokerProducerResponse - abandoned chan struct{} - stopchan chan struct{} - - buffer *produceSet - timer <-chan time.Time - timerFired bool - - closing error - currentRetries map[string]map[int32]error -} - -func (bp *brokerProducer) run() { - var output chan<- *produceSet - Logger.Printf("producer/broker/%d starting up\n", bp.broker.ID()) - - for { - select { - case msg, ok := <-bp.input: - if !ok { - Logger.Printf("producer/broker/%d input chan closed\n", bp.broker.ID()) - bp.shutdown() - return - } - - if msg == nil { - continue - } - - if msg.flags&syn == syn { - Logger.Printf("producer/broker/%d state change to [open] on %s/%d\n", - bp.broker.ID(), msg.Topic, msg.Partition) - if bp.currentRetries[msg.Topic] == nil { - bp.currentRetries[msg.Topic] = make(map[int32]error) - } - bp.currentRetries[msg.Topic][msg.Partition] = nil - bp.parent.inFlight.Done() - continue - } - - if reason := bp.needsRetry(msg); reason != nil { - bp.parent.retryMessage(msg, reason) - - if bp.closing == nil && msg.flags&fin == fin { - // we were retrying this partition but we can start processing again - delete(bp.currentRetries[msg.Topic], msg.Partition) - Logger.Printf("producer/broker/%d state change to [closed] on %s/%d\n", - bp.broker.ID(), msg.Topic, msg.Partition) - } - - continue - } - - if bp.buffer.wouldOverflow(msg) { - Logger.Printf("producer/broker/%d maximum request accumulated, waiting for space\n", bp.broker.ID()) - if err := bp.waitForSpace(msg, false); err != nil { - bp.parent.retryMessage(msg, err) - continue - } - } - - if bp.parent.txnmgr.producerID != noProducerID && bp.buffer.producerEpoch != msg.producerEpoch { - // The epoch was reset, need to roll the buffer over - Logger.Printf("producer/broker/%d detected epoch rollover, waiting for new buffer\n", bp.broker.ID()) - if err := bp.waitForSpace(msg, true); err != nil { - bp.parent.retryMessage(msg, err) - continue - } - } - if err := bp.buffer.add(msg); err != nil { - bp.parent.returnError(msg, err) - continue - } - - if bp.parent.conf.Producer.Flush.Frequency > 0 && bp.timer == nil { - bp.timer = time.After(bp.parent.conf.Producer.Flush.Frequency) - } - case <-bp.timer: - bp.timerFired = true - case output <- bp.buffer: - bp.rollOver() - case response, ok := <-bp.responses: - if ok { - bp.handleResponse(response) - } - case <-bp.stopchan: - Logger.Printf( - "producer/broker/%d run loop asked to stop\n", bp.broker.ID()) - return - } - - if bp.timerFired || bp.buffer.readyToFlush() { - output = bp.output - } else { - output = nil - } - } -} - -func (bp *brokerProducer) shutdown() { - for !bp.buffer.empty() { - select { - case response := <-bp.responses: - bp.handleResponse(response) - case bp.output <- bp.buffer: - bp.rollOver() - } - } - close(bp.output) - for response := range bp.responses { - bp.handleResponse(response) - } - close(bp.stopchan) - Logger.Printf("producer/broker/%d shut down\n", bp.broker.ID()) -} - -func (bp *brokerProducer) needsRetry(msg *ProducerMessage) error { - if bp.closing != nil { - return bp.closing - } - - return bp.currentRetries[msg.Topic][msg.Partition] -} - -func (bp *brokerProducer) waitForSpace(msg *ProducerMessage, forceRollover bool) error { - for { - select { - case response := <-bp.responses: - bp.handleResponse(response) - // handling a response can change our state, so re-check some things - if reason := bp.needsRetry(msg); reason != nil { - return reason - } else if !bp.buffer.wouldOverflow(msg) && !forceRollover { - return nil - } - case bp.output <- bp.buffer: - bp.rollOver() - return nil - } - } -} - -func (bp *brokerProducer) rollOver() { - bp.timer = nil - bp.timerFired = false - bp.buffer = newProduceSet(bp.parent) -} - -func (bp *brokerProducer) handleResponse(response *brokerProducerResponse) { - if response.err != nil { - bp.handleError(response.set, response.err) - } else { - bp.handleSuccess(response.set, response.res) - } - - if bp.buffer.empty() { - bp.rollOver() // this can happen if the response invalidated our buffer - } -} - -func (bp *brokerProducer) handleSuccess(sent *produceSet, response *ProduceResponse) { - // we iterate through the blocks in the request set, not the response, so that we notice - // if the response is missing a block completely - var retryTopics []string - sent.eachPartition(func(topic string, partition int32, pSet *partitionSet) { - if response == nil { - // this only happens when RequiredAcks is NoResponse, so we have to assume success - bp.parent.returnSuccesses(pSet.msgs) - return - } - - block := response.GetBlock(topic, partition) - if block == nil { - bp.parent.returnErrors(pSet.msgs, ErrIncompleteResponse) - return - } - - switch block.Err { - // Success - case ErrNoError: - if bp.parent.conf.Version.IsAtLeast(V0_10_0_0) && !block.Timestamp.IsZero() { - for _, msg := range pSet.msgs { - msg.Timestamp = block.Timestamp - } - } - for i, msg := range pSet.msgs { - msg.Offset = block.Offset + int64(i) - } - bp.parent.returnSuccesses(pSet.msgs) - // Duplicate - case ErrDuplicateSequenceNumber: - bp.parent.returnSuccesses(pSet.msgs) - // Retriable errors - case ErrInvalidMessage, ErrUnknownTopicOrPartition, ErrLeaderNotAvailable, ErrNotLeaderForPartition, - ErrRequestTimedOut, ErrNotEnoughReplicas, ErrNotEnoughReplicasAfterAppend: - if bp.parent.conf.Producer.Retry.Max <= 0 { - bp.parent.abandonBrokerConnection(bp.broker) - bp.parent.returnErrors(pSet.msgs, block.Err) - } else { - retryTopics = append(retryTopics, topic) - } - // Other non-retriable errors - default: - if bp.parent.conf.Producer.Retry.Max <= 0 { - bp.parent.abandonBrokerConnection(bp.broker) - } - bp.parent.returnErrors(pSet.msgs, block.Err) - } - }) - - if len(retryTopics) > 0 { - if bp.parent.conf.Producer.Idempotent { - err := bp.parent.client.RefreshMetadata(retryTopics...) - if err != nil { - Logger.Printf("Failed refreshing metadata because of %v\n", err) - } - } - - sent.eachPartition(func(topic string, partition int32, pSet *partitionSet) { - block := response.GetBlock(topic, partition) - if block == nil { - // handled in the previous "eachPartition" loop - return - } - - switch block.Err { - case ErrInvalidMessage, ErrUnknownTopicOrPartition, ErrLeaderNotAvailable, ErrNotLeaderForPartition, - ErrRequestTimedOut, ErrNotEnoughReplicas, ErrNotEnoughReplicasAfterAppend: - Logger.Printf("producer/broker/%d state change to [retrying] on %s/%d because %v\n", - bp.broker.ID(), topic, partition, block.Err) - if bp.currentRetries[topic] == nil { - bp.currentRetries[topic] = make(map[int32]error) - } - bp.currentRetries[topic][partition] = block.Err - if bp.parent.conf.Producer.Idempotent { - go bp.parent.retryBatch(topic, partition, pSet, block.Err) - } else { - bp.parent.retryMessages(pSet.msgs, block.Err) - } - // dropping the following messages has the side effect of incrementing their retry count - bp.parent.retryMessages(bp.buffer.dropPartition(topic, partition), block.Err) - } - }) - } -} - -func (p *asyncProducer) retryBatch(topic string, partition int32, pSet *partitionSet, kerr KError) { - Logger.Printf("Retrying batch for %v-%d because of %s\n", topic, partition, kerr) - produceSet := newProduceSet(p) - produceSet.msgs[topic] = make(map[int32]*partitionSet) - produceSet.msgs[topic][partition] = pSet - produceSet.bufferBytes += pSet.bufferBytes - produceSet.bufferCount += len(pSet.msgs) - for _, msg := range pSet.msgs { - if msg.retries >= p.conf.Producer.Retry.Max { - p.returnError(msg, kerr) - return - } - msg.retries++ - } - - // it's expected that a metadata refresh has been requested prior to calling retryBatch - leader, err := p.client.Leader(topic, partition) - if err != nil { - Logger.Printf("Failed retrying batch for %v-%d because of %v while looking up for new leader\n", topic, partition, err) - for _, msg := range pSet.msgs { - p.returnError(msg, kerr) - } - return - } - bp := p.getBrokerProducer(leader) - bp.output <- produceSet -} - -func (bp *brokerProducer) handleError(sent *produceSet, err error) { - switch err.(type) { - case PacketEncodingError: - sent.eachPartition(func(topic string, partition int32, pSet *partitionSet) { - bp.parent.returnErrors(pSet.msgs, err) - }) - default: - Logger.Printf("producer/broker/%d state change to [closing] because %s\n", bp.broker.ID(), err) - bp.parent.abandonBrokerConnection(bp.broker) - _ = bp.broker.Close() - bp.closing = err - sent.eachPartition(func(topic string, partition int32, pSet *partitionSet) { - bp.parent.retryMessages(pSet.msgs, err) - }) - bp.buffer.eachPartition(func(topic string, partition int32, pSet *partitionSet) { - bp.parent.retryMessages(pSet.msgs, err) - }) - bp.rollOver() - } -} - -// singleton -// effectively a "bridge" between the flushers and the dispatcher in order to avoid deadlock -// based on https://godoc.org/github.com/eapache/channels#InfiniteChannel -func (p *asyncProducer) retryHandler() { - var msg *ProducerMessage - buf := queue.New() - - for { - if buf.Length() == 0 { - msg = <-p.retries - } else { - select { - case msg = <-p.retries: - case p.input <- buf.Peek().(*ProducerMessage): - buf.Remove() - continue - } - } - - if msg == nil { - return - } - - buf.Add(msg) - } -} - -// utility functions - -func (p *asyncProducer) shutdown() { - Logger.Println("Producer shutting down.") - p.inFlight.Add(1) - p.input <- &ProducerMessage{flags: shutdown} - - p.inFlight.Wait() - - err := p.client.Close() - if err != nil { - Logger.Println("producer/shutdown failed to close the embedded client:", err) - } - - close(p.input) - close(p.retries) - close(p.errors) - close(p.successes) -} - -func (p *asyncProducer) returnError(msg *ProducerMessage, err error) { - // We need to reset the producer ID epoch if we set a sequence number on it, because the broker - // will never see a message with this number, so we can never continue the sequence. - if msg.hasSequence { - Logger.Printf("producer/txnmanager rolling over epoch due to publish failure on %s/%d", msg.Topic, msg.Partition) - p.txnmgr.bumpEpoch() - } - msg.clear() - pErr := &ProducerError{Msg: msg, Err: err} - if p.conf.Producer.Return.Errors { - p.errors <- pErr - } else { - Logger.Println(pErr) - } - p.inFlight.Done() -} - -func (p *asyncProducer) returnErrors(batch []*ProducerMessage, err error) { - for _, msg := range batch { - p.returnError(msg, err) - } -} - -func (p *asyncProducer) returnSuccesses(batch []*ProducerMessage) { - for _, msg := range batch { - if p.conf.Producer.Return.Successes { - msg.clear() - p.successes <- msg - } - p.inFlight.Done() - } -} - -func (p *asyncProducer) retryMessage(msg *ProducerMessage, err error) { - if msg.retries >= p.conf.Producer.Retry.Max { - p.returnError(msg, err) - } else { - msg.retries++ - p.retries <- msg - } -} - -func (p *asyncProducer) retryMessages(batch []*ProducerMessage, err error) { - for _, msg := range batch { - p.retryMessage(msg, err) - } -} - -func (p *asyncProducer) getBrokerProducer(broker *Broker) *brokerProducer { - p.brokerLock.Lock() - defer p.brokerLock.Unlock() - - bp := p.brokers[broker] - - if bp == nil { - bp = p.newBrokerProducer(broker) - p.brokers[broker] = bp - p.brokerRefs[bp] = 0 - } - - p.brokerRefs[bp]++ - - return bp -} - -func (p *asyncProducer) unrefBrokerProducer(broker *Broker, bp *brokerProducer) { - p.brokerLock.Lock() - defer p.brokerLock.Unlock() - - p.brokerRefs[bp]-- - if p.brokerRefs[bp] == 0 { - close(bp.input) - delete(p.brokerRefs, bp) - - if p.brokers[broker] == bp { - delete(p.brokers, broker) - } - } -} - -func (p *asyncProducer) abandonBrokerConnection(broker *Broker) { - p.brokerLock.Lock() - defer p.brokerLock.Unlock() - - bc, ok := p.brokers[broker] - if ok && bc.abandoned != nil { - close(bc.abandoned) - } - - delete(p.brokers, broker) -} diff --git a/vendor/github.com/Shopify/sarama/balance_strategy.go b/vendor/github.com/Shopify/sarama/balance_strategy.go deleted file mode 100644 index 0ce7fea1..00000000 --- a/vendor/github.com/Shopify/sarama/balance_strategy.go +++ /dev/null @@ -1,1061 +0,0 @@ -package sarama - -import ( - "container/heap" - "math" - "sort" - "strings" -) - -const ( - // RangeBalanceStrategyName identifies strategies that use the range partition assignment strategy - RangeBalanceStrategyName = "range" - - // RoundRobinBalanceStrategyName identifies strategies that use the round-robin partition assignment strategy - RoundRobinBalanceStrategyName = "roundrobin" - - // StickyBalanceStrategyName identifies strategies that use the sticky-partition assignment strategy - StickyBalanceStrategyName = "sticky" - - defaultGeneration = -1 -) - -// BalanceStrategyPlan is the results of any BalanceStrategy.Plan attempt. -// It contains an allocation of topic/partitions by memberID in the form of -// a `memberID -> topic -> partitions` map. -type BalanceStrategyPlan map[string]map[string][]int32 - -// Add assigns a topic with a number partitions to a member. -func (p BalanceStrategyPlan) Add(memberID, topic string, partitions ...int32) { - if len(partitions) == 0 { - return - } - if _, ok := p[memberID]; !ok { - p[memberID] = make(map[string][]int32, 1) - } - p[memberID][topic] = append(p[memberID][topic], partitions...) -} - -// -------------------------------------------------------------------- - -// BalanceStrategy is used to balance topics and partitions -// across members of a consumer group -type BalanceStrategy interface { - // Name uniquely identifies the strategy. - Name() string - - // Plan accepts a map of `memberID -> metadata` and a map of `topic -> partitions` - // and returns a distribution plan. - Plan(members map[string]ConsumerGroupMemberMetadata, topics map[string][]int32) (BalanceStrategyPlan, error) - - // AssignmentData returns the serialized assignment data for the specified - // memberID - AssignmentData(memberID string, topics map[string][]int32, generationID int32) ([]byte, error) -} - -// -------------------------------------------------------------------- - -// BalanceStrategyRange is the default and assigns partitions as ranges to consumer group members. -// Example with one topic T with six partitions (0..5) and two members (M1, M2): -// M1: {T: [0, 1, 2]} -// M2: {T: [3, 4, 5]} -var BalanceStrategyRange = &balanceStrategy{ - name: RangeBalanceStrategyName, - coreFn: func(plan BalanceStrategyPlan, memberIDs []string, topic string, partitions []int32) { - step := float64(len(partitions)) / float64(len(memberIDs)) - - for i, memberID := range memberIDs { - pos := float64(i) - min := int(math.Floor(pos*step + 0.5)) - max := int(math.Floor((pos+1)*step + 0.5)) - plan.Add(memberID, topic, partitions[min:max]...) - } - }, -} - -// BalanceStrategyRoundRobin assigns partitions to members in alternating order. -// Example with topic T with six partitions (0..5) and two members (M1, M2): -// M1: {T: [0, 2, 4]} -// M2: {T: [1, 3, 5]} -var BalanceStrategyRoundRobin = &balanceStrategy{ - name: RoundRobinBalanceStrategyName, - coreFn: func(plan BalanceStrategyPlan, memberIDs []string, topic string, partitions []int32) { - for i, part := range partitions { - memberID := memberIDs[i%len(memberIDs)] - plan.Add(memberID, topic, part) - } - }, -} - -// BalanceStrategySticky assigns partitions to members with an attempt to preserve earlier assignments -// while maintain a balanced partition distribution. -// Example with topic T with six partitions (0..5) and two members (M1, M2): -// M1: {T: [0, 2, 4]} -// M2: {T: [1, 3, 5]} -// -// On reassignment with an additional consumer, you might get an assignment plan like: -// M1: {T: [0, 2]} -// M2: {T: [1, 3]} -// M3: {T: [4, 5]} -// -var BalanceStrategySticky = &stickyBalanceStrategy{} - -// -------------------------------------------------------------------- - -type balanceStrategy struct { - name string - coreFn func(plan BalanceStrategyPlan, memberIDs []string, topic string, partitions []int32) -} - -// Name implements BalanceStrategy. -func (s *balanceStrategy) Name() string { return s.name } - -// Plan implements BalanceStrategy. -func (s *balanceStrategy) Plan(members map[string]ConsumerGroupMemberMetadata, topics map[string][]int32) (BalanceStrategyPlan, error) { - // Build members by topic map - mbt := make(map[string][]string) - for memberID, meta := range members { - for _, topic := range meta.Topics { - mbt[topic] = append(mbt[topic], memberID) - } - } - - // Sort members for each topic - for topic, memberIDs := range mbt { - sort.Sort(&balanceStrategySortable{ - topic: topic, - memberIDs: memberIDs, - }) - } - - // Assemble plan - plan := make(BalanceStrategyPlan, len(members)) - for topic, memberIDs := range mbt { - s.coreFn(plan, memberIDs, topic, topics[topic]) - } - return plan, nil -} - -// AssignmentData simple strategies do not require any shared assignment data -func (s *balanceStrategy) AssignmentData(memberID string, topics map[string][]int32, generationID int32) ([]byte, error) { - return nil, nil -} - -type balanceStrategySortable struct { - topic string - memberIDs []string -} - -func (p balanceStrategySortable) Len() int { return len(p.memberIDs) } -func (p balanceStrategySortable) Swap(i, j int) { - p.memberIDs[i], p.memberIDs[j] = p.memberIDs[j], p.memberIDs[i] -} -func (p balanceStrategySortable) Less(i, j int) bool { - return balanceStrategyHashValue(p.topic, p.memberIDs[i]) < balanceStrategyHashValue(p.topic, p.memberIDs[j]) -} - -func balanceStrategyHashValue(vv ...string) uint32 { - h := uint32(2166136261) - for _, s := range vv { - for _, c := range s { - h ^= uint32(c) - h *= 16777619 - } - } - return h -} - -type stickyBalanceStrategy struct { - movements partitionMovements -} - -// Name implements BalanceStrategy. -func (s *stickyBalanceStrategy) Name() string { return StickyBalanceStrategyName } - -// Plan implements BalanceStrategy. -func (s *stickyBalanceStrategy) Plan(members map[string]ConsumerGroupMemberMetadata, topics map[string][]int32) (BalanceStrategyPlan, error) { - // track partition movements during generation of the partition assignment plan - s.movements = partitionMovements{ - Movements: make(map[topicPartitionAssignment]consumerPair), - PartitionMovementsByTopic: make(map[string]map[consumerPair]map[topicPartitionAssignment]bool), - } - - // prepopulate the current assignment state from userdata on the consumer group members - currentAssignment, prevAssignment, err := prepopulateCurrentAssignments(members) - if err != nil { - return nil, err - } - - // determine if we're dealing with a completely fresh assignment, or if there's existing assignment state - isFreshAssignment := false - if len(currentAssignment) == 0 { - isFreshAssignment = true - } - - // create a mapping of all current topic partitions and the consumers that can be assigned to them - partition2AllPotentialConsumers := make(map[topicPartitionAssignment][]string) - for topic, partitions := range topics { - for _, partition := range partitions { - partition2AllPotentialConsumers[topicPartitionAssignment{Topic: topic, Partition: partition}] = []string{} - } - } - - // create a mapping of all consumers to all potential topic partitions that can be assigned to them - // also, populate the mapping of partitions to potential consumers - consumer2AllPotentialPartitions := make(map[string][]topicPartitionAssignment, len(members)) - for memberID, meta := range members { - consumer2AllPotentialPartitions[memberID] = make([]topicPartitionAssignment, 0) - for _, topicSubscription := range meta.Topics { - // only evaluate topic subscriptions that are present in the supplied topics map - if _, found := topics[topicSubscription]; found { - for _, partition := range topics[topicSubscription] { - topicPartition := topicPartitionAssignment{Topic: topicSubscription, Partition: partition} - consumer2AllPotentialPartitions[memberID] = append(consumer2AllPotentialPartitions[memberID], topicPartition) - partition2AllPotentialConsumers[topicPartition] = append(partition2AllPotentialConsumers[topicPartition], memberID) - } - } - } - - // add this consumer to currentAssignment (with an empty topic partition assignment) if it does not already exist - if _, exists := currentAssignment[memberID]; !exists { - currentAssignment[memberID] = make([]topicPartitionAssignment, 0) - } - } - - // create a mapping of each partition to its current consumer, where possible - currentPartitionConsumers := make(map[topicPartitionAssignment]string, len(currentAssignment)) - unvisitedPartitions := make(map[topicPartitionAssignment]bool, len(partition2AllPotentialConsumers)) - for partition := range partition2AllPotentialConsumers { - unvisitedPartitions[partition] = true - } - var unassignedPartitions []topicPartitionAssignment - for memberID, partitions := range currentAssignment { - var keepPartitions []topicPartitionAssignment - for _, partition := range partitions { - // If this partition no longer exists at all, likely due to the - // topic being deleted, we remove the partition from the member. - if _, exists := partition2AllPotentialConsumers[partition]; !exists { - continue - } - delete(unvisitedPartitions, partition) - currentPartitionConsumers[partition] = memberID - - if !strsContains(members[memberID].Topics, partition.Topic) { - unassignedPartitions = append(unassignedPartitions, partition) - continue - } - keepPartitions = append(keepPartitions, partition) - } - currentAssignment[memberID] = keepPartitions - } - for unvisited := range unvisitedPartitions { - unassignedPartitions = append(unassignedPartitions, unvisited) - } - - // sort the topic partitions in order of priority for reassignment - sortedPartitions := sortPartitions(currentAssignment, prevAssignment, isFreshAssignment, partition2AllPotentialConsumers, consumer2AllPotentialPartitions) - - // at this point we have preserved all valid topic partition to consumer assignments and removed - // all invalid topic partitions and invalid consumers. Now we need to assign unassignedPartitions - // to consumers so that the topic partition assignments are as balanced as possible. - - // an ascending sorted set of consumers based on how many topic partitions are already assigned to them - sortedCurrentSubscriptions := sortMemberIDsByPartitionAssignments(currentAssignment) - s.balance(currentAssignment, prevAssignment, sortedPartitions, unassignedPartitions, sortedCurrentSubscriptions, consumer2AllPotentialPartitions, partition2AllPotentialConsumers, currentPartitionConsumers) - - // Assemble plan - plan := make(BalanceStrategyPlan, len(currentAssignment)) - for memberID, assignments := range currentAssignment { - if len(assignments) == 0 { - plan[memberID] = make(map[string][]int32) - } else { - for _, assignment := range assignments { - plan.Add(memberID, assignment.Topic, assignment.Partition) - } - } - } - return plan, nil -} - -// AssignmentData serializes the set of topics currently assigned to the -// specified member as part of the supplied balance plan -func (s *stickyBalanceStrategy) AssignmentData(memberID string, topics map[string][]int32, generationID int32) ([]byte, error) { - return encode(&StickyAssignorUserDataV1{ - Topics: topics, - Generation: generationID, - }, nil) -} - -func strsContains(s []string, value string) bool { - for _, entry := range s { - if entry == value { - return true - } - } - return false -} - -// Balance assignments across consumers for maximum fairness and stickiness. -func (s *stickyBalanceStrategy) balance(currentAssignment map[string][]topicPartitionAssignment, prevAssignment map[topicPartitionAssignment]consumerGenerationPair, sortedPartitions []topicPartitionAssignment, unassignedPartitions []topicPartitionAssignment, sortedCurrentSubscriptions []string, consumer2AllPotentialPartitions map[string][]topicPartitionAssignment, partition2AllPotentialConsumers map[topicPartitionAssignment][]string, currentPartitionConsumer map[topicPartitionAssignment]string) { - initializing := false - if len(sortedCurrentSubscriptions) == 0 || len(currentAssignment[sortedCurrentSubscriptions[0]]) == 0 { - initializing = true - } - - // assign all unassigned partitions - for _, partition := range unassignedPartitions { - // skip if there is no potential consumer for the partition - if len(partition2AllPotentialConsumers[partition]) == 0 { - continue - } - sortedCurrentSubscriptions = assignPartition(partition, sortedCurrentSubscriptions, currentAssignment, consumer2AllPotentialPartitions, currentPartitionConsumer) - } - - // narrow down the reassignment scope to only those partitions that can actually be reassigned - for partition := range partition2AllPotentialConsumers { - if !canTopicPartitionParticipateInReassignment(partition, partition2AllPotentialConsumers) { - sortedPartitions = removeTopicPartitionFromMemberAssignments(sortedPartitions, partition) - } - } - - // narrow down the reassignment scope to only those consumers that are subject to reassignment - fixedAssignments := make(map[string][]topicPartitionAssignment) - for memberID := range consumer2AllPotentialPartitions { - if !canConsumerParticipateInReassignment(memberID, currentAssignment, consumer2AllPotentialPartitions, partition2AllPotentialConsumers) { - fixedAssignments[memberID] = currentAssignment[memberID] - delete(currentAssignment, memberID) - sortedCurrentSubscriptions = sortMemberIDsByPartitionAssignments(currentAssignment) - } - } - - // create a deep copy of the current assignment so we can revert to it if we do not get a more balanced assignment later - preBalanceAssignment := deepCopyAssignment(currentAssignment) - preBalancePartitionConsumers := make(map[topicPartitionAssignment]string, len(currentPartitionConsumer)) - for k, v := range currentPartitionConsumer { - preBalancePartitionConsumers[k] = v - } - - reassignmentPerformed := s.performReassignments(sortedPartitions, currentAssignment, prevAssignment, sortedCurrentSubscriptions, consumer2AllPotentialPartitions, partition2AllPotentialConsumers, currentPartitionConsumer) - - // if we are not preserving existing assignments and we have made changes to the current assignment - // make sure we are getting a more balanced assignment; otherwise, revert to previous assignment - if !initializing && reassignmentPerformed && getBalanceScore(currentAssignment) >= getBalanceScore(preBalanceAssignment) { - currentAssignment = deepCopyAssignment(preBalanceAssignment) - currentPartitionConsumer = make(map[topicPartitionAssignment]string, len(preBalancePartitionConsumers)) - for k, v := range preBalancePartitionConsumers { - currentPartitionConsumer[k] = v - } - } - - // add the fixed assignments (those that could not change) back - for consumer, assignments := range fixedAssignments { - currentAssignment[consumer] = assignments - } -} - -// Calculate the balance score of the given assignment, as the sum of assigned partitions size difference of all consumer pairs. -// A perfectly balanced assignment (with all consumers getting the same number of partitions) has a balance score of 0. -// Lower balance score indicates a more balanced assignment. -func getBalanceScore(assignment map[string][]topicPartitionAssignment) int { - consumer2AssignmentSize := make(map[string]int, len(assignment)) - for memberID, partitions := range assignment { - consumer2AssignmentSize[memberID] = len(partitions) - } - - var score float64 - for memberID, consumerAssignmentSize := range consumer2AssignmentSize { - delete(consumer2AssignmentSize, memberID) - for _, otherConsumerAssignmentSize := range consumer2AssignmentSize { - score += math.Abs(float64(consumerAssignmentSize - otherConsumerAssignmentSize)) - } - } - return int(score) -} - -// Determine whether the current assignment plan is balanced. -func isBalanced(currentAssignment map[string][]topicPartitionAssignment, allSubscriptions map[string][]topicPartitionAssignment) bool { - sortedCurrentSubscriptions := sortMemberIDsByPartitionAssignments(currentAssignment) - min := len(currentAssignment[sortedCurrentSubscriptions[0]]) - max := len(currentAssignment[sortedCurrentSubscriptions[len(sortedCurrentSubscriptions)-1]]) - if min >= max-1 { - // if minimum and maximum numbers of partitions assigned to consumers differ by at most one return true - return true - } - - // create a mapping from partitions to the consumer assigned to them - allPartitions := make(map[topicPartitionAssignment]string) - for memberID, partitions := range currentAssignment { - for _, partition := range partitions { - if _, exists := allPartitions[partition]; exists { - Logger.Printf("Topic %s Partition %d is assigned more than one consumer", partition.Topic, partition.Partition) - } - allPartitions[partition] = memberID - } - } - - // for each consumer that does not have all the topic partitions it can get make sure none of the topic partitions it - // could but did not get cannot be moved to it (because that would break the balance) - for _, memberID := range sortedCurrentSubscriptions { - consumerPartitions := currentAssignment[memberID] - consumerPartitionCount := len(consumerPartitions) - - // skip if this consumer already has all the topic partitions it can get - if consumerPartitionCount == len(allSubscriptions[memberID]) { - continue - } - - // otherwise make sure it cannot get any more - potentialTopicPartitions := allSubscriptions[memberID] - for _, partition := range potentialTopicPartitions { - if !memberAssignmentsIncludeTopicPartition(currentAssignment[memberID], partition) { - otherConsumer := allPartitions[partition] - otherConsumerPartitionCount := len(currentAssignment[otherConsumer]) - if consumerPartitionCount < otherConsumerPartitionCount { - return false - } - } - } - } - return true -} - -// Reassign all topic partitions that need reassignment until balanced. -func (s *stickyBalanceStrategy) performReassignments(reassignablePartitions []topicPartitionAssignment, currentAssignment map[string][]topicPartitionAssignment, prevAssignment map[topicPartitionAssignment]consumerGenerationPair, sortedCurrentSubscriptions []string, consumer2AllPotentialPartitions map[string][]topicPartitionAssignment, partition2AllPotentialConsumers map[topicPartitionAssignment][]string, currentPartitionConsumer map[topicPartitionAssignment]string) bool { - reassignmentPerformed := false - modified := false - - // repeat reassignment until no partition can be moved to improve the balance - for { - modified = false - // reassign all reassignable partitions (starting from the partition with least potential consumers and if needed) - // until the full list is processed or a balance is achieved - for _, partition := range reassignablePartitions { - if isBalanced(currentAssignment, consumer2AllPotentialPartitions) { - break - } - - // the partition must have at least two consumers - if len(partition2AllPotentialConsumers[partition]) <= 1 { - Logger.Printf("Expected more than one potential consumer for partition %s topic %d", partition.Topic, partition.Partition) - } - - // the partition must have a consumer - consumer := currentPartitionConsumer[partition] - if consumer == "" { - Logger.Printf("Expected topic %s partition %d to be assigned to a consumer", partition.Topic, partition.Partition) - } - - if _, exists := prevAssignment[partition]; exists { - if len(currentAssignment[consumer]) > (len(currentAssignment[prevAssignment[partition].MemberID]) + 1) { - sortedCurrentSubscriptions = s.reassignPartition(partition, currentAssignment, sortedCurrentSubscriptions, currentPartitionConsumer, prevAssignment[partition].MemberID) - reassignmentPerformed = true - modified = true - continue - } - } - - // check if a better-suited consumer exists for the partition; if so, reassign it - for _, otherConsumer := range partition2AllPotentialConsumers[partition] { - if len(currentAssignment[consumer]) > (len(currentAssignment[otherConsumer]) + 1) { - sortedCurrentSubscriptions = s.reassignPartitionToNewConsumer(partition, currentAssignment, sortedCurrentSubscriptions, currentPartitionConsumer, consumer2AllPotentialPartitions) - reassignmentPerformed = true - modified = true - break - } - } - } - if !modified { - return reassignmentPerformed - } - } -} - -// Identify a new consumer for a topic partition and reassign it. -func (s *stickyBalanceStrategy) reassignPartitionToNewConsumer(partition topicPartitionAssignment, currentAssignment map[string][]topicPartitionAssignment, sortedCurrentSubscriptions []string, currentPartitionConsumer map[topicPartitionAssignment]string, consumer2AllPotentialPartitions map[string][]topicPartitionAssignment) []string { - for _, anotherConsumer := range sortedCurrentSubscriptions { - if memberAssignmentsIncludeTopicPartition(consumer2AllPotentialPartitions[anotherConsumer], partition) { - return s.reassignPartition(partition, currentAssignment, sortedCurrentSubscriptions, currentPartitionConsumer, anotherConsumer) - } - } - return sortedCurrentSubscriptions -} - -// Reassign a specific partition to a new consumer -func (s *stickyBalanceStrategy) reassignPartition(partition topicPartitionAssignment, currentAssignment map[string][]topicPartitionAssignment, sortedCurrentSubscriptions []string, currentPartitionConsumer map[topicPartitionAssignment]string, newConsumer string) []string { - consumer := currentPartitionConsumer[partition] - // find the correct partition movement considering the stickiness requirement - partitionToBeMoved := s.movements.getTheActualPartitionToBeMoved(partition, consumer, newConsumer) - return s.processPartitionMovement(partitionToBeMoved, newConsumer, currentAssignment, sortedCurrentSubscriptions, currentPartitionConsumer) -} - -// Track the movement of a topic partition after assignment -func (s *stickyBalanceStrategy) processPartitionMovement(partition topicPartitionAssignment, newConsumer string, currentAssignment map[string][]topicPartitionAssignment, sortedCurrentSubscriptions []string, currentPartitionConsumer map[topicPartitionAssignment]string) []string { - oldConsumer := currentPartitionConsumer[partition] - s.movements.movePartition(partition, oldConsumer, newConsumer) - - currentAssignment[oldConsumer] = removeTopicPartitionFromMemberAssignments(currentAssignment[oldConsumer], partition) - currentAssignment[newConsumer] = append(currentAssignment[newConsumer], partition) - currentPartitionConsumer[partition] = newConsumer - return sortMemberIDsByPartitionAssignments(currentAssignment) -} - -// Determine whether a specific consumer should be considered for topic partition assignment. -func canConsumerParticipateInReassignment(memberID string, currentAssignment map[string][]topicPartitionAssignment, consumer2AllPotentialPartitions map[string][]topicPartitionAssignment, partition2AllPotentialConsumers map[topicPartitionAssignment][]string) bool { - currentPartitions := currentAssignment[memberID] - currentAssignmentSize := len(currentPartitions) - maxAssignmentSize := len(consumer2AllPotentialPartitions[memberID]) - if currentAssignmentSize > maxAssignmentSize { - Logger.Printf("The consumer %s is assigned more partitions than the maximum possible", memberID) - } - if currentAssignmentSize < maxAssignmentSize { - // if a consumer is not assigned all its potential partitions it is subject to reassignment - return true - } - for _, partition := range currentPartitions { - if canTopicPartitionParticipateInReassignment(partition, partition2AllPotentialConsumers) { - return true - } - } - return false -} - -// Only consider reassigning those topic partitions that have two or more potential consumers. -func canTopicPartitionParticipateInReassignment(partition topicPartitionAssignment, partition2AllPotentialConsumers map[topicPartitionAssignment][]string) bool { - return len(partition2AllPotentialConsumers[partition]) >= 2 -} - -// The assignment should improve the overall balance of the partition assignments to consumers. -func assignPartition(partition topicPartitionAssignment, sortedCurrentSubscriptions []string, currentAssignment map[string][]topicPartitionAssignment, consumer2AllPotentialPartitions map[string][]topicPartitionAssignment, currentPartitionConsumer map[topicPartitionAssignment]string) []string { - for _, memberID := range sortedCurrentSubscriptions { - if memberAssignmentsIncludeTopicPartition(consumer2AllPotentialPartitions[memberID], partition) { - currentAssignment[memberID] = append(currentAssignment[memberID], partition) - currentPartitionConsumer[partition] = memberID - break - } - } - return sortMemberIDsByPartitionAssignments(currentAssignment) -} - -// Deserialize topic partition assignment data to aid with creation of a sticky assignment. -func deserializeTopicPartitionAssignment(userDataBytes []byte) (StickyAssignorUserData, error) { - userDataV1 := &StickyAssignorUserDataV1{} - if err := decode(userDataBytes, userDataV1); err != nil { - userDataV0 := &StickyAssignorUserDataV0{} - if err := decode(userDataBytes, userDataV0); err != nil { - return nil, err - } - return userDataV0, nil - } - return userDataV1, nil -} - -// filterAssignedPartitions returns a map of consumer group members to their list of previously-assigned topic partitions, limited -// to those topic partitions currently reported by the Kafka cluster. -func filterAssignedPartitions(currentAssignment map[string][]topicPartitionAssignment, partition2AllPotentialConsumers map[topicPartitionAssignment][]string) map[string][]topicPartitionAssignment { - assignments := deepCopyAssignment(currentAssignment) - for memberID, partitions := range assignments { - // perform in-place filtering - i := 0 - for _, partition := range partitions { - if _, exists := partition2AllPotentialConsumers[partition]; exists { - partitions[i] = partition - i++ - } - } - assignments[memberID] = partitions[:i] - } - return assignments -} - -func removeTopicPartitionFromMemberAssignments(assignments []topicPartitionAssignment, topic topicPartitionAssignment) []topicPartitionAssignment { - for i, assignment := range assignments { - if assignment == topic { - return append(assignments[:i], assignments[i+1:]...) - } - } - return assignments -} - -func memberAssignmentsIncludeTopicPartition(assignments []topicPartitionAssignment, topic topicPartitionAssignment) bool { - for _, assignment := range assignments { - if assignment == topic { - return true - } - } - return false -} - -func sortPartitions(currentAssignment map[string][]topicPartitionAssignment, partitionsWithADifferentPreviousAssignment map[topicPartitionAssignment]consumerGenerationPair, isFreshAssignment bool, partition2AllPotentialConsumers map[topicPartitionAssignment][]string, consumer2AllPotentialPartitions map[string][]topicPartitionAssignment) []topicPartitionAssignment { - unassignedPartitions := make(map[topicPartitionAssignment]bool, len(partition2AllPotentialConsumers)) - for partition := range partition2AllPotentialConsumers { - unassignedPartitions[partition] = true - } - - sortedPartitions := make([]topicPartitionAssignment, 0) - if !isFreshAssignment && areSubscriptionsIdentical(partition2AllPotentialConsumers, consumer2AllPotentialPartitions) { - // if this is a reassignment and the subscriptions are identical (all consumers can consumer from all topics) - // then we just need to simply list partitions in a round robin fashion (from consumers with - // most assigned partitions to those with least) - assignments := filterAssignedPartitions(currentAssignment, partition2AllPotentialConsumers) - - // use priority-queue to evaluate consumer group members in descending-order based on - // the number of topic partition assignments (i.e. consumers with most assignments first) - pq := make(assignmentPriorityQueue, len(assignments)) - i := 0 - for consumerID, consumerAssignments := range assignments { - pq[i] = &consumerGroupMember{ - id: consumerID, - assignments: consumerAssignments, - } - i++ - } - heap.Init(&pq) - - for { - // loop until no consumer-group members remain - if pq.Len() == 0 { - break - } - member := pq[0] - - // partitions that were assigned to a different consumer last time - var prevPartitionIndex int - for i, partition := range member.assignments { - if _, exists := partitionsWithADifferentPreviousAssignment[partition]; exists { - prevPartitionIndex = i - break - } - } - - if len(member.assignments) > 0 { - partition := member.assignments[prevPartitionIndex] - sortedPartitions = append(sortedPartitions, partition) - delete(unassignedPartitions, partition) - if prevPartitionIndex == 0 { - member.assignments = member.assignments[1:] - } else { - member.assignments = append(member.assignments[:prevPartitionIndex], member.assignments[prevPartitionIndex+1:]...) - } - heap.Fix(&pq, 0) - } else { - heap.Pop(&pq) - } - } - - for partition := range unassignedPartitions { - sortedPartitions = append(sortedPartitions, partition) - } - } else { - // an ascending sorted set of topic partitions based on how many consumers can potentially use them - sortedPartitions = sortPartitionsByPotentialConsumerAssignments(partition2AllPotentialConsumers) - } - return sortedPartitions -} - -func sortMemberIDsByPartitionAssignments(assignments map[string][]topicPartitionAssignment) []string { - // sort the members by the number of partition assignments in ascending order - sortedMemberIDs := make([]string, 0, len(assignments)) - for memberID := range assignments { - sortedMemberIDs = append(sortedMemberIDs, memberID) - } - sort.SliceStable(sortedMemberIDs, func(i, j int) bool { - ret := len(assignments[sortedMemberIDs[i]]) - len(assignments[sortedMemberIDs[j]]) - if ret == 0 { - return sortedMemberIDs[i] < sortedMemberIDs[j] - } - return len(assignments[sortedMemberIDs[i]]) < len(assignments[sortedMemberIDs[j]]) - }) - return sortedMemberIDs -} - -func sortPartitionsByPotentialConsumerAssignments(partition2AllPotentialConsumers map[topicPartitionAssignment][]string) []topicPartitionAssignment { - // sort the members by the number of partition assignments in descending order - sortedPartionIDs := make([]topicPartitionAssignment, len(partition2AllPotentialConsumers)) - i := 0 - for partition := range partition2AllPotentialConsumers { - sortedPartionIDs[i] = partition - i++ - } - sort.Slice(sortedPartionIDs, func(i, j int) bool { - if len(partition2AllPotentialConsumers[sortedPartionIDs[i]]) == len(partition2AllPotentialConsumers[sortedPartionIDs[j]]) { - ret := strings.Compare(sortedPartionIDs[i].Topic, sortedPartionIDs[j].Topic) - if ret == 0 { - return sortedPartionIDs[i].Partition < sortedPartionIDs[j].Partition - } - return ret < 0 - } - return len(partition2AllPotentialConsumers[sortedPartionIDs[i]]) < len(partition2AllPotentialConsumers[sortedPartionIDs[j]]) - }) - return sortedPartionIDs -} - -func deepCopyAssignment(assignment map[string][]topicPartitionAssignment) map[string][]topicPartitionAssignment { - m := make(map[string][]topicPartitionAssignment, len(assignment)) - for memberID, subscriptions := range assignment { - m[memberID] = append(subscriptions[:0:0], subscriptions...) - } - return m -} - -func areSubscriptionsIdentical(partition2AllPotentialConsumers map[topicPartitionAssignment][]string, consumer2AllPotentialPartitions map[string][]topicPartitionAssignment) bool { - curMembers := make(map[string]int) - for _, cur := range partition2AllPotentialConsumers { - if len(curMembers) == 0 { - for _, curMembersElem := range cur { - curMembers[curMembersElem]++ - } - continue - } - - if len(curMembers) != len(cur) { - return false - } - - yMap := make(map[string]int) - for _, yElem := range cur { - yMap[yElem]++ - } - - for curMembersMapKey, curMembersMapVal := range curMembers { - if yMap[curMembersMapKey] != curMembersMapVal { - return false - } - } - } - - curPartitions := make(map[topicPartitionAssignment]int) - for _, cur := range consumer2AllPotentialPartitions { - if len(curPartitions) == 0 { - for _, curPartitionElem := range cur { - curPartitions[curPartitionElem]++ - } - continue - } - - if len(curPartitions) != len(cur) { - return false - } - - yMap := make(map[topicPartitionAssignment]int) - for _, yElem := range cur { - yMap[yElem]++ - } - - for curMembersMapKey, curMembersMapVal := range curPartitions { - if yMap[curMembersMapKey] != curMembersMapVal { - return false - } - } - } - return true -} - -// We need to process subscriptions' user data with each consumer's reported generation in mind -// higher generations overwrite lower generations in case of a conflict -// note that a conflict could exist only if user data is for different generations -func prepopulateCurrentAssignments(members map[string]ConsumerGroupMemberMetadata) (map[string][]topicPartitionAssignment, map[topicPartitionAssignment]consumerGenerationPair, error) { - currentAssignment := make(map[string][]topicPartitionAssignment) - prevAssignment := make(map[topicPartitionAssignment]consumerGenerationPair) - - // for each partition we create a sorted map of its consumers by generation - sortedPartitionConsumersByGeneration := make(map[topicPartitionAssignment]map[int]string) - for memberID, meta := range members { - consumerUserData, err := deserializeTopicPartitionAssignment(meta.UserData) - if err != nil { - return nil, nil, err - } - for _, partition := range consumerUserData.partitions() { - if consumers, exists := sortedPartitionConsumersByGeneration[partition]; exists { - if consumerUserData.hasGeneration() { - if _, generationExists := consumers[consumerUserData.generation()]; generationExists { - // same partition is assigned to two consumers during the same rebalance. - // log a warning and skip this record - Logger.Printf("Topic %s Partition %d is assigned to multiple consumers following sticky assignment generation %d", partition.Topic, partition.Partition, consumerUserData.generation()) - continue - } else { - consumers[consumerUserData.generation()] = memberID - } - } else { - consumers[defaultGeneration] = memberID - } - } else { - generation := defaultGeneration - if consumerUserData.hasGeneration() { - generation = consumerUserData.generation() - } - sortedPartitionConsumersByGeneration[partition] = map[int]string{generation: memberID} - } - } - } - - // prevAssignment holds the prior ConsumerGenerationPair (before current) of each partition - // current and previous consumers are the last two consumers of each partition in the above sorted map - for partition, consumers := range sortedPartitionConsumersByGeneration { - // sort consumers by generation in decreasing order - var generations []int - for generation := range consumers { - generations = append(generations, generation) - } - sort.Sort(sort.Reverse(sort.IntSlice(generations))) - - consumer := consumers[generations[0]] - if _, exists := currentAssignment[consumer]; !exists { - currentAssignment[consumer] = []topicPartitionAssignment{partition} - } else { - currentAssignment[consumer] = append(currentAssignment[consumer], partition) - } - - // check for previous assignment, if any - if len(generations) > 1 { - prevAssignment[partition] = consumerGenerationPair{ - MemberID: consumers[generations[1]], - Generation: generations[1], - } - } - } - return currentAssignment, prevAssignment, nil -} - -type consumerGenerationPair struct { - MemberID string - Generation int -} - -// consumerPair represents a pair of Kafka consumer ids involved in a partition reassignment. -type consumerPair struct { - SrcMemberID string - DstMemberID string -} - -// partitionMovements maintains some data structures to simplify lookup of partition movements among consumers. -type partitionMovements struct { - PartitionMovementsByTopic map[string]map[consumerPair]map[topicPartitionAssignment]bool - Movements map[topicPartitionAssignment]consumerPair -} - -func (p *partitionMovements) removeMovementRecordOfPartition(partition topicPartitionAssignment) consumerPair { - pair := p.Movements[partition] - delete(p.Movements, partition) - - partitionMovementsForThisTopic := p.PartitionMovementsByTopic[partition.Topic] - delete(partitionMovementsForThisTopic[pair], partition) - if len(partitionMovementsForThisTopic[pair]) == 0 { - delete(partitionMovementsForThisTopic, pair) - } - if len(p.PartitionMovementsByTopic[partition.Topic]) == 0 { - delete(p.PartitionMovementsByTopic, partition.Topic) - } - return pair -} - -func (p *partitionMovements) addPartitionMovementRecord(partition topicPartitionAssignment, pair consumerPair) { - p.Movements[partition] = pair - if _, exists := p.PartitionMovementsByTopic[partition.Topic]; !exists { - p.PartitionMovementsByTopic[partition.Topic] = make(map[consumerPair]map[topicPartitionAssignment]bool) - } - partitionMovementsForThisTopic := p.PartitionMovementsByTopic[partition.Topic] - if _, exists := partitionMovementsForThisTopic[pair]; !exists { - partitionMovementsForThisTopic[pair] = make(map[topicPartitionAssignment]bool) - } - partitionMovementsForThisTopic[pair][partition] = true -} - -func (p *partitionMovements) movePartition(partition topicPartitionAssignment, oldConsumer, newConsumer string) { - pair := consumerPair{ - SrcMemberID: oldConsumer, - DstMemberID: newConsumer, - } - if _, exists := p.Movements[partition]; exists { - // this partition has previously moved - existingPair := p.removeMovementRecordOfPartition(partition) - if existingPair.DstMemberID != oldConsumer { - Logger.Printf("Existing pair DstMemberID %s was not equal to the oldConsumer ID %s", existingPair.DstMemberID, oldConsumer) - } - if existingPair.SrcMemberID != newConsumer { - // the partition is not moving back to its previous consumer - p.addPartitionMovementRecord(partition, consumerPair{ - SrcMemberID: existingPair.SrcMemberID, - DstMemberID: newConsumer, - }) - } - } else { - p.addPartitionMovementRecord(partition, pair) - } -} - -func (p *partitionMovements) getTheActualPartitionToBeMoved(partition topicPartitionAssignment, oldConsumer, newConsumer string) topicPartitionAssignment { - if _, exists := p.PartitionMovementsByTopic[partition.Topic]; !exists { - return partition - } - if _, exists := p.Movements[partition]; exists { - // this partition has previously moved - if oldConsumer != p.Movements[partition].DstMemberID { - Logger.Printf("Partition movement DstMemberID %s was not equal to the oldConsumer ID %s", p.Movements[partition].DstMemberID, oldConsumer) - } - oldConsumer = p.Movements[partition].SrcMemberID - } - - partitionMovementsForThisTopic := p.PartitionMovementsByTopic[partition.Topic] - reversePair := consumerPair{ - SrcMemberID: newConsumer, - DstMemberID: oldConsumer, - } - if _, exists := partitionMovementsForThisTopic[reversePair]; !exists { - return partition - } - var reversePairPartition topicPartitionAssignment - for otherPartition := range partitionMovementsForThisTopic[reversePair] { - reversePairPartition = otherPartition - } - return reversePairPartition -} - -func (p *partitionMovements) isLinked(src, dst string, pairs []consumerPair, currentPath []string) ([]string, bool) { - if src == dst { - return currentPath, false - } - if len(pairs) == 0 { - return currentPath, false - } - for _, pair := range pairs { - if src == pair.SrcMemberID && dst == pair.DstMemberID { - currentPath = append(currentPath, src, dst) - return currentPath, true - } - } - - for _, pair := range pairs { - if pair.SrcMemberID == src { - // create a deep copy of the pairs, excluding the current pair - reducedSet := make([]consumerPair, len(pairs)-1) - i := 0 - for _, p := range pairs { - if p != pair { - reducedSet[i] = pair - i++ - } - } - - currentPath = append(currentPath, pair.SrcMemberID) - return p.isLinked(pair.DstMemberID, dst, reducedSet, currentPath) - } - } - return currentPath, false -} - -func (p *partitionMovements) in(cycle []string, cycles [][]string) bool { - superCycle := make([]string, len(cycle)-1) - for i := 0; i < len(cycle)-1; i++ { - superCycle[i] = cycle[i] - } - superCycle = append(superCycle, cycle...) - for _, foundCycle := range cycles { - if len(foundCycle) == len(cycle) && indexOfSubList(superCycle, foundCycle) != -1 { - return true - } - } - return false -} - -func (p *partitionMovements) hasCycles(pairs []consumerPair) bool { - cycles := make([][]string, 0) - for _, pair := range pairs { - // create a deep copy of the pairs, excluding the current pair - reducedPairs := make([]consumerPair, len(pairs)-1) - i := 0 - for _, p := range pairs { - if p != pair { - reducedPairs[i] = pair - i++ - } - } - if path, linked := p.isLinked(pair.DstMemberID, pair.SrcMemberID, reducedPairs, []string{pair.SrcMemberID}); linked { - if !p.in(path, cycles) { - cycles = append(cycles, path) - Logger.Printf("A cycle of length %d was found: %v", len(path)-1, path) - } - } - } - - // for now we want to make sure there is no partition movements of the same topic between a pair of consumers. - // the odds of finding a cycle among more than two consumers seem to be very low (according to various randomized - // tests with the given sticky algorithm) that it should not worth the added complexity of handling those cases. - for _, cycle := range cycles { - if len(cycle) == 3 { - return true - } - } - return false -} - -func (p *partitionMovements) isSticky() bool { - for topic, movements := range p.PartitionMovementsByTopic { - movementPairs := make([]consumerPair, len(movements)) - i := 0 - for pair := range movements { - movementPairs[i] = pair - i++ - } - if p.hasCycles(movementPairs) { - Logger.Printf("Stickiness is violated for topic %s", topic) - Logger.Printf("Partition movements for this topic occurred among the following consumer pairs: %v", movements) - return false - } - } - return true -} - -func indexOfSubList(source []string, target []string) int { - targetSize := len(target) - maxCandidate := len(source) - targetSize -nextCand: - for candidate := 0; candidate <= maxCandidate; candidate++ { - j := candidate - for i := 0; i < targetSize; i++ { - if target[i] != source[j] { - // Element mismatch, try next cand - continue nextCand - } - j++ - } - // All elements of candidate matched target - return candidate - } - return -1 -} - -type consumerGroupMember struct { - id string - assignments []topicPartitionAssignment -} - -// assignmentPriorityQueue is a priority-queue of consumer group members that is sorted -// in descending order (most assignments to least assignments). -type assignmentPriorityQueue []*consumerGroupMember - -func (pq assignmentPriorityQueue) Len() int { return len(pq) } - -func (pq assignmentPriorityQueue) Less(i, j int) bool { - // order asssignment priority queue in descending order using assignment-count/member-id - if len(pq[i].assignments) == len(pq[j].assignments) { - return strings.Compare(pq[i].id, pq[j].id) > 0 - } - return len(pq[i].assignments) > len(pq[j].assignments) -} - -func (pq assignmentPriorityQueue) Swap(i, j int) { - pq[i], pq[j] = pq[j], pq[i] -} - -func (pq *assignmentPriorityQueue) Push(x interface{}) { - member := x.(*consumerGroupMember) - *pq = append(*pq, member) -} - -func (pq *assignmentPriorityQueue) Pop() interface{} { - old := *pq - n := len(old) - member := old[n-1] - *pq = old[0 : n-1] - return member -} diff --git a/vendor/github.com/Shopify/sarama/broker.go b/vendor/github.com/Shopify/sarama/broker.go deleted file mode 100644 index 5858a23c..00000000 --- a/vendor/github.com/Shopify/sarama/broker.go +++ /dev/null @@ -1,1441 +0,0 @@ -package sarama - -import ( - "crypto/tls" - "encoding/binary" - "fmt" - "io" - "net" - "sort" - "strconv" - "strings" - "sync" - "sync/atomic" - "time" - - "github.com/rcrowley/go-metrics" -) - -// Broker represents a single Kafka broker connection. All operations on this object are entirely concurrency-safe. -type Broker struct { - conf *Config - rack *string - - id int32 - addr string - correlationID int32 - conn net.Conn - connErr error - lock sync.Mutex - opened int32 - responses chan responsePromise - done chan bool - - registeredMetrics []string - - incomingByteRate metrics.Meter - requestRate metrics.Meter - requestSize metrics.Histogram - requestLatency metrics.Histogram - outgoingByteRate metrics.Meter - responseRate metrics.Meter - responseSize metrics.Histogram - requestsInFlight metrics.Counter - brokerIncomingByteRate metrics.Meter - brokerRequestRate metrics.Meter - brokerRequestSize metrics.Histogram - brokerRequestLatency metrics.Histogram - brokerOutgoingByteRate metrics.Meter - brokerResponseRate metrics.Meter - brokerResponseSize metrics.Histogram - brokerRequestsInFlight metrics.Counter - - kerberosAuthenticator GSSAPIKerberosAuth -} - -// SASLMechanism specifies the SASL mechanism the client uses to authenticate with the broker -type SASLMechanism string - -const ( - // SASLTypeOAuth represents the SASL/OAUTHBEARER mechanism (Kafka 2.0.0+) - SASLTypeOAuth = "OAUTHBEARER" - // SASLTypePlaintext represents the SASL/PLAIN mechanism - SASLTypePlaintext = "PLAIN" - // SASLTypeSCRAMSHA256 represents the SCRAM-SHA-256 mechanism. - SASLTypeSCRAMSHA256 = "SCRAM-SHA-256" - // SASLTypeSCRAMSHA512 represents the SCRAM-SHA-512 mechanism. - SASLTypeSCRAMSHA512 = "SCRAM-SHA-512" - SASLTypeGSSAPI = "GSSAPI" - // SASLHandshakeV0 is v0 of the Kafka SASL handshake protocol. Client and - // server negotiate SASL auth using opaque packets. - SASLHandshakeV0 = int16(0) - // SASLHandshakeV1 is v1 of the Kafka SASL handshake protocol. Client and - // server negotiate SASL by wrapping tokens with Kafka protocol headers. - SASLHandshakeV1 = int16(1) - // SASLExtKeyAuth is the reserved extension key name sent as part of the - // SASL/OAUTHBEARER initial client response - SASLExtKeyAuth = "auth" -) - -// AccessToken contains an access token used to authenticate a -// SASL/OAUTHBEARER client along with associated metadata. -type AccessToken struct { - // Token is the access token payload. - Token string - // Extensions is a optional map of arbitrary key-value pairs that can be - // sent with the SASL/OAUTHBEARER initial client response. These values are - // ignored by the SASL server if they are unexpected. This feature is only - // supported by Kafka >= 2.1.0. - Extensions map[string]string -} - -// AccessTokenProvider is the interface that encapsulates how implementors -// can generate access tokens for Kafka broker authentication. -type AccessTokenProvider interface { - // Token returns an access token. The implementation should ensure token - // reuse so that multiple calls at connect time do not create multiple - // tokens. The implementation should also periodically refresh the token in - // order to guarantee that each call returns an unexpired token. This - // method should not block indefinitely--a timeout error should be returned - // after a short period of inactivity so that the broker connection logic - // can log debugging information and retry. - Token() (*AccessToken, error) -} - -// SCRAMClient is a an interface to a SCRAM -// client implementation. -type SCRAMClient interface { - // Begin prepares the client for the SCRAM exchange - // with the server with a user name and a password - Begin(userName, password, authzID string) error - // Step steps client through the SCRAM exchange. It is - // called repeatedly until it errors or `Done` returns true. - Step(challenge string) (response string, err error) - // Done should return true when the SCRAM conversation - // is over. - Done() bool -} - -type responsePromise struct { - requestTime time.Time - correlationID int32 - headerVersion int16 - packets chan []byte - errors chan error -} - -// NewBroker creates and returns a Broker targeting the given host:port address. -// This does not attempt to actually connect, you have to call Open() for that. -func NewBroker(addr string) *Broker { - return &Broker{id: -1, addr: addr} -} - -// Open tries to connect to the Broker if it is not already connected or connecting, but does not block -// waiting for the connection to complete. This means that any subsequent operations on the broker will -// block waiting for the connection to succeed or fail. To get the effect of a fully synchronous Open call, -// follow it by a call to Connected(). The only errors Open will return directly are ConfigurationError or -// AlreadyConnected. If conf is nil, the result of NewConfig() is used. -func (b *Broker) Open(conf *Config) error { - if !atomic.CompareAndSwapInt32(&b.opened, 0, 1) { - return ErrAlreadyConnected - } - - if conf == nil { - conf = NewConfig() - } - - err := conf.Validate() - if err != nil { - return err - } - - b.lock.Lock() - - go withRecover(func() { - defer b.lock.Unlock() - - dialer := conf.getDialer() - b.conn, b.connErr = dialer.Dial("tcp", b.addr) - if b.connErr != nil { - Logger.Printf("Failed to connect to broker %s: %s\n", b.addr, b.connErr) - b.conn = nil - atomic.StoreInt32(&b.opened, 0) - return - } - if conf.Net.TLS.Enable { - b.conn = tls.Client(b.conn, validServerNameTLS(b.addr, conf.Net.TLS.Config)) - } - - b.conn = newBufConn(b.conn) - b.conf = conf - - // Create or reuse the global metrics shared between brokers - b.incomingByteRate = metrics.GetOrRegisterMeter("incoming-byte-rate", conf.MetricRegistry) - b.requestRate = metrics.GetOrRegisterMeter("request-rate", conf.MetricRegistry) - b.requestSize = getOrRegisterHistogram("request-size", conf.MetricRegistry) - b.requestLatency = getOrRegisterHistogram("request-latency-in-ms", conf.MetricRegistry) - b.outgoingByteRate = metrics.GetOrRegisterMeter("outgoing-byte-rate", conf.MetricRegistry) - b.responseRate = metrics.GetOrRegisterMeter("response-rate", conf.MetricRegistry) - b.responseSize = getOrRegisterHistogram("response-size", conf.MetricRegistry) - b.requestsInFlight = metrics.GetOrRegisterCounter("requests-in-flight", conf.MetricRegistry) - // Do not gather metrics for seeded broker (only used during bootstrap) because they share - // the same id (-1) and are already exposed through the global metrics above - if b.id >= 0 { - b.registerMetrics() - } - - if conf.Net.SASL.Enable { - b.connErr = b.authenticateViaSASL() - - if b.connErr != nil { - err = b.conn.Close() - if err == nil { - Logger.Printf("Closed connection to broker %s\n", b.addr) - } else { - Logger.Printf("Error while closing connection to broker %s: %s\n", b.addr, err) - } - b.conn = nil - atomic.StoreInt32(&b.opened, 0) - return - } - } - - b.done = make(chan bool) - b.responses = make(chan responsePromise, b.conf.Net.MaxOpenRequests-1) - - if b.id >= 0 { - Logger.Printf("Connected to broker at %s (registered as #%d)\n", b.addr, b.id) - } else { - Logger.Printf("Connected to broker at %s (unregistered)\n", b.addr) - } - go withRecover(b.responseReceiver) - }) - - return nil -} - -// Connected returns true if the broker is connected and false otherwise. If the broker is not -// connected but it had tried to connect, the error from that connection attempt is also returned. -func (b *Broker) Connected() (bool, error) { - b.lock.Lock() - defer b.lock.Unlock() - - return b.conn != nil, b.connErr -} - -//Close closes the broker resources -func (b *Broker) Close() error { - b.lock.Lock() - defer b.lock.Unlock() - - if b.conn == nil { - return ErrNotConnected - } - - close(b.responses) - <-b.done - - err := b.conn.Close() - - b.conn = nil - b.connErr = nil - b.done = nil - b.responses = nil - - b.unregisterMetrics() - - if err == nil { - Logger.Printf("Closed connection to broker %s\n", b.addr) - } else { - Logger.Printf("Error while closing connection to broker %s: %s\n", b.addr, err) - } - - atomic.StoreInt32(&b.opened, 0) - - return err -} - -// ID returns the broker ID retrieved from Kafka's metadata, or -1 if that is not known. -func (b *Broker) ID() int32 { - return b.id -} - -// Addr returns the broker address as either retrieved from Kafka's metadata or passed to NewBroker. -func (b *Broker) Addr() string { - return b.addr -} - -// Rack returns the broker's rack as retrieved from Kafka's metadata or the -// empty string if it is not known. The returned value corresponds to the -// broker's broker.rack configuration setting. Requires protocol version to be -// at least v0.10.0.0. -func (b *Broker) Rack() string { - if b.rack == nil { - return "" - } - return *b.rack -} - -//GetMetadata send a metadata request and returns a metadata response or error -func (b *Broker) GetMetadata(request *MetadataRequest) (*MetadataResponse, error) { - response := new(MetadataResponse) - - err := b.sendAndReceive(request, response) - - if err != nil { - return nil, err - } - - return response, nil -} - -//GetConsumerMetadata send a consumer metadata request and returns a consumer metadata response or error -func (b *Broker) GetConsumerMetadata(request *ConsumerMetadataRequest) (*ConsumerMetadataResponse, error) { - response := new(ConsumerMetadataResponse) - - err := b.sendAndReceive(request, response) - - if err != nil { - return nil, err - } - - return response, nil -} - -//FindCoordinator sends a find coordinate request and returns a response or error -func (b *Broker) FindCoordinator(request *FindCoordinatorRequest) (*FindCoordinatorResponse, error) { - response := new(FindCoordinatorResponse) - - err := b.sendAndReceive(request, response) - - if err != nil { - return nil, err - } - - return response, nil -} - -//GetAvailableOffsets return an offset response or error -func (b *Broker) GetAvailableOffsets(request *OffsetRequest) (*OffsetResponse, error) { - response := new(OffsetResponse) - - err := b.sendAndReceive(request, response) - - if err != nil { - return nil, err - } - - return response, nil -} - -//Produce returns a produce response or error -func (b *Broker) Produce(request *ProduceRequest) (*ProduceResponse, error) { - var ( - response *ProduceResponse - err error - ) - - if request.RequiredAcks == NoResponse { - err = b.sendAndReceive(request, nil) - } else { - response = new(ProduceResponse) - err = b.sendAndReceive(request, response) - } - - if err != nil { - return nil, err - } - - return response, nil -} - -//Fetch returns a FetchResponse or error -func (b *Broker) Fetch(request *FetchRequest) (*FetchResponse, error) { - response := new(FetchResponse) - - err := b.sendAndReceive(request, response) - if err != nil { - return nil, err - } - - return response, nil -} - -//CommitOffset return an Offset commit response or error -func (b *Broker) CommitOffset(request *OffsetCommitRequest) (*OffsetCommitResponse, error) { - response := new(OffsetCommitResponse) - - err := b.sendAndReceive(request, response) - if err != nil { - return nil, err - } - - return response, nil -} - -//FetchOffset returns an offset fetch response or error -func (b *Broker) FetchOffset(request *OffsetFetchRequest) (*OffsetFetchResponse, error) { - response := new(OffsetFetchResponse) - - err := b.sendAndReceive(request, response) - if err != nil { - return nil, err - } - - return response, nil -} - -//JoinGroup returns a join group response or error -func (b *Broker) JoinGroup(request *JoinGroupRequest) (*JoinGroupResponse, error) { - response := new(JoinGroupResponse) - - err := b.sendAndReceive(request, response) - if err != nil { - return nil, err - } - - return response, nil -} - -//SyncGroup returns a sync group response or error -func (b *Broker) SyncGroup(request *SyncGroupRequest) (*SyncGroupResponse, error) { - response := new(SyncGroupResponse) - - err := b.sendAndReceive(request, response) - if err != nil { - return nil, err - } - - return response, nil -} - -//LeaveGroup return a leave group response or error -func (b *Broker) LeaveGroup(request *LeaveGroupRequest) (*LeaveGroupResponse, error) { - response := new(LeaveGroupResponse) - - err := b.sendAndReceive(request, response) - if err != nil { - return nil, err - } - - return response, nil -} - -//Heartbeat returns a heartbeat response or error -func (b *Broker) Heartbeat(request *HeartbeatRequest) (*HeartbeatResponse, error) { - response := new(HeartbeatResponse) - - err := b.sendAndReceive(request, response) - if err != nil { - return nil, err - } - - return response, nil -} - -//ListGroups return a list group response or error -func (b *Broker) ListGroups(request *ListGroupsRequest) (*ListGroupsResponse, error) { - response := new(ListGroupsResponse) - - err := b.sendAndReceive(request, response) - if err != nil { - return nil, err - } - - return response, nil -} - -//DescribeGroups return describe group response or error -func (b *Broker) DescribeGroups(request *DescribeGroupsRequest) (*DescribeGroupsResponse, error) { - response := new(DescribeGroupsResponse) - - err := b.sendAndReceive(request, response) - if err != nil { - return nil, err - } - - return response, nil -} - -//ApiVersions return api version response or error -func (b *Broker) ApiVersions(request *ApiVersionsRequest) (*ApiVersionsResponse, error) { - response := new(ApiVersionsResponse) - - err := b.sendAndReceive(request, response) - if err != nil { - return nil, err - } - - return response, nil -} - -//CreateTopics send a create topic request and returns create topic response -func (b *Broker) CreateTopics(request *CreateTopicsRequest) (*CreateTopicsResponse, error) { - response := new(CreateTopicsResponse) - - err := b.sendAndReceive(request, response) - if err != nil { - return nil, err - } - - return response, nil -} - -//DeleteTopics sends a delete topic request and returns delete topic response -func (b *Broker) DeleteTopics(request *DeleteTopicsRequest) (*DeleteTopicsResponse, error) { - response := new(DeleteTopicsResponse) - - err := b.sendAndReceive(request, response) - if err != nil { - return nil, err - } - - return response, nil -} - -//CreatePartitions sends a create partition request and returns create -//partitions response or error -func (b *Broker) CreatePartitions(request *CreatePartitionsRequest) (*CreatePartitionsResponse, error) { - response := new(CreatePartitionsResponse) - - err := b.sendAndReceive(request, response) - if err != nil { - return nil, err - } - - return response, nil -} - -//AlterPartitionReassignments sends a alter partition reassignments request and -//returns alter partition reassignments response -func (b *Broker) AlterPartitionReassignments(request *AlterPartitionReassignmentsRequest) (*AlterPartitionReassignmentsResponse, error) { - response := new(AlterPartitionReassignmentsResponse) - - err := b.sendAndReceive(request, response) - if err != nil { - return nil, err - } - - return response, nil -} - -//ListPartitionReassignments sends a list partition reassignments request and -//returns list partition reassignments response -func (b *Broker) ListPartitionReassignments(request *ListPartitionReassignmentsRequest) (*ListPartitionReassignmentsResponse, error) { - response := new(ListPartitionReassignmentsResponse) - - err := b.sendAndReceive(request, response) - if err != nil { - return nil, err - } - - return response, nil -} - -//DeleteRecords send a request to delete records and return delete record -//response or error -func (b *Broker) DeleteRecords(request *DeleteRecordsRequest) (*DeleteRecordsResponse, error) { - response := new(DeleteRecordsResponse) - - err := b.sendAndReceive(request, response) - if err != nil { - return nil, err - } - - return response, nil -} - -//DescribeAcls sends a describe acl request and returns a response or error -func (b *Broker) DescribeAcls(request *DescribeAclsRequest) (*DescribeAclsResponse, error) { - response := new(DescribeAclsResponse) - - err := b.sendAndReceive(request, response) - if err != nil { - return nil, err - } - - return response, nil -} - -//CreateAcls sends a create acl request and returns a response or error -func (b *Broker) CreateAcls(request *CreateAclsRequest) (*CreateAclsResponse, error) { - response := new(CreateAclsResponse) - - err := b.sendAndReceive(request, response) - if err != nil { - return nil, err - } - - return response, nil -} - -//DeleteAcls sends a delete acl request and returns a response or error -func (b *Broker) DeleteAcls(request *DeleteAclsRequest) (*DeleteAclsResponse, error) { - response := new(DeleteAclsResponse) - - err := b.sendAndReceive(request, response) - if err != nil { - return nil, err - } - - return response, nil -} - -//InitProducerID sends an init producer request and returns a response or error -func (b *Broker) InitProducerID(request *InitProducerIDRequest) (*InitProducerIDResponse, error) { - response := new(InitProducerIDResponse) - - err := b.sendAndReceive(request, response) - if err != nil { - return nil, err - } - - return response, nil -} - -//AddPartitionsToTxn send a request to add partition to txn and returns -//a response or error -func (b *Broker) AddPartitionsToTxn(request *AddPartitionsToTxnRequest) (*AddPartitionsToTxnResponse, error) { - response := new(AddPartitionsToTxnResponse) - - err := b.sendAndReceive(request, response) - if err != nil { - return nil, err - } - - return response, nil -} - -//AddOffsetsToTxn sends a request to add offsets to txn and returns a response -//or error -func (b *Broker) AddOffsetsToTxn(request *AddOffsetsToTxnRequest) (*AddOffsetsToTxnResponse, error) { - response := new(AddOffsetsToTxnResponse) - - err := b.sendAndReceive(request, response) - if err != nil { - return nil, err - } - - return response, nil -} - -//EndTxn sends a request to end txn and returns a response or error -func (b *Broker) EndTxn(request *EndTxnRequest) (*EndTxnResponse, error) { - response := new(EndTxnResponse) - - err := b.sendAndReceive(request, response) - if err != nil { - return nil, err - } - - return response, nil -} - -//TxnOffsetCommit sends a request to commit transaction offsets and returns -//a response or error -func (b *Broker) TxnOffsetCommit(request *TxnOffsetCommitRequest) (*TxnOffsetCommitResponse, error) { - response := new(TxnOffsetCommitResponse) - - err := b.sendAndReceive(request, response) - if err != nil { - return nil, err - } - - return response, nil -} - -//DescribeConfigs sends a request to describe config and returns a response or -//error -func (b *Broker) DescribeConfigs(request *DescribeConfigsRequest) (*DescribeConfigsResponse, error) { - response := new(DescribeConfigsResponse) - - err := b.sendAndReceive(request, response) - if err != nil { - return nil, err - } - - return response, nil -} - -//AlterConfigs sends a request to alter config and return a response or error -func (b *Broker) AlterConfigs(request *AlterConfigsRequest) (*AlterConfigsResponse, error) { - response := new(AlterConfigsResponse) - - err := b.sendAndReceive(request, response) - if err != nil { - return nil, err - } - - return response, nil -} - -//DeleteGroups sends a request to delete groups and returns a response or error -func (b *Broker) DeleteGroups(request *DeleteGroupsRequest) (*DeleteGroupsResponse, error) { - response := new(DeleteGroupsResponse) - - if err := b.sendAndReceive(request, response); err != nil { - return nil, err - } - - return response, nil -} - -//DescribeLogDirs sends a request to get the broker's log dir paths and sizes -func (b *Broker) DescribeLogDirs(request *DescribeLogDirsRequest) (*DescribeLogDirsResponse, error) { - response := new(DescribeLogDirsResponse) - - err := b.sendAndReceive(request, response) - if err != nil { - return nil, err - } - - return response, nil -} - -// readFull ensures the conn ReadDeadline has been setup before making a -// call to io.ReadFull -func (b *Broker) readFull(buf []byte) (n int, err error) { - if err := b.conn.SetReadDeadline(time.Now().Add(b.conf.Net.ReadTimeout)); err != nil { - return 0, err - } - - return io.ReadFull(b.conn, buf) -} - -// write ensures the conn WriteDeadline has been setup before making a -// call to conn.Write -func (b *Broker) write(buf []byte) (n int, err error) { - if err := b.conn.SetWriteDeadline(time.Now().Add(b.conf.Net.WriteTimeout)); err != nil { - return 0, err - } - - return b.conn.Write(buf) -} - -func (b *Broker) send(rb protocolBody, promiseResponse bool, responseHeaderVersion int16) (*responsePromise, error) { - b.lock.Lock() - defer b.lock.Unlock() - - if b.conn == nil { - if b.connErr != nil { - return nil, b.connErr - } - return nil, ErrNotConnected - } - - if !b.conf.Version.IsAtLeast(rb.requiredVersion()) { - return nil, ErrUnsupportedVersion - } - - req := &request{correlationID: b.correlationID, clientID: b.conf.ClientID, body: rb} - buf, err := encode(req, b.conf.MetricRegistry) - if err != nil { - return nil, err - } - - requestTime := time.Now() - // Will be decremented in responseReceiver (except error or request with NoResponse) - b.addRequestInFlightMetrics(1) - bytes, err := b.write(buf) - b.updateOutgoingCommunicationMetrics(bytes) - if err != nil { - b.addRequestInFlightMetrics(-1) - return nil, err - } - b.correlationID++ - - if !promiseResponse { - // Record request latency without the response - b.updateRequestLatencyAndInFlightMetrics(time.Since(requestTime)) - return nil, nil - } - - promise := responsePromise{requestTime, req.correlationID, responseHeaderVersion, make(chan []byte), make(chan error)} - b.responses <- promise - - return &promise, nil -} - -func (b *Broker) sendAndReceive(req protocolBody, res protocolBody) error { - responseHeaderVersion := int16(-1) - if res != nil { - responseHeaderVersion = res.headerVersion() - } - - promise, err := b.send(req, res != nil, responseHeaderVersion) - if err != nil { - return err - } - - if promise == nil { - return nil - } - - select { - case buf := <-promise.packets: - return versionedDecode(buf, res, req.version()) - case err = <-promise.errors: - return err - } -} - -func (b *Broker) decode(pd packetDecoder, version int16) (err error) { - b.id, err = pd.getInt32() - if err != nil { - return err - } - - host, err := pd.getString() - if err != nil { - return err - } - - port, err := pd.getInt32() - if err != nil { - return err - } - - if version >= 1 { - b.rack, err = pd.getNullableString() - if err != nil { - return err - } - } - - b.addr = net.JoinHostPort(host, fmt.Sprint(port)) - if _, _, err := net.SplitHostPort(b.addr); err != nil { - return err - } - - return nil -} - -func (b *Broker) encode(pe packetEncoder, version int16) (err error) { - host, portstr, err := net.SplitHostPort(b.addr) - if err != nil { - return err - } - - port, err := strconv.Atoi(portstr) - if err != nil { - return err - } - - pe.putInt32(b.id) - - err = pe.putString(host) - if err != nil { - return err - } - - pe.putInt32(int32(port)) - - if version >= 1 { - err = pe.putNullableString(b.rack) - if err != nil { - return err - } - } - - return nil -} - -func (b *Broker) responseReceiver() { - var dead error - - for response := range b.responses { - if dead != nil { - // This was previously incremented in send() and - // we are not calling updateIncomingCommunicationMetrics() - b.addRequestInFlightMetrics(-1) - response.errors <- dead - continue - } - - var headerLength = getHeaderLength(response.headerVersion) - header := make([]byte, headerLength) - - bytesReadHeader, err := b.readFull(header) - requestLatency := time.Since(response.requestTime) - if err != nil { - b.updateIncomingCommunicationMetrics(bytesReadHeader, requestLatency) - dead = err - response.errors <- err - continue - } - - decodedHeader := responseHeader{} - err = versionedDecode(header, &decodedHeader, response.headerVersion) - if err != nil { - b.updateIncomingCommunicationMetrics(bytesReadHeader, requestLatency) - dead = err - response.errors <- err - continue - } - if decodedHeader.correlationID != response.correlationID { - b.updateIncomingCommunicationMetrics(bytesReadHeader, requestLatency) - // TODO if decoded ID < cur ID, discard until we catch up - // TODO if decoded ID > cur ID, save it so when cur ID catches up we have a response - dead = PacketDecodingError{fmt.Sprintf("correlation ID didn't match, wanted %d, got %d", response.correlationID, decodedHeader.correlationID)} - response.errors <- dead - continue - } - - buf := make([]byte, decodedHeader.length-int32(headerLength)+4) - bytesReadBody, err := b.readFull(buf) - b.updateIncomingCommunicationMetrics(bytesReadHeader+bytesReadBody, requestLatency) - if err != nil { - dead = err - response.errors <- err - continue - } - - response.packets <- buf - } - close(b.done) -} - -func getHeaderLength(headerVersion int16) int8 { - if headerVersion < 1 { - return 8 - } else { - // header contains additional tagged field length (0), we don't support actual tags yet. - return 9 - } -} - -func (b *Broker) authenticateViaSASL() error { - switch b.conf.Net.SASL.Mechanism { - case SASLTypeOAuth: - return b.sendAndReceiveSASLOAuth(b.conf.Net.SASL.TokenProvider) - case SASLTypeSCRAMSHA256, SASLTypeSCRAMSHA512: - return b.sendAndReceiveSASLSCRAMv1() - case SASLTypeGSSAPI: - return b.sendAndReceiveKerberos() - default: - return b.sendAndReceiveSASLPlainAuth() - } -} - -func (b *Broker) sendAndReceiveKerberos() error { - b.kerberosAuthenticator.Config = &b.conf.Net.SASL.GSSAPI - if b.kerberosAuthenticator.NewKerberosClientFunc == nil { - b.kerberosAuthenticator.NewKerberosClientFunc = NewKerberosClient - } - return b.kerberosAuthenticator.Authorize(b) -} - -func (b *Broker) sendAndReceiveSASLHandshake(saslType SASLMechanism, version int16) error { - rb := &SaslHandshakeRequest{Mechanism: string(saslType), Version: version} - - req := &request{correlationID: b.correlationID, clientID: b.conf.ClientID, body: rb} - buf, err := encode(req, b.conf.MetricRegistry) - if err != nil { - return err - } - - requestTime := time.Now() - // Will be decremented in updateIncomingCommunicationMetrics (except error) - b.addRequestInFlightMetrics(1) - bytes, err := b.write(buf) - b.updateOutgoingCommunicationMetrics(bytes) - if err != nil { - b.addRequestInFlightMetrics(-1) - Logger.Printf("Failed to send SASL handshake %s: %s\n", b.addr, err.Error()) - return err - } - b.correlationID++ - - header := make([]byte, 8) // response header - _, err = b.readFull(header) - if err != nil { - b.addRequestInFlightMetrics(-1) - Logger.Printf("Failed to read SASL handshake header : %s\n", err.Error()) - return err - } - - length := binary.BigEndian.Uint32(header[:4]) - payload := make([]byte, length-4) - n, err := b.readFull(payload) - if err != nil { - b.addRequestInFlightMetrics(-1) - Logger.Printf("Failed to read SASL handshake payload : %s\n", err.Error()) - return err - } - - b.updateIncomingCommunicationMetrics(n+8, time.Since(requestTime)) - res := &SaslHandshakeResponse{} - - err = versionedDecode(payload, res, 0) - if err != nil { - Logger.Printf("Failed to parse SASL handshake : %s\n", err.Error()) - return err - } - - if res.Err != ErrNoError { - Logger.Printf("Invalid SASL Mechanism : %s\n", res.Err.Error()) - return res.Err - } - - Logger.Print("Successful SASL handshake. Available mechanisms: ", res.EnabledMechanisms) - return nil -} - -// Kafka 0.10.x supported SASL PLAIN/Kerberos via KAFKA-3149 (KIP-43). -// Kafka 1.x.x onward added a SaslAuthenticate request/response message which -// wraps the SASL flow in the Kafka protocol, which allows for returning -// meaningful errors on authentication failure. -// -// In SASL Plain, Kafka expects the auth header to be in the following format -// Message format (from https://tools.ietf.org/html/rfc4616): -// -// message = [authzid] UTF8NUL authcid UTF8NUL passwd -// authcid = 1*SAFE ; MUST accept up to 255 octets -// authzid = 1*SAFE ; MUST accept up to 255 octets -// passwd = 1*SAFE ; MUST accept up to 255 octets -// UTF8NUL = %x00 ; UTF-8 encoded NUL character -// -// SAFE = UTF1 / UTF2 / UTF3 / UTF4 -// ;; any UTF-8 encoded Unicode character except NUL -// -// With SASL v0 handshake and auth then: -// When credentials are valid, Kafka returns a 4 byte array of null characters. -// When credentials are invalid, Kafka closes the connection. -// -// With SASL v1 handshake and auth then: -// When credentials are invalid, Kafka replies with a SaslAuthenticate response -// containing an error code and message detailing the authentication failure. -func (b *Broker) sendAndReceiveSASLPlainAuth() error { - // default to V0 to allow for backward compatibility when SASL is enabled - // but not the handshake - if b.conf.Net.SASL.Handshake { - handshakeErr := b.sendAndReceiveSASLHandshake(SASLTypePlaintext, b.conf.Net.SASL.Version) - if handshakeErr != nil { - Logger.Printf("Error while performing SASL handshake %s\n", b.addr) - return handshakeErr - } - } - - if b.conf.Net.SASL.Version == SASLHandshakeV1 { - return b.sendAndReceiveV1SASLPlainAuth() - } - return b.sendAndReceiveV0SASLPlainAuth() -} - -// sendAndReceiveV0SASLPlainAuth flows the v0 sasl auth NOT wrapped in the kafka protocol -func (b *Broker) sendAndReceiveV0SASLPlainAuth() error { - length := len(b.conf.Net.SASL.AuthIdentity) + 1 + len(b.conf.Net.SASL.User) + 1 + len(b.conf.Net.SASL.Password) - authBytes := make([]byte, length+4) //4 byte length header + auth data - binary.BigEndian.PutUint32(authBytes, uint32(length)) - copy(authBytes[4:], b.conf.Net.SASL.AuthIdentity+"\x00"+b.conf.Net.SASL.User+"\x00"+b.conf.Net.SASL.Password) - - requestTime := time.Now() - // Will be decremented in updateIncomingCommunicationMetrics (except error) - b.addRequestInFlightMetrics(1) - bytesWritten, err := b.write(authBytes) - b.updateOutgoingCommunicationMetrics(bytesWritten) - if err != nil { - b.addRequestInFlightMetrics(-1) - Logger.Printf("Failed to write SASL auth header to broker %s: %s\n", b.addr, err.Error()) - return err - } - - header := make([]byte, 4) - n, err := b.readFull(header) - b.updateIncomingCommunicationMetrics(n, time.Since(requestTime)) - // If the credentials are valid, we would get a 4 byte response filled with null characters. - // Otherwise, the broker closes the connection and we get an EOF - if err != nil { - Logger.Printf("Failed to read response while authenticating with SASL to broker %s: %s\n", b.addr, err.Error()) - return err - } - - Logger.Printf("SASL authentication successful with broker %s:%v - %v\n", b.addr, n, header) - return nil -} - -// sendAndReceiveV1SASLPlainAuth flows the v1 sasl authentication using the kafka protocol -func (b *Broker) sendAndReceiveV1SASLPlainAuth() error { - correlationID := b.correlationID - - requestTime := time.Now() - - // Will be decremented in updateIncomingCommunicationMetrics (except error) - b.addRequestInFlightMetrics(1) - bytesWritten, err := b.sendSASLPlainAuthClientResponse(correlationID) - b.updateOutgoingCommunicationMetrics(bytesWritten) - - if err != nil { - b.addRequestInFlightMetrics(-1) - Logger.Printf("Failed to write SASL auth header to broker %s: %s\n", b.addr, err.Error()) - return err - } - - b.correlationID++ - - bytesRead, err := b.receiveSASLServerResponse(&SaslAuthenticateResponse{}, correlationID) - b.updateIncomingCommunicationMetrics(bytesRead, time.Since(requestTime)) - - // With v1 sasl we get an error message set in the response we can return - if err != nil { - Logger.Printf("Error returned from broker during SASL flow %s: %s\n", b.addr, err.Error()) - return err - } - - return nil -} - -// sendAndReceiveSASLOAuth performs the authentication flow as described by KIP-255 -// https://cwiki.apache.org/confluence/pages/viewpage.action?pageId=75968876 -func (b *Broker) sendAndReceiveSASLOAuth(provider AccessTokenProvider) error { - if err := b.sendAndReceiveSASLHandshake(SASLTypeOAuth, SASLHandshakeV1); err != nil { - return err - } - - token, err := provider.Token() - if err != nil { - return err - } - - message, err := buildClientFirstMessage(token) - if err != nil { - return err - } - - challenged, err := b.sendClientMessage(message) - if err != nil { - return err - } - - if challenged { - // Abort the token exchange. The broker returns the failure code. - _, err = b.sendClientMessage([]byte(`\x01`)) - } - - return err -} - -// sendClientMessage sends a SASL/OAUTHBEARER client message and returns true -// if the broker responds with a challenge, in which case the token is -// rejected. -func (b *Broker) sendClientMessage(message []byte) (bool, error) { - requestTime := time.Now() - // Will be decremented in updateIncomingCommunicationMetrics (except error) - b.addRequestInFlightMetrics(1) - correlationID := b.correlationID - - bytesWritten, err := b.sendSASLOAuthBearerClientMessage(message, correlationID) - b.updateOutgoingCommunicationMetrics(bytesWritten) - if err != nil { - b.addRequestInFlightMetrics(-1) - return false, err - } - - b.correlationID++ - - res := &SaslAuthenticateResponse{} - bytesRead, err := b.receiveSASLServerResponse(res, correlationID) - - requestLatency := time.Since(requestTime) - b.updateIncomingCommunicationMetrics(bytesRead, requestLatency) - - isChallenge := len(res.SaslAuthBytes) > 0 - - if isChallenge && err != nil { - Logger.Printf("Broker rejected authentication token: %s", res.SaslAuthBytes) - } - - return isChallenge, err -} - -func (b *Broker) sendAndReceiveSASLSCRAMv1() error { - if err := b.sendAndReceiveSASLHandshake(b.conf.Net.SASL.Mechanism, SASLHandshakeV1); err != nil { - return err - } - - scramClient := b.conf.Net.SASL.SCRAMClientGeneratorFunc() - if err := scramClient.Begin(b.conf.Net.SASL.User, b.conf.Net.SASL.Password, b.conf.Net.SASL.SCRAMAuthzID); err != nil { - return fmt.Errorf("failed to start SCRAM exchange with the server: %s", err.Error()) - } - - msg, err := scramClient.Step("") - if err != nil { - return fmt.Errorf("failed to advance the SCRAM exchange: %s", err.Error()) - } - - for !scramClient.Done() { - requestTime := time.Now() - // Will be decremented in updateIncomingCommunicationMetrics (except error) - b.addRequestInFlightMetrics(1) - correlationID := b.correlationID - bytesWritten, err := b.sendSaslAuthenticateRequest(correlationID, []byte(msg)) - b.updateOutgoingCommunicationMetrics(bytesWritten) - if err != nil { - b.addRequestInFlightMetrics(-1) - Logger.Printf("Failed to write SASL auth header to broker %s: %s\n", b.addr, err.Error()) - return err - } - - b.correlationID++ - challenge, err := b.receiveSaslAuthenticateResponse(correlationID) - if err != nil { - b.addRequestInFlightMetrics(-1) - Logger.Printf("Failed to read response while authenticating with SASL to broker %s: %s\n", b.addr, err.Error()) - return err - } - - b.updateIncomingCommunicationMetrics(len(challenge), time.Since(requestTime)) - msg, err = scramClient.Step(string(challenge)) - if err != nil { - Logger.Println("SASL authentication failed", err) - return err - } - } - - Logger.Println("SASL authentication succeeded") - return nil -} - -func (b *Broker) sendSaslAuthenticateRequest(correlationID int32, msg []byte) (int, error) { - rb := &SaslAuthenticateRequest{msg} - req := &request{correlationID: correlationID, clientID: b.conf.ClientID, body: rb} - buf, err := encode(req, b.conf.MetricRegistry) - if err != nil { - return 0, err - } - - return b.write(buf) -} - -func (b *Broker) receiveSaslAuthenticateResponse(correlationID int32) ([]byte, error) { - buf := make([]byte, responseLengthSize+correlationIDSize) - _, err := b.readFull(buf) - if err != nil { - return nil, err - } - - header := responseHeader{} - err = versionedDecode(buf, &header, 0) - if err != nil { - return nil, err - } - - if header.correlationID != correlationID { - return nil, fmt.Errorf("correlation ID didn't match, wanted %d, got %d", b.correlationID, header.correlationID) - } - - buf = make([]byte, header.length-correlationIDSize) - _, err = b.readFull(buf) - if err != nil { - return nil, err - } - - res := &SaslAuthenticateResponse{} - if err := versionedDecode(buf, res, 0); err != nil { - return nil, err - } - if res.Err != ErrNoError { - return nil, res.Err - } - return res.SaslAuthBytes, nil -} - -// Build SASL/OAUTHBEARER initial client response as described by RFC-7628 -// https://tools.ietf.org/html/rfc7628 -func buildClientFirstMessage(token *AccessToken) ([]byte, error) { - var ext string - - if token.Extensions != nil && len(token.Extensions) > 0 { - if _, ok := token.Extensions[SASLExtKeyAuth]; ok { - return []byte{}, fmt.Errorf("the extension `%s` is invalid", SASLExtKeyAuth) - } - ext = "\x01" + mapToString(token.Extensions, "=", "\x01") - } - - resp := []byte(fmt.Sprintf("n,,\x01auth=Bearer %s%s\x01\x01", token.Token, ext)) - - return resp, nil -} - -// mapToString returns a list of key-value pairs ordered by key. -// keyValSep separates the key from the value. elemSep separates each pair. -func mapToString(extensions map[string]string, keyValSep string, elemSep string) string { - buf := make([]string, 0, len(extensions)) - - for k, v := range extensions { - buf = append(buf, k+keyValSep+v) - } - - sort.Strings(buf) - - return strings.Join(buf, elemSep) -} - -func (b *Broker) sendSASLPlainAuthClientResponse(correlationID int32) (int, error) { - authBytes := []byte(b.conf.Net.SASL.AuthIdentity + "\x00" + b.conf.Net.SASL.User + "\x00" + b.conf.Net.SASL.Password) - rb := &SaslAuthenticateRequest{authBytes} - req := &request{correlationID: correlationID, clientID: b.conf.ClientID, body: rb} - buf, err := encode(req, b.conf.MetricRegistry) - if err != nil { - return 0, err - } - - return b.write(buf) -} - -func (b *Broker) sendSASLOAuthBearerClientMessage(initialResp []byte, correlationID int32) (int, error) { - rb := &SaslAuthenticateRequest{initialResp} - - req := &request{correlationID: correlationID, clientID: b.conf.ClientID, body: rb} - - buf, err := encode(req, b.conf.MetricRegistry) - if err != nil { - return 0, err - } - - return b.write(buf) -} - -func (b *Broker) receiveSASLServerResponse(res *SaslAuthenticateResponse, correlationID int32) (int, error) { - buf := make([]byte, responseLengthSize+correlationIDSize) - bytesRead, err := b.readFull(buf) - if err != nil { - return bytesRead, err - } - - header := responseHeader{} - err = versionedDecode(buf, &header, 0) - if err != nil { - return bytesRead, err - } - - if header.correlationID != correlationID { - return bytesRead, fmt.Errorf("correlation ID didn't match, wanted %d, got %d", b.correlationID, header.correlationID) - } - - buf = make([]byte, header.length-correlationIDSize) - c, err := b.readFull(buf) - bytesRead += c - if err != nil { - return bytesRead, err - } - - if err := versionedDecode(buf, res, 0); err != nil { - return bytesRead, err - } - - if res.Err != ErrNoError { - return bytesRead, res.Err - } - - return bytesRead, nil -} - -func (b *Broker) updateIncomingCommunicationMetrics(bytes int, requestLatency time.Duration) { - b.updateRequestLatencyAndInFlightMetrics(requestLatency) - b.responseRate.Mark(1) - - if b.brokerResponseRate != nil { - b.brokerResponseRate.Mark(1) - } - - responseSize := int64(bytes) - b.incomingByteRate.Mark(responseSize) - if b.brokerIncomingByteRate != nil { - b.brokerIncomingByteRate.Mark(responseSize) - } - - b.responseSize.Update(responseSize) - if b.brokerResponseSize != nil { - b.brokerResponseSize.Update(responseSize) - } -} - -func (b *Broker) updateRequestLatencyAndInFlightMetrics(requestLatency time.Duration) { - requestLatencyInMs := int64(requestLatency / time.Millisecond) - b.requestLatency.Update(requestLatencyInMs) - - if b.brokerRequestLatency != nil { - b.brokerRequestLatency.Update(requestLatencyInMs) - } - - b.addRequestInFlightMetrics(-1) -} - -func (b *Broker) addRequestInFlightMetrics(i int64) { - b.requestsInFlight.Inc(i) - if b.brokerRequestsInFlight != nil { - b.brokerRequestsInFlight.Inc(i) - } -} - -func (b *Broker) updateOutgoingCommunicationMetrics(bytes int) { - b.requestRate.Mark(1) - if b.brokerRequestRate != nil { - b.brokerRequestRate.Mark(1) - } - - requestSize := int64(bytes) - b.outgoingByteRate.Mark(requestSize) - if b.brokerOutgoingByteRate != nil { - b.brokerOutgoingByteRate.Mark(requestSize) - } - - b.requestSize.Update(requestSize) - if b.brokerRequestSize != nil { - b.brokerRequestSize.Update(requestSize) - } -} - -func (b *Broker) registerMetrics() { - b.brokerIncomingByteRate = b.registerMeter("incoming-byte-rate") - b.brokerRequestRate = b.registerMeter("request-rate") - b.brokerRequestSize = b.registerHistogram("request-size") - b.brokerRequestLatency = b.registerHistogram("request-latency-in-ms") - b.brokerOutgoingByteRate = b.registerMeter("outgoing-byte-rate") - b.brokerResponseRate = b.registerMeter("response-rate") - b.brokerResponseSize = b.registerHistogram("response-size") - b.brokerRequestsInFlight = b.registerCounter("requests-in-flight") -} - -func (b *Broker) unregisterMetrics() { - for _, name := range b.registeredMetrics { - b.conf.MetricRegistry.Unregister(name) - } - b.registeredMetrics = nil -} - -func (b *Broker) registerMeter(name string) metrics.Meter { - nameForBroker := getMetricNameForBroker(name, b) - b.registeredMetrics = append(b.registeredMetrics, nameForBroker) - return metrics.GetOrRegisterMeter(nameForBroker, b.conf.MetricRegistry) -} - -func (b *Broker) registerHistogram(name string) metrics.Histogram { - nameForBroker := getMetricNameForBroker(name, b) - b.registeredMetrics = append(b.registeredMetrics, nameForBroker) - return getOrRegisterHistogram(nameForBroker, b.conf.MetricRegistry) -} - -func (b *Broker) registerCounter(name string) metrics.Counter { - nameForBroker := getMetricNameForBroker(name, b) - b.registeredMetrics = append(b.registeredMetrics, nameForBroker) - return metrics.GetOrRegisterCounter(nameForBroker, b.conf.MetricRegistry) -} - -func validServerNameTLS(addr string, cfg *tls.Config) *tls.Config { - if cfg == nil { - cfg = &tls.Config{} - } - if cfg.ServerName != "" { - return cfg - } - - c := cfg.Clone() - sn, _, err := net.SplitHostPort(addr) - if err != nil { - Logger.Println(fmt.Errorf("failed to get ServerName from addr %w", err)) - } - c.ServerName = sn - return c -} diff --git a/vendor/github.com/Shopify/sarama/client.go b/vendor/github.com/Shopify/sarama/client.go deleted file mode 100644 index b2951934..00000000 --- a/vendor/github.com/Shopify/sarama/client.go +++ /dev/null @@ -1,1070 +0,0 @@ -package sarama - -import ( - "math/rand" - "sort" - "sync" - "time" -) - -// Client is a generic Kafka client. It manages connections to one or more Kafka brokers. -// You MUST call Close() on a client to avoid leaks, it will not be garbage-collected -// automatically when it passes out of scope. It is safe to share a client amongst many -// users, however Kafka will process requests from a single client strictly in serial, -// so it is generally more efficient to use the default one client per producer/consumer. -type Client interface { - // Config returns the Config struct of the client. This struct should not be - // altered after it has been created. - Config() *Config - - // Controller returns the cluster controller broker. It will return a - // locally cached value if it's available. You can call RefreshController - // to update the cached value. Requires Kafka 0.10 or higher. - Controller() (*Broker, error) - - // RefreshController retrieves the cluster controller from fresh metadata - // and stores it in the local cache. Requires Kafka 0.10 or higher. - RefreshController() (*Broker, error) - - // Brokers returns the current set of active brokers as retrieved from cluster metadata. - Brokers() []*Broker - - // Topics returns the set of available topics as retrieved from cluster metadata. - Topics() ([]string, error) - - // Partitions returns the sorted list of all partition IDs for the given topic. - Partitions(topic string) ([]int32, error) - - // WritablePartitions returns the sorted list of all writable partition IDs for - // the given topic, where "writable" means "having a valid leader accepting - // writes". - WritablePartitions(topic string) ([]int32, error) - - // Leader returns the broker object that is the leader of the current - // topic/partition, as determined by querying the cluster metadata. - Leader(topic string, partitionID int32) (*Broker, error) - - // Replicas returns the set of all replica IDs for the given partition. - Replicas(topic string, partitionID int32) ([]int32, error) - - // InSyncReplicas returns the set of all in-sync replica IDs for the given - // partition. In-sync replicas are replicas which are fully caught up with - // the partition leader. - InSyncReplicas(topic string, partitionID int32) ([]int32, error) - - // OfflineReplicas returns the set of all offline replica IDs for the given - // partition. Offline replicas are replicas which are offline - OfflineReplicas(topic string, partitionID int32) ([]int32, error) - - // RefreshMetadata takes a list of topics and queries the cluster to refresh the - // available metadata for those topics. If no topics are provided, it will refresh - // metadata for all topics. - RefreshMetadata(topics ...string) error - - // GetOffset queries the cluster to get the most recent available offset at the - // given time (in milliseconds) on the topic/partition combination. - // Time should be OffsetOldest for the earliest available offset, - // OffsetNewest for the offset of the message that will be produced next, or a time. - GetOffset(topic string, partitionID int32, time int64) (int64, error) - - // Coordinator returns the coordinating broker for a consumer group. It will - // return a locally cached value if it's available. You can call - // RefreshCoordinator to update the cached value. This function only works on - // Kafka 0.8.2 and higher. - Coordinator(consumerGroup string) (*Broker, error) - - // RefreshCoordinator retrieves the coordinator for a consumer group and stores it - // in local cache. This function only works on Kafka 0.8.2 and higher. - RefreshCoordinator(consumerGroup string) error - - // InitProducerID retrieves information required for Idempotent Producer - InitProducerID() (*InitProducerIDResponse, error) - - // Close shuts down all broker connections managed by this client. It is required - // to call this function before a client object passes out of scope, as it will - // otherwise leak memory. You must close any Producers or Consumers using a client - // before you close the client. - Close() error - - // Closed returns true if the client has already had Close called on it - Closed() bool -} - -const ( - // OffsetNewest stands for the log head offset, i.e. the offset that will be - // assigned to the next message that will be produced to the partition. You - // can send this to a client's GetOffset method to get this offset, or when - // calling ConsumePartition to start consuming new messages. - OffsetNewest int64 = -1 - // OffsetOldest stands for the oldest offset available on the broker for a - // partition. You can send this to a client's GetOffset method to get this - // offset, or when calling ConsumePartition to start consuming from the - // oldest offset that is still available on the broker. - OffsetOldest int64 = -2 -) - -type client struct { - conf *Config - closer, closed chan none // for shutting down background metadata updater - - // the broker addresses given to us through the constructor are not guaranteed to be returned in - // the cluster metadata (I *think* it only returns brokers who are currently leading partitions?) - // so we store them separately - seedBrokers []*Broker - deadSeeds []*Broker - - controllerID int32 // cluster controller broker id - brokers map[int32]*Broker // maps broker ids to brokers - metadata map[string]map[int32]*PartitionMetadata // maps topics to partition ids to metadata - metadataTopics map[string]none // topics that need to collect metadata - coordinators map[string]int32 // Maps consumer group names to coordinating broker IDs - - // If the number of partitions is large, we can get some churn calling cachedPartitions, - // so the result is cached. It is important to update this value whenever metadata is changed - cachedPartitionsResults map[string][maxPartitionIndex][]int32 - - lock sync.RWMutex // protects access to the maps that hold cluster state. -} - -// NewClient creates a new Client. It connects to one of the given broker addresses -// and uses that broker to automatically fetch metadata on the rest of the kafka cluster. If metadata cannot -// be retrieved from any of the given broker addresses, the client is not created. -func NewClient(addrs []string, conf *Config) (Client, error) { - Logger.Println("Initializing new client") - - if conf == nil { - conf = NewConfig() - } - - if err := conf.Validate(); err != nil { - return nil, err - } - - if len(addrs) < 1 { - return nil, ConfigurationError("You must provide at least one broker address") - } - - client := &client{ - conf: conf, - closer: make(chan none), - closed: make(chan none), - brokers: make(map[int32]*Broker), - metadata: make(map[string]map[int32]*PartitionMetadata), - metadataTopics: make(map[string]none), - cachedPartitionsResults: make(map[string][maxPartitionIndex][]int32), - coordinators: make(map[string]int32), - } - - random := rand.New(rand.NewSource(time.Now().UnixNano())) - for _, index := range random.Perm(len(addrs)) { - client.seedBrokers = append(client.seedBrokers, NewBroker(addrs[index])) - } - - if conf.Metadata.Full { - // do an initial fetch of all cluster metadata by specifying an empty list of topics - err := client.RefreshMetadata() - switch err { - case nil: - break - case ErrLeaderNotAvailable, ErrReplicaNotAvailable, ErrTopicAuthorizationFailed, ErrClusterAuthorizationFailed: - // indicates that maybe part of the cluster is down, but is not fatal to creating the client - Logger.Println(err) - default: - close(client.closed) // we haven't started the background updater yet, so we have to do this manually - _ = client.Close() - return nil, err - } - } - go withRecover(client.backgroundMetadataUpdater) - - Logger.Println("Successfully initialized new client") - - return client, nil -} - -func (client *client) Config() *Config { - return client.conf -} - -func (client *client) Brokers() []*Broker { - client.lock.RLock() - defer client.lock.RUnlock() - brokers := make([]*Broker, 0, len(client.brokers)) - for _, broker := range client.brokers { - brokers = append(brokers, broker) - } - return brokers -} - -func (client *client) InitProducerID() (*InitProducerIDResponse, error) { - var err error - for broker := client.any(); broker != nil; broker = client.any() { - req := &InitProducerIDRequest{} - - response, err := broker.InitProducerID(req) - switch err.(type) { - case nil: - return response, nil - default: - // some error, remove that broker and try again - Logger.Printf("Client got error from broker %d when issuing InitProducerID : %v\n", broker.ID(), err) - _ = broker.Close() - client.deregisterBroker(broker) - } - } - return nil, err -} - -func (client *client) Close() error { - if client.Closed() { - // Chances are this is being called from a defer() and the error will go unobserved - // so we go ahead and log the event in this case. - Logger.Printf("Close() called on already closed client") - return ErrClosedClient - } - - // shutdown and wait for the background thread before we take the lock, to avoid races - close(client.closer) - <-client.closed - - client.lock.Lock() - defer client.lock.Unlock() - Logger.Println("Closing Client") - - for _, broker := range client.brokers { - safeAsyncClose(broker) - } - - for _, broker := range client.seedBrokers { - safeAsyncClose(broker) - } - - client.brokers = nil - client.metadata = nil - client.metadataTopics = nil - - return nil -} - -func (client *client) Closed() bool { - client.lock.RLock() - defer client.lock.RUnlock() - - return client.brokers == nil -} - -func (client *client) Topics() ([]string, error) { - if client.Closed() { - return nil, ErrClosedClient - } - - client.lock.RLock() - defer client.lock.RUnlock() - - ret := make([]string, 0, len(client.metadata)) - for topic := range client.metadata { - ret = append(ret, topic) - } - - return ret, nil -} - -func (client *client) MetadataTopics() ([]string, error) { - if client.Closed() { - return nil, ErrClosedClient - } - - client.lock.RLock() - defer client.lock.RUnlock() - - ret := make([]string, 0, len(client.metadataTopics)) - for topic := range client.metadataTopics { - ret = append(ret, topic) - } - - return ret, nil -} - -func (client *client) Partitions(topic string) ([]int32, error) { - if client.Closed() { - return nil, ErrClosedClient - } - - partitions := client.cachedPartitions(topic, allPartitions) - - if len(partitions) == 0 { - err := client.RefreshMetadata(topic) - if err != nil { - return nil, err - } - partitions = client.cachedPartitions(topic, allPartitions) - } - - // no partitions found after refresh metadata - if len(partitions) == 0 { - return nil, ErrUnknownTopicOrPartition - } - - return partitions, nil -} - -func (client *client) WritablePartitions(topic string) ([]int32, error) { - if client.Closed() { - return nil, ErrClosedClient - } - - partitions := client.cachedPartitions(topic, writablePartitions) - - // len==0 catches when it's nil (no such topic) and the odd case when every single - // partition is undergoing leader election simultaneously. Callers have to be able to handle - // this function returning an empty slice (which is a valid return value) but catching it - // here the first time (note we *don't* catch it below where we return ErrUnknownTopicOrPartition) triggers - // a metadata refresh as a nicety so callers can just try again and don't have to manually - // trigger a refresh (otherwise they'd just keep getting a stale cached copy). - if len(partitions) == 0 { - err := client.RefreshMetadata(topic) - if err != nil { - return nil, err - } - partitions = client.cachedPartitions(topic, writablePartitions) - } - - if partitions == nil { - return nil, ErrUnknownTopicOrPartition - } - - return partitions, nil -} - -func (client *client) Replicas(topic string, partitionID int32) ([]int32, error) { - if client.Closed() { - return nil, ErrClosedClient - } - - metadata := client.cachedMetadata(topic, partitionID) - - if metadata == nil { - err := client.RefreshMetadata(topic) - if err != nil { - return nil, err - } - metadata = client.cachedMetadata(topic, partitionID) - } - - if metadata == nil { - return nil, ErrUnknownTopicOrPartition - } - - if metadata.Err == ErrReplicaNotAvailable { - return dupInt32Slice(metadata.Replicas), metadata.Err - } - return dupInt32Slice(metadata.Replicas), nil -} - -func (client *client) InSyncReplicas(topic string, partitionID int32) ([]int32, error) { - if client.Closed() { - return nil, ErrClosedClient - } - - metadata := client.cachedMetadata(topic, partitionID) - - if metadata == nil { - err := client.RefreshMetadata(topic) - if err != nil { - return nil, err - } - metadata = client.cachedMetadata(topic, partitionID) - } - - if metadata == nil { - return nil, ErrUnknownTopicOrPartition - } - - if metadata.Err == ErrReplicaNotAvailable { - return dupInt32Slice(metadata.Isr), metadata.Err - } - return dupInt32Slice(metadata.Isr), nil -} - -func (client *client) OfflineReplicas(topic string, partitionID int32) ([]int32, error) { - if client.Closed() { - return nil, ErrClosedClient - } - - metadata := client.cachedMetadata(topic, partitionID) - - if metadata == nil { - err := client.RefreshMetadata(topic) - if err != nil { - return nil, err - } - metadata = client.cachedMetadata(topic, partitionID) - } - - if metadata == nil { - return nil, ErrUnknownTopicOrPartition - } - - if metadata.Err == ErrReplicaNotAvailable { - return dupInt32Slice(metadata.OfflineReplicas), metadata.Err - } - return dupInt32Slice(metadata.OfflineReplicas), nil -} - -func (client *client) Leader(topic string, partitionID int32) (*Broker, error) { - if client.Closed() { - return nil, ErrClosedClient - } - - leader, err := client.cachedLeader(topic, partitionID) - - if leader == nil { - err = client.RefreshMetadata(topic) - if err != nil { - return nil, err - } - leader, err = client.cachedLeader(topic, partitionID) - } - - return leader, err -} - -func (client *client) RefreshMetadata(topics ...string) error { - if client.Closed() { - return ErrClosedClient - } - - // Prior to 0.8.2, Kafka will throw exceptions on an empty topic and not return a proper - // error. This handles the case by returning an error instead of sending it - // off to Kafka. See: https://github.com/Shopify/sarama/pull/38#issuecomment-26362310 - for _, topic := range topics { - if len(topic) == 0 { - return ErrInvalidTopic // this is the error that 0.8.2 and later correctly return - } - } - - deadline := time.Time{} - if client.conf.Metadata.Timeout > 0 { - deadline = time.Now().Add(client.conf.Metadata.Timeout) - } - return client.tryRefreshMetadata(topics, client.conf.Metadata.Retry.Max, deadline) -} - -func (client *client) GetOffset(topic string, partitionID int32, time int64) (int64, error) { - if client.Closed() { - return -1, ErrClosedClient - } - - offset, err := client.getOffset(topic, partitionID, time) - - if err != nil { - if err := client.RefreshMetadata(topic); err != nil { - return -1, err - } - return client.getOffset(topic, partitionID, time) - } - - return offset, err -} - -func (client *client) Controller() (*Broker, error) { - if client.Closed() { - return nil, ErrClosedClient - } - - if !client.conf.Version.IsAtLeast(V0_10_0_0) { - return nil, ErrUnsupportedVersion - } - - controller := client.cachedController() - if controller == nil { - if err := client.refreshMetadata(); err != nil { - return nil, err - } - controller = client.cachedController() - } - - if controller == nil { - return nil, ErrControllerNotAvailable - } - - _ = controller.Open(client.conf) - return controller, nil -} - -// deregisterController removes the cached controllerID -func (client *client) deregisterController() { - client.lock.Lock() - defer client.lock.Unlock() - delete(client.brokers, client.controllerID) -} - -// RefreshController retrieves the cluster controller from fresh metadata -// and stores it in the local cache. Requires Kafka 0.10 or higher. -func (client *client) RefreshController() (*Broker, error) { - if client.Closed() { - return nil, ErrClosedClient - } - - client.deregisterController() - - if err := client.refreshMetadata(); err != nil { - return nil, err - } - - controller := client.cachedController() - if controller == nil { - return nil, ErrControllerNotAvailable - } - - _ = controller.Open(client.conf) - return controller, nil -} - -func (client *client) Coordinator(consumerGroup string) (*Broker, error) { - if client.Closed() { - return nil, ErrClosedClient - } - - coordinator := client.cachedCoordinator(consumerGroup) - - if coordinator == nil { - if err := client.RefreshCoordinator(consumerGroup); err != nil { - return nil, err - } - coordinator = client.cachedCoordinator(consumerGroup) - } - - if coordinator == nil { - return nil, ErrConsumerCoordinatorNotAvailable - } - - _ = coordinator.Open(client.conf) - return coordinator, nil -} - -func (client *client) RefreshCoordinator(consumerGroup string) error { - if client.Closed() { - return ErrClosedClient - } - - response, err := client.getConsumerMetadata(consumerGroup, client.conf.Metadata.Retry.Max) - if err != nil { - return err - } - - client.lock.Lock() - defer client.lock.Unlock() - client.registerBroker(response.Coordinator) - client.coordinators[consumerGroup] = response.Coordinator.ID() - return nil -} - -// private broker management helpers - -func (client *client) updateBroker(brokers []*Broker) { - var currentBroker = make(map[int32]*Broker, len(brokers)) - - for _, broker := range brokers { - currentBroker[broker.ID()] = broker - if client.brokers[broker.ID()] == nil { // add new broker - client.brokers[broker.ID()] = broker - Logger.Printf("client/brokers registered new broker #%d at %s", broker.ID(), broker.Addr()) - } else if broker.Addr() != client.brokers[broker.ID()].Addr() { // replace broker with new address - safeAsyncClose(client.brokers[broker.ID()]) - client.brokers[broker.ID()] = broker - Logger.Printf("client/brokers replaced registered broker #%d with %s", broker.ID(), broker.Addr()) - } - } - - for id, broker := range client.brokers { - if _, exist := currentBroker[id]; !exist { // remove old broker - safeAsyncClose(broker) - delete(client.brokers, id) - Logger.Printf("client/broker remove invalid broker #%d with %s", broker.ID(), broker.Addr()) - } - } -} - -// registerBroker makes sure a broker received by a Metadata or Coordinator request is registered -// in the brokers map. It returns the broker that is registered, which may be the provided broker, -// or a previously registered Broker instance. You must hold the write lock before calling this function. -func (client *client) registerBroker(broker *Broker) { - if client.brokers == nil { - Logger.Printf("cannot register broker #%d at %s, client already closed", broker.ID(), broker.Addr()) - return - } - - if client.brokers[broker.ID()] == nil { - client.brokers[broker.ID()] = broker - Logger.Printf("client/brokers registered new broker #%d at %s", broker.ID(), broker.Addr()) - } else if broker.Addr() != client.brokers[broker.ID()].Addr() { - safeAsyncClose(client.brokers[broker.ID()]) - client.brokers[broker.ID()] = broker - Logger.Printf("client/brokers replaced registered broker #%d with %s", broker.ID(), broker.Addr()) - } -} - -// deregisterBroker removes a broker from the seedsBroker list, and if it's -// not the seedbroker, removes it from brokers map completely. -func (client *client) deregisterBroker(broker *Broker) { - client.lock.Lock() - defer client.lock.Unlock() - - if len(client.seedBrokers) > 0 && broker == client.seedBrokers[0] { - client.deadSeeds = append(client.deadSeeds, broker) - client.seedBrokers = client.seedBrokers[1:] - } else { - // we do this so that our loop in `tryRefreshMetadata` doesn't go on forever, - // but we really shouldn't have to; once that loop is made better this case can be - // removed, and the function generally can be renamed from `deregisterBroker` to - // `nextSeedBroker` or something - Logger.Printf("client/brokers deregistered broker #%d at %s", broker.ID(), broker.Addr()) - delete(client.brokers, broker.ID()) - } -} - -func (client *client) resurrectDeadBrokers() { - client.lock.Lock() - defer client.lock.Unlock() - - Logger.Printf("client/brokers resurrecting %d dead seed brokers", len(client.deadSeeds)) - client.seedBrokers = append(client.seedBrokers, client.deadSeeds...) - client.deadSeeds = nil -} - -func (client *client) any() *Broker { - client.lock.RLock() - defer client.lock.RUnlock() - - if len(client.seedBrokers) > 0 { - _ = client.seedBrokers[0].Open(client.conf) - return client.seedBrokers[0] - } - - // not guaranteed to be random *or* deterministic - for _, broker := range client.brokers { - _ = broker.Open(client.conf) - return broker - } - - return nil -} - -// private caching/lazy metadata helpers - -type partitionType int - -const ( - allPartitions partitionType = iota - writablePartitions - // If you add any more types, update the partition cache in update() - - // Ensure this is the last partition type value - maxPartitionIndex -) - -func (client *client) cachedMetadata(topic string, partitionID int32) *PartitionMetadata { - client.lock.RLock() - defer client.lock.RUnlock() - - partitions := client.metadata[topic] - if partitions != nil { - return partitions[partitionID] - } - - return nil -} - -func (client *client) cachedPartitions(topic string, partitionSet partitionType) []int32 { - client.lock.RLock() - defer client.lock.RUnlock() - - partitions, exists := client.cachedPartitionsResults[topic] - - if !exists { - return nil - } - return partitions[partitionSet] -} - -func (client *client) setPartitionCache(topic string, partitionSet partitionType) []int32 { - partitions := client.metadata[topic] - - if partitions == nil { - return nil - } - - ret := make([]int32, 0, len(partitions)) - for _, partition := range partitions { - if partitionSet == writablePartitions && partition.Err == ErrLeaderNotAvailable { - continue - } - ret = append(ret, partition.ID) - } - - sort.Sort(int32Slice(ret)) - return ret -} - -func (client *client) cachedLeader(topic string, partitionID int32) (*Broker, error) { - client.lock.RLock() - defer client.lock.RUnlock() - - partitions := client.metadata[topic] - if partitions != nil { - metadata, ok := partitions[partitionID] - if ok { - if metadata.Err == ErrLeaderNotAvailable { - return nil, ErrLeaderNotAvailable - } - b := client.brokers[metadata.Leader] - if b == nil { - return nil, ErrLeaderNotAvailable - } - _ = b.Open(client.conf) - return b, nil - } - } - - return nil, ErrUnknownTopicOrPartition -} - -func (client *client) getOffset(topic string, partitionID int32, time int64) (int64, error) { - broker, err := client.Leader(topic, partitionID) - if err != nil { - return -1, err - } - - request := &OffsetRequest{} - if client.conf.Version.IsAtLeast(V0_10_1_0) { - request.Version = 1 - } - request.AddBlock(topic, partitionID, time, 1) - - response, err := broker.GetAvailableOffsets(request) - if err != nil { - _ = broker.Close() - return -1, err - } - - block := response.GetBlock(topic, partitionID) - if block == nil { - _ = broker.Close() - return -1, ErrIncompleteResponse - } - if block.Err != ErrNoError { - return -1, block.Err - } - if len(block.Offsets) != 1 { - return -1, ErrOffsetOutOfRange - } - - return block.Offsets[0], nil -} - -// core metadata update logic - -func (client *client) backgroundMetadataUpdater() { - defer close(client.closed) - - if client.conf.Metadata.RefreshFrequency == time.Duration(0) { - return - } - - ticker := time.NewTicker(client.conf.Metadata.RefreshFrequency) - defer ticker.Stop() - - for { - select { - case <-ticker.C: - if err := client.refreshMetadata(); err != nil { - Logger.Println("Client background metadata update:", err) - } - case <-client.closer: - return - } - } -} - -func (client *client) refreshMetadata() error { - var topics []string - - if !client.conf.Metadata.Full { - if specificTopics, err := client.MetadataTopics(); err != nil { - return err - } else if len(specificTopics) == 0 { - return ErrNoTopicsToUpdateMetadata - } else { - topics = specificTopics - } - } - - if err := client.RefreshMetadata(topics...); err != nil { - return err - } - - return nil -} - -func (client *client) tryRefreshMetadata(topics []string, attemptsRemaining int, deadline time.Time) error { - pastDeadline := func(backoff time.Duration) bool { - if !deadline.IsZero() && time.Now().Add(backoff).After(deadline) { - // we are past the deadline - return true - } - return false - } - retry := func(err error) error { - if attemptsRemaining > 0 { - backoff := client.computeBackoff(attemptsRemaining) - if pastDeadline(backoff) { - Logger.Println("client/metadata skipping last retries as we would go past the metadata timeout") - return err - } - Logger.Printf("client/metadata retrying after %dms... (%d attempts remaining)\n", backoff/time.Millisecond, attemptsRemaining) - if backoff > 0 { - time.Sleep(backoff) - } - return client.tryRefreshMetadata(topics, attemptsRemaining-1, deadline) - } - return err - } - - broker := client.any() - for ; broker != nil && !pastDeadline(0); broker = client.any() { - allowAutoTopicCreation := true - if len(topics) > 0 { - Logger.Printf("client/metadata fetching metadata for %v from broker %s\n", topics, broker.addr) - } else { - allowAutoTopicCreation = false - Logger.Printf("client/metadata fetching metadata for all topics from broker %s\n", broker.addr) - } - - req := &MetadataRequest{Topics: topics, AllowAutoTopicCreation: allowAutoTopicCreation} - if client.conf.Version.IsAtLeast(V1_0_0_0) { - req.Version = 5 - } else if client.conf.Version.IsAtLeast(V0_10_0_0) { - req.Version = 1 - } - response, err := broker.GetMetadata(req) - switch err.(type) { - case nil: - allKnownMetaData := len(topics) == 0 - // valid response, use it - shouldRetry, err := client.updateMetadata(response, allKnownMetaData) - if shouldRetry { - Logger.Println("client/metadata found some partitions to be leaderless") - return retry(err) // note: err can be nil - } - return err - - case PacketEncodingError: - // didn't even send, return the error - return err - - case KError: - // if SASL auth error return as this _should_ be a non retryable err for all brokers - if err.(KError) == ErrSASLAuthenticationFailed { - Logger.Println("client/metadata failed SASL authentication") - return err - } - - if err.(KError) == ErrTopicAuthorizationFailed { - Logger.Println("client is not authorized to access this topic. The topics were: ", topics) - return err - } - // else remove that broker and try again - Logger.Printf("client/metadata got error from broker %d while fetching metadata: %v\n", broker.ID(), err) - _ = broker.Close() - client.deregisterBroker(broker) - - default: - // some other error, remove that broker and try again - Logger.Printf("client/metadata got error from broker %d while fetching metadata: %v\n", broker.ID(), err) - _ = broker.Close() - client.deregisterBroker(broker) - } - } - - if broker != nil { - Logger.Printf("client/metadata not fetching metadata from broker %s as we would go past the metadata timeout\n", broker.addr) - return retry(ErrOutOfBrokers) - } - - Logger.Println("client/metadata no available broker to send metadata request to") - client.resurrectDeadBrokers() - return retry(ErrOutOfBrokers) -} - -// if no fatal error, returns a list of topics that need retrying due to ErrLeaderNotAvailable -func (client *client) updateMetadata(data *MetadataResponse, allKnownMetaData bool) (retry bool, err error) { - if client.Closed() { - return - } - - client.lock.Lock() - defer client.lock.Unlock() - - // For all the brokers we received: - // - if it is a new ID, save it - // - if it is an existing ID, but the address we have is stale, discard the old one and save it - // - if some brokers is not exist in it, remove old broker - // - otherwise ignore it, replacing our existing one would just bounce the connection - client.updateBroker(data.Brokers) - - client.controllerID = data.ControllerID - - if allKnownMetaData { - client.metadata = make(map[string]map[int32]*PartitionMetadata) - client.metadataTopics = make(map[string]none) - client.cachedPartitionsResults = make(map[string][maxPartitionIndex][]int32) - } - for _, topic := range data.Topics { - // topics must be added firstly to `metadataTopics` to guarantee that all - // requested topics must be recorded to keep them trackable for periodically - // metadata refresh. - if _, exists := client.metadataTopics[topic.Name]; !exists { - client.metadataTopics[topic.Name] = none{} - } - delete(client.metadata, topic.Name) - delete(client.cachedPartitionsResults, topic.Name) - - switch topic.Err { - case ErrNoError: - // no-op - case ErrInvalidTopic, ErrTopicAuthorizationFailed: // don't retry, don't store partial results - err = topic.Err - continue - case ErrUnknownTopicOrPartition: // retry, do not store partial partition results - err = topic.Err - retry = true - continue - case ErrLeaderNotAvailable: // retry, but store partial partition results - retry = true - default: // don't retry, don't store partial results - Logger.Printf("Unexpected topic-level metadata error: %s", topic.Err) - err = topic.Err - continue - } - - client.metadata[topic.Name] = make(map[int32]*PartitionMetadata, len(topic.Partitions)) - for _, partition := range topic.Partitions { - client.metadata[topic.Name][partition.ID] = partition - if partition.Err == ErrLeaderNotAvailable { - retry = true - } - } - - var partitionCache [maxPartitionIndex][]int32 - partitionCache[allPartitions] = client.setPartitionCache(topic.Name, allPartitions) - partitionCache[writablePartitions] = client.setPartitionCache(topic.Name, writablePartitions) - client.cachedPartitionsResults[topic.Name] = partitionCache - } - - return -} - -func (client *client) cachedCoordinator(consumerGroup string) *Broker { - client.lock.RLock() - defer client.lock.RUnlock() - if coordinatorID, ok := client.coordinators[consumerGroup]; ok { - return client.brokers[coordinatorID] - } - return nil -} - -func (client *client) cachedController() *Broker { - client.lock.RLock() - defer client.lock.RUnlock() - - return client.brokers[client.controllerID] -} - -func (client *client) computeBackoff(attemptsRemaining int) time.Duration { - if client.conf.Metadata.Retry.BackoffFunc != nil { - maxRetries := client.conf.Metadata.Retry.Max - retries := maxRetries - attemptsRemaining - return client.conf.Metadata.Retry.BackoffFunc(retries, maxRetries) - } - return client.conf.Metadata.Retry.Backoff -} - -func (client *client) getConsumerMetadata(consumerGroup string, attemptsRemaining int) (*FindCoordinatorResponse, error) { - retry := func(err error) (*FindCoordinatorResponse, error) { - if attemptsRemaining > 0 { - backoff := client.computeBackoff(attemptsRemaining) - Logger.Printf("client/coordinator retrying after %dms... (%d attempts remaining)\n", backoff/time.Millisecond, attemptsRemaining) - time.Sleep(backoff) - return client.getConsumerMetadata(consumerGroup, attemptsRemaining-1) - } - return nil, err - } - - for broker := client.any(); broker != nil; broker = client.any() { - Logger.Printf("client/coordinator requesting coordinator for consumergroup %s from %s\n", consumerGroup, broker.Addr()) - - request := new(FindCoordinatorRequest) - request.CoordinatorKey = consumerGroup - request.CoordinatorType = CoordinatorGroup - - response, err := broker.FindCoordinator(request) - - if err != nil { - Logger.Printf("client/coordinator request to broker %s failed: %s\n", broker.Addr(), err) - - switch err.(type) { - case PacketEncodingError: - return nil, err - default: - _ = broker.Close() - client.deregisterBroker(broker) - continue - } - } - - switch response.Err { - case ErrNoError: - Logger.Printf("client/coordinator coordinator for consumergroup %s is #%d (%s)\n", consumerGroup, response.Coordinator.ID(), response.Coordinator.Addr()) - return response, nil - - case ErrConsumerCoordinatorNotAvailable: - Logger.Printf("client/coordinator coordinator for consumer group %s is not available\n", consumerGroup) - - // This is very ugly, but this scenario will only happen once per cluster. - // The __consumer_offsets topic only has to be created one time. - // The number of partitions not configurable, but partition 0 should always exist. - if _, err := client.Leader("__consumer_offsets", 0); err != nil { - Logger.Printf("client/coordinator the __consumer_offsets topic is not initialized completely yet. Waiting 2 seconds...\n") - time.Sleep(2 * time.Second) - } - - return retry(ErrConsumerCoordinatorNotAvailable) - case ErrGroupAuthorizationFailed: - Logger.Printf("client was not authorized to access group %s while attempting to find coordinator", consumerGroup) - return retry(ErrGroupAuthorizationFailed) - - default: - return nil, response.Err - } - } - - Logger.Println("client/coordinator no available broker to send consumer metadata request to") - client.resurrectDeadBrokers() - return retry(ErrOutOfBrokers) -} - -// nopCloserClient embeds an existing Client, but disables -// the Close method (yet all other methods pass -// through unchanged). This is for use in larger structs -// where it is undesirable to close the client that was -// passed in by the caller. -type nopCloserClient struct { - Client -} - -// Close intercepts and purposely does not call the underlying -// client's Close() method. -func (ncc *nopCloserClient) Close() error { - return nil -} diff --git a/vendor/github.com/Shopify/sarama/compress.go b/vendor/github.com/Shopify/sarama/compress.go deleted file mode 100644 index 12cd7c3d..00000000 --- a/vendor/github.com/Shopify/sarama/compress.go +++ /dev/null @@ -1,194 +0,0 @@ -package sarama - -import ( - "bytes" - "compress/gzip" - "fmt" - "sync" - - snappy "github.com/eapache/go-xerial-snappy" - "github.com/pierrec/lz4" -) - -var ( - lz4WriterPool = sync.Pool{ - New: func() interface{} { - return lz4.NewWriter(nil) - }, - } - - gzipWriterPool = sync.Pool{ - New: func() interface{} { - return gzip.NewWriter(nil) - }, - } - gzipWriterPoolForCompressionLevel1 = sync.Pool{ - New: func() interface{} { - gz, err := gzip.NewWriterLevel(nil, 1) - if err != nil { - panic(err) - } - return gz - }, - } - gzipWriterPoolForCompressionLevel2 = sync.Pool{ - New: func() interface{} { - gz, err := gzip.NewWriterLevel(nil, 2) - if err != nil { - panic(err) - } - return gz - }, - } - gzipWriterPoolForCompressionLevel3 = sync.Pool{ - New: func() interface{} { - gz, err := gzip.NewWriterLevel(nil, 3) - if err != nil { - panic(err) - } - return gz - }, - } - gzipWriterPoolForCompressionLevel4 = sync.Pool{ - New: func() interface{} { - gz, err := gzip.NewWriterLevel(nil, 4) - if err != nil { - panic(err) - } - return gz - }, - } - gzipWriterPoolForCompressionLevel5 = sync.Pool{ - New: func() interface{} { - gz, err := gzip.NewWriterLevel(nil, 5) - if err != nil { - panic(err) - } - return gz - }, - } - gzipWriterPoolForCompressionLevel6 = sync.Pool{ - New: func() interface{} { - gz, err := gzip.NewWriterLevel(nil, 6) - if err != nil { - panic(err) - } - return gz - }, - } - gzipWriterPoolForCompressionLevel7 = sync.Pool{ - New: func() interface{} { - gz, err := gzip.NewWriterLevel(nil, 7) - if err != nil { - panic(err) - } - return gz - }, - } - gzipWriterPoolForCompressionLevel8 = sync.Pool{ - New: func() interface{} { - gz, err := gzip.NewWriterLevel(nil, 8) - if err != nil { - panic(err) - } - return gz - }, - } - gzipWriterPoolForCompressionLevel9 = sync.Pool{ - New: func() interface{} { - gz, err := gzip.NewWriterLevel(nil, 9) - if err != nil { - panic(err) - } - return gz - }, - } -) - -func compress(cc CompressionCodec, level int, data []byte) ([]byte, error) { - switch cc { - case CompressionNone: - return data, nil - case CompressionGZIP: - var ( - err error - buf bytes.Buffer - writer *gzip.Writer - ) - - switch level { - case CompressionLevelDefault: - writer = gzipWriterPool.Get().(*gzip.Writer) - defer gzipWriterPool.Put(writer) - writer.Reset(&buf) - case 1: - writer = gzipWriterPoolForCompressionLevel1.Get().(*gzip.Writer) - defer gzipWriterPoolForCompressionLevel1.Put(writer) - writer.Reset(&buf) - case 2: - writer = gzipWriterPoolForCompressionLevel2.Get().(*gzip.Writer) - defer gzipWriterPoolForCompressionLevel2.Put(writer) - writer.Reset(&buf) - case 3: - writer = gzipWriterPoolForCompressionLevel3.Get().(*gzip.Writer) - defer gzipWriterPoolForCompressionLevel3.Put(writer) - writer.Reset(&buf) - case 4: - writer = gzipWriterPoolForCompressionLevel4.Get().(*gzip.Writer) - defer gzipWriterPoolForCompressionLevel4.Put(writer) - writer.Reset(&buf) - case 5: - writer = gzipWriterPoolForCompressionLevel5.Get().(*gzip.Writer) - defer gzipWriterPoolForCompressionLevel5.Put(writer) - writer.Reset(&buf) - case 6: - writer = gzipWriterPoolForCompressionLevel6.Get().(*gzip.Writer) - defer gzipWriterPoolForCompressionLevel6.Put(writer) - writer.Reset(&buf) - case 7: - writer = gzipWriterPoolForCompressionLevel7.Get().(*gzip.Writer) - defer gzipWriterPoolForCompressionLevel7.Put(writer) - writer.Reset(&buf) - case 8: - writer = gzipWriterPoolForCompressionLevel8.Get().(*gzip.Writer) - defer gzipWriterPoolForCompressionLevel8.Put(writer) - writer.Reset(&buf) - case 9: - writer = gzipWriterPoolForCompressionLevel9.Get().(*gzip.Writer) - defer gzipWriterPoolForCompressionLevel9.Put(writer) - writer.Reset(&buf) - default: - writer, err = gzip.NewWriterLevel(&buf, level) - if err != nil { - return nil, err - } - } - if _, err := writer.Write(data); err != nil { - return nil, err - } - if err := writer.Close(); err != nil { - return nil, err - } - return buf.Bytes(), nil - case CompressionSnappy: - return snappy.Encode(data), nil - case CompressionLZ4: - writer := lz4WriterPool.Get().(*lz4.Writer) - defer lz4WriterPool.Put(writer) - - var buf bytes.Buffer - writer.Reset(&buf) - - if _, err := writer.Write(data); err != nil { - return nil, err - } - if err := writer.Close(); err != nil { - return nil, err - } - return buf.Bytes(), nil - case CompressionZSTD: - return zstdCompress(nil, data) - default: - return nil, PacketEncodingError{fmt.Sprintf("unsupported compression codec (%d)", cc)} - } -} diff --git a/vendor/github.com/Shopify/sarama/config.go b/vendor/github.com/Shopify/sarama/config.go deleted file mode 100644 index 9b7ce7ae..00000000 --- a/vendor/github.com/Shopify/sarama/config.go +++ /dev/null @@ -1,765 +0,0 @@ -package sarama - -import ( - "compress/gzip" - "crypto/tls" - "fmt" - "io/ioutil" - "net" - "regexp" - "time" - - "github.com/rcrowley/go-metrics" - "golang.org/x/net/proxy" -) - -const defaultClientID = "sarama" - -var validID = regexp.MustCompile(`\A[A-Za-z0-9._-]+\z`) - -// Config is used to pass multiple configuration options to Sarama's constructors. -type Config struct { - // Admin is the namespace for ClusterAdmin properties used by the administrative Kafka client. - Admin struct { - Retry struct { - // The total number of times to retry sending (retriable) admin requests (default 5). - // Similar to the `retries` setting of the JVM AdminClientConfig. - Max int - // Backoff time between retries of a failed request (default 100ms) - Backoff time.Duration - } - // The maximum duration the administrative Kafka client will wait for ClusterAdmin operations, - // including topics, brokers, configurations and ACLs (defaults to 3 seconds). - Timeout time.Duration - } - - // Net is the namespace for network-level properties used by the Broker, and - // shared by the Client/Producer/Consumer. - Net struct { - // How many outstanding requests a connection is allowed to have before - // sending on it blocks (default 5). - MaxOpenRequests int - - // All three of the below configurations are similar to the - // `socket.timeout.ms` setting in JVM kafka. All of them default - // to 30 seconds. - DialTimeout time.Duration // How long to wait for the initial connection. - ReadTimeout time.Duration // How long to wait for a response. - WriteTimeout time.Duration // How long to wait for a transmit. - - TLS struct { - // Whether or not to use TLS when connecting to the broker - // (defaults to false). - Enable bool - // The TLS configuration to use for secure connections if - // enabled (defaults to nil). - Config *tls.Config - } - - // SASL based authentication with broker. While there are multiple SASL authentication methods - // the current implementation is limited to plaintext (SASL/PLAIN) authentication - SASL struct { - // Whether or not to use SASL authentication when connecting to the broker - // (defaults to false). - Enable bool - // SASLMechanism is the name of the enabled SASL mechanism. - // Possible values: OAUTHBEARER, PLAIN (defaults to PLAIN). - Mechanism SASLMechanism - // Version is the SASL Protocol Version to use - // Kafka > 1.x should use V1, except on Azure EventHub which use V0 - Version int16 - // Whether or not to send the Kafka SASL handshake first if enabled - // (defaults to true). You should only set this to false if you're using - // a non-Kafka SASL proxy. - Handshake bool - // AuthIdentity is an (optional) authorization identity (authzid) to - // use for SASL/PLAIN authentication (if different from User) when - // an authenticated user is permitted to act as the presented - // alternative user. See RFC4616 for details. - AuthIdentity string - // User is the authentication identity (authcid) to present for - // SASL/PLAIN or SASL/SCRAM authentication - User string - // Password for SASL/PLAIN authentication - Password string - // authz id used for SASL/SCRAM authentication - SCRAMAuthzID string - // SCRAMClientGeneratorFunc is a generator of a user provided implementation of a SCRAM - // client used to perform the SCRAM exchange with the server. - SCRAMClientGeneratorFunc func() SCRAMClient - // TokenProvider is a user-defined callback for generating - // access tokens for SASL/OAUTHBEARER auth. See the - // AccessTokenProvider interface docs for proper implementation - // guidelines. - TokenProvider AccessTokenProvider - - GSSAPI GSSAPIConfig - } - - // KeepAlive specifies the keep-alive period for an active network connection (defaults to 0). - // If zero or positive, keep-alives are enabled. - // If negative, keep-alives are disabled. - KeepAlive time.Duration - - // LocalAddr is the local address to use when dialing an - // address. The address must be of a compatible type for the - // network being dialed. - // If nil, a local address is automatically chosen. - LocalAddr net.Addr - - Proxy struct { - // Whether or not to use proxy when connecting to the broker - // (defaults to false). - Enable bool - // The proxy dialer to use enabled (defaults to nil). - Dialer proxy.Dialer - } - } - - // Metadata is the namespace for metadata management properties used by the - // Client, and shared by the Producer/Consumer. - Metadata struct { - Retry struct { - // The total number of times to retry a metadata request when the - // cluster is in the middle of a leader election (default 3). - Max int - // How long to wait for leader election to occur before retrying - // (default 250ms). Similar to the JVM's `retry.backoff.ms`. - Backoff time.Duration - // Called to compute backoff time dynamically. Useful for implementing - // more sophisticated backoff strategies. This takes precedence over - // `Backoff` if set. - BackoffFunc func(retries, maxRetries int) time.Duration - } - // How frequently to refresh the cluster metadata in the background. - // Defaults to 10 minutes. Set to 0 to disable. Similar to - // `topic.metadata.refresh.interval.ms` in the JVM version. - RefreshFrequency time.Duration - - // Whether to maintain a full set of metadata for all topics, or just - // the minimal set that has been necessary so far. The full set is simpler - // and usually more convenient, but can take up a substantial amount of - // memory if you have many topics and partitions. Defaults to true. - Full bool - - // How long to wait for a successful metadata response. - // Disabled by default which means a metadata request against an unreachable - // cluster (all brokers are unreachable or unresponsive) can take up to - // `Net.[Dial|Read]Timeout * BrokerCount * (Metadata.Retry.Max + 1) + Metadata.Retry.Backoff * Metadata.Retry.Max` - // to fail. - Timeout time.Duration - } - - // Producer is the namespace for configuration related to producing messages, - // used by the Producer. - Producer struct { - // The maximum permitted size of a message (defaults to 1000000). Should be - // set equal to or smaller than the broker's `message.max.bytes`. - MaxMessageBytes int - // The level of acknowledgement reliability needed from the broker (defaults - // to WaitForLocal). Equivalent to the `request.required.acks` setting of the - // JVM producer. - RequiredAcks RequiredAcks - // The maximum duration the broker will wait the receipt of the number of - // RequiredAcks (defaults to 10 seconds). This is only relevant when - // RequiredAcks is set to WaitForAll or a number > 1. Only supports - // millisecond resolution, nanoseconds will be truncated. Equivalent to - // the JVM producer's `request.timeout.ms` setting. - Timeout time.Duration - // The type of compression to use on messages (defaults to no compression). - // Similar to `compression.codec` setting of the JVM producer. - Compression CompressionCodec - // The level of compression to use on messages. The meaning depends - // on the actual compression type used and defaults to default compression - // level for the codec. - CompressionLevel int - // Generates partitioners for choosing the partition to send messages to - // (defaults to hashing the message key). Similar to the `partitioner.class` - // setting for the JVM producer. - Partitioner PartitionerConstructor - // If enabled, the producer will ensure that exactly one copy of each message is - // written. - Idempotent bool - - // Return specifies what channels will be populated. If they are set to true, - // you must read from the respective channels to prevent deadlock. If, - // however, this config is used to create a `SyncProducer`, both must be set - // to true and you shall not read from the channels since the producer does - // this internally. - Return struct { - // If enabled, successfully delivered messages will be returned on the - // Successes channel (default disabled). - Successes bool - - // If enabled, messages that failed to deliver will be returned on the - // Errors channel, including error (default enabled). - Errors bool - } - - // The following config options control how often messages are batched up and - // sent to the broker. By default, messages are sent as fast as possible, and - // all messages received while the current batch is in-flight are placed - // into the subsequent batch. - Flush struct { - // The best-effort number of bytes needed to trigger a flush. Use the - // global sarama.MaxRequestSize to set a hard upper limit. - Bytes int - // The best-effort number of messages needed to trigger a flush. Use - // `MaxMessages` to set a hard upper limit. - Messages int - // The best-effort frequency of flushes. Equivalent to - // `queue.buffering.max.ms` setting of JVM producer. - Frequency time.Duration - // The maximum number of messages the producer will send in a single - // broker request. Defaults to 0 for unlimited. Similar to - // `queue.buffering.max.messages` in the JVM producer. - MaxMessages int - } - - Retry struct { - // The total number of times to retry sending a message (default 3). - // Similar to the `message.send.max.retries` setting of the JVM producer. - Max int - // How long to wait for the cluster to settle between retries - // (default 100ms). Similar to the `retry.backoff.ms` setting of the - // JVM producer. - Backoff time.Duration - // Called to compute backoff time dynamically. Useful for implementing - // more sophisticated backoff strategies. This takes precedence over - // `Backoff` if set. - BackoffFunc func(retries, maxRetries int) time.Duration - } - - // Interceptors to be called when the producer dispatcher reads the - // message for the first time. Interceptors allows to intercept and - // possible mutate the message before they are published to Kafka - // cluster. *ProducerMessage modified by the first interceptor's - // OnSend() is passed to the second interceptor OnSend(), and so on in - // the interceptor chain. - Interceptors []ProducerInterceptor - } - - // Consumer is the namespace for configuration related to consuming messages, - // used by the Consumer. - Consumer struct { - - // Group is the namespace for configuring consumer group. - Group struct { - Session struct { - // The timeout used to detect consumer failures when using Kafka's group management facility. - // The consumer sends periodic heartbeats to indicate its liveness to the broker. - // If no heartbeats are received by the broker before the expiration of this session timeout, - // then the broker will remove this consumer from the group and initiate a rebalance. - // Note that the value must be in the allowable range as configured in the broker configuration - // by `group.min.session.timeout.ms` and `group.max.session.timeout.ms` (default 10s) - Timeout time.Duration - } - Heartbeat struct { - // The expected time between heartbeats to the consumer coordinator when using Kafka's group - // management facilities. Heartbeats are used to ensure that the consumer's session stays active and - // to facilitate rebalancing when new consumers join or leave the group. - // The value must be set lower than Consumer.Group.Session.Timeout, but typically should be set no - // higher than 1/3 of that value. - // It can be adjusted even lower to control the expected time for normal rebalances (default 3s) - Interval time.Duration - } - Rebalance struct { - // Strategy for allocating topic partitions to members (default BalanceStrategyRange) - Strategy BalanceStrategy - // The maximum allowed time for each worker to join the group once a rebalance has begun. - // This is basically a limit on the amount of time needed for all tasks to flush any pending - // data and commit offsets. If the timeout is exceeded, then the worker will be removed from - // the group, which will cause offset commit failures (default 60s). - Timeout time.Duration - - Retry struct { - // When a new consumer joins a consumer group the set of consumers attempt to "rebalance" - // the load to assign partitions to each consumer. If the set of consumers changes while - // this assignment is taking place the rebalance will fail and retry. This setting controls - // the maximum number of attempts before giving up (default 4). - Max int - // Backoff time between retries during rebalance (default 2s) - Backoff time.Duration - } - } - Member struct { - // Custom metadata to include when joining the group. The user data for all joined members - // can be retrieved by sending a DescribeGroupRequest to the broker that is the - // coordinator for the group. - UserData []byte - } - } - - Retry struct { - // How long to wait after a failing to read from a partition before - // trying again (default 2s). - Backoff time.Duration - // Called to compute backoff time dynamically. Useful for implementing - // more sophisticated backoff strategies. This takes precedence over - // `Backoff` if set. - BackoffFunc func(retries int) time.Duration - } - - // Fetch is the namespace for controlling how many bytes are retrieved by any - // given request. - Fetch struct { - // The minimum number of message bytes to fetch in a request - the broker - // will wait until at least this many are available. The default is 1, - // as 0 causes the consumer to spin when no messages are available. - // Equivalent to the JVM's `fetch.min.bytes`. - Min int32 - // The default number of message bytes to fetch from the broker in each - // request (default 1MB). This should be larger than the majority of - // your messages, or else the consumer will spend a lot of time - // negotiating sizes and not actually consuming. Similar to the JVM's - // `fetch.message.max.bytes`. - Default int32 - // The maximum number of message bytes to fetch from the broker in a - // single request. Messages larger than this will return - // ErrMessageTooLarge and will not be consumable, so you must be sure - // this is at least as large as your largest message. Defaults to 0 - // (no limit). Similar to the JVM's `fetch.message.max.bytes`. The - // global `sarama.MaxResponseSize` still applies. - Max int32 - } - // The maximum amount of time the broker will wait for Consumer.Fetch.Min - // bytes to become available before it returns fewer than that anyways. The - // default is 250ms, since 0 causes the consumer to spin when no events are - // available. 100-500ms is a reasonable range for most cases. Kafka only - // supports precision up to milliseconds; nanoseconds will be truncated. - // Equivalent to the JVM's `fetch.wait.max.ms`. - MaxWaitTime time.Duration - - // The maximum amount of time the consumer expects a message takes to - // process for the user. If writing to the Messages channel takes longer - // than this, that partition will stop fetching more messages until it - // can proceed again. - // Note that, since the Messages channel is buffered, the actual grace time is - // (MaxProcessingTime * ChannelBufferSize). Defaults to 100ms. - // If a message is not written to the Messages channel between two ticks - // of the expiryTicker then a timeout is detected. - // Using a ticker instead of a timer to detect timeouts should typically - // result in many fewer calls to Timer functions which may result in a - // significant performance improvement if many messages are being sent - // and timeouts are infrequent. - // The disadvantage of using a ticker instead of a timer is that - // timeouts will be less accurate. That is, the effective timeout could - // be between `MaxProcessingTime` and `2 * MaxProcessingTime`. For - // example, if `MaxProcessingTime` is 100ms then a delay of 180ms - // between two messages being sent may not be recognized as a timeout. - MaxProcessingTime time.Duration - - // Return specifies what channels will be populated. If they are set to true, - // you must read from them to prevent deadlock. - Return struct { - // If enabled, any errors that occurred while consuming are returned on - // the Errors channel (default disabled). - Errors bool - } - - // Offsets specifies configuration for how and when to commit consumed - // offsets. This currently requires the manual use of an OffsetManager - // but will eventually be automated. - Offsets struct { - // Deprecated: CommitInterval exists for historical compatibility - // and should not be used. Please use Consumer.Offsets.AutoCommit - CommitInterval time.Duration - - // AutoCommit specifies configuration for commit messages automatically. - AutoCommit struct { - // Whether or not to auto-commit updated offsets back to the broker. - // (default enabled). - Enable bool - - // How frequently to commit updated offsets. Ineffective unless - // auto-commit is enabled (default 1s) - Interval time.Duration - } - - // The initial offset to use if no offset was previously committed. - // Should be OffsetNewest or OffsetOldest. Defaults to OffsetNewest. - Initial int64 - - // The retention duration for committed offsets. If zero, disabled - // (in which case the `offsets.retention.minutes` option on the - // broker will be used). Kafka only supports precision up to - // milliseconds; nanoseconds will be truncated. Requires Kafka - // broker version 0.9.0 or later. - // (default is 0: disabled). - Retention time.Duration - - Retry struct { - // The total number of times to retry failing commit - // requests during OffsetManager shutdown (default 3). - Max int - } - } - - // IsolationLevel support 2 mode: - // - use `ReadUncommitted` (default) to consume and return all messages in message channel - // - use `ReadCommitted` to hide messages that are part of an aborted transaction - IsolationLevel IsolationLevel - - // Interceptors to be called just before the record is sent to the - // messages channel. Interceptors allows to intercept and possible - // mutate the message before they are returned to the client. - // *ConsumerMessage modified by the first interceptor's OnConsume() is - // passed to the second interceptor OnConsume(), and so on in the - // interceptor chain. - Interceptors []ConsumerInterceptor - } - - // A user-provided string sent with every request to the brokers for logging, - // debugging, and auditing purposes. Defaults to "sarama", but you should - // probably set it to something specific to your application. - ClientID string - // A rack identifier for this client. This can be any string value which - // indicates where this client is physically located. - // It corresponds with the broker config 'broker.rack' - RackID string - // The number of events to buffer in internal and external channels. This - // permits the producer and consumer to continue processing some messages - // in the background while user code is working, greatly improving throughput. - // Defaults to 256. - ChannelBufferSize int - // The version of Kafka that Sarama will assume it is running against. - // Defaults to the oldest supported stable version. Since Kafka provides - // backwards-compatibility, setting it to a version older than you have - // will not break anything, although it may prevent you from using the - // latest features. Setting it to a version greater than you are actually - // running may lead to random breakage. - Version KafkaVersion - // The registry to define metrics into. - // Defaults to a local registry. - // If you want to disable metrics gathering, set "metrics.UseNilMetrics" to "true" - // prior to starting Sarama. - // See Examples on how to use the metrics registry - MetricRegistry metrics.Registry -} - -// NewConfig returns a new configuration instance with sane defaults. -func NewConfig() *Config { - c := &Config{} - - c.Admin.Retry.Max = 5 - c.Admin.Retry.Backoff = 100 * time.Millisecond - c.Admin.Timeout = 3 * time.Second - - c.Net.MaxOpenRequests = 5 - c.Net.DialTimeout = 30 * time.Second - c.Net.ReadTimeout = 30 * time.Second - c.Net.WriteTimeout = 30 * time.Second - c.Net.SASL.Handshake = true - c.Net.SASL.Version = SASLHandshakeV0 - - c.Metadata.Retry.Max = 3 - c.Metadata.Retry.Backoff = 250 * time.Millisecond - c.Metadata.RefreshFrequency = 10 * time.Minute - c.Metadata.Full = true - - c.Producer.MaxMessageBytes = 1000000 - c.Producer.RequiredAcks = WaitForLocal - c.Producer.Timeout = 10 * time.Second - c.Producer.Partitioner = NewHashPartitioner - c.Producer.Retry.Max = 3 - c.Producer.Retry.Backoff = 100 * time.Millisecond - c.Producer.Return.Errors = true - c.Producer.CompressionLevel = CompressionLevelDefault - - c.Consumer.Fetch.Min = 1 - c.Consumer.Fetch.Default = 1024 * 1024 - c.Consumer.Retry.Backoff = 2 * time.Second - c.Consumer.MaxWaitTime = 250 * time.Millisecond - c.Consumer.MaxProcessingTime = 100 * time.Millisecond - c.Consumer.Return.Errors = false - c.Consumer.Offsets.AutoCommit.Enable = true - c.Consumer.Offsets.AutoCommit.Interval = 1 * time.Second - c.Consumer.Offsets.Initial = OffsetNewest - c.Consumer.Offsets.Retry.Max = 3 - - c.Consumer.Group.Session.Timeout = 10 * time.Second - c.Consumer.Group.Heartbeat.Interval = 3 * time.Second - c.Consumer.Group.Rebalance.Strategy = BalanceStrategyRange - c.Consumer.Group.Rebalance.Timeout = 60 * time.Second - c.Consumer.Group.Rebalance.Retry.Max = 4 - c.Consumer.Group.Rebalance.Retry.Backoff = 2 * time.Second - - c.ClientID = defaultClientID - c.ChannelBufferSize = 256 - c.Version = MinVersion - c.MetricRegistry = metrics.NewRegistry() - - return c -} - -// Validate checks a Config instance. It will return a -// ConfigurationError if the specified values don't make sense. -func (c *Config) Validate() error { - // some configuration values should be warned on but not fail completely, do those first - if !c.Net.TLS.Enable && c.Net.TLS.Config != nil { - Logger.Println("Net.TLS is disabled but a non-nil configuration was provided.") - } - if !c.Net.SASL.Enable { - if c.Net.SASL.User != "" { - Logger.Println("Net.SASL is disabled but a non-empty username was provided.") - } - if c.Net.SASL.Password != "" { - Logger.Println("Net.SASL is disabled but a non-empty password was provided.") - } - } - if c.Producer.RequiredAcks > 1 { - Logger.Println("Producer.RequiredAcks > 1 is deprecated and will raise an exception with kafka >= 0.8.2.0.") - } - if c.Producer.MaxMessageBytes >= int(MaxRequestSize) { - Logger.Println("Producer.MaxMessageBytes must be smaller than MaxRequestSize; it will be ignored.") - } - if c.Producer.Flush.Bytes >= int(MaxRequestSize) { - Logger.Println("Producer.Flush.Bytes must be smaller than MaxRequestSize; it will be ignored.") - } - if (c.Producer.Flush.Bytes > 0 || c.Producer.Flush.Messages > 0) && c.Producer.Flush.Frequency == 0 { - Logger.Println("Producer.Flush: Bytes or Messages are set, but Frequency is not; messages may not get flushed.") - } - if c.Producer.Timeout%time.Millisecond != 0 { - Logger.Println("Producer.Timeout only supports millisecond resolution; nanoseconds will be truncated.") - } - if c.Consumer.MaxWaitTime < 100*time.Millisecond { - Logger.Println("Consumer.MaxWaitTime is very low, which can cause high CPU and network usage. See documentation for details.") - } - if c.Consumer.MaxWaitTime%time.Millisecond != 0 { - Logger.Println("Consumer.MaxWaitTime only supports millisecond precision; nanoseconds will be truncated.") - } - if c.Consumer.Offsets.Retention%time.Millisecond != 0 { - Logger.Println("Consumer.Offsets.Retention only supports millisecond precision; nanoseconds will be truncated.") - } - if c.Consumer.Group.Session.Timeout%time.Millisecond != 0 { - Logger.Println("Consumer.Group.Session.Timeout only supports millisecond precision; nanoseconds will be truncated.") - } - if c.Consumer.Group.Heartbeat.Interval%time.Millisecond != 0 { - Logger.Println("Consumer.Group.Heartbeat.Interval only supports millisecond precision; nanoseconds will be truncated.") - } - if c.Consumer.Group.Rebalance.Timeout%time.Millisecond != 0 { - Logger.Println("Consumer.Group.Rebalance.Timeout only supports millisecond precision; nanoseconds will be truncated.") - } - if c.ClientID == defaultClientID { - Logger.Println("ClientID is the default of 'sarama', you should consider setting it to something application-specific.") - } - - // validate Net values - switch { - case c.Net.MaxOpenRequests <= 0: - return ConfigurationError("Net.MaxOpenRequests must be > 0") - case c.Net.DialTimeout <= 0: - return ConfigurationError("Net.DialTimeout must be > 0") - case c.Net.ReadTimeout <= 0: - return ConfigurationError("Net.ReadTimeout must be > 0") - case c.Net.WriteTimeout <= 0: - return ConfigurationError("Net.WriteTimeout must be > 0") - case c.Net.SASL.Enable: - if c.Net.SASL.Mechanism == "" { - c.Net.SASL.Mechanism = SASLTypePlaintext - } - - switch c.Net.SASL.Mechanism { - case SASLTypePlaintext: - if c.Net.SASL.User == "" { - return ConfigurationError("Net.SASL.User must not be empty when SASL is enabled") - } - if c.Net.SASL.Password == "" { - return ConfigurationError("Net.SASL.Password must not be empty when SASL is enabled") - } - case SASLTypeOAuth: - if c.Net.SASL.TokenProvider == nil { - return ConfigurationError("An AccessTokenProvider instance must be provided to Net.SASL.TokenProvider") - } - case SASLTypeSCRAMSHA256, SASLTypeSCRAMSHA512: - if c.Net.SASL.User == "" { - return ConfigurationError("Net.SASL.User must not be empty when SASL is enabled") - } - if c.Net.SASL.Password == "" { - return ConfigurationError("Net.SASL.Password must not be empty when SASL is enabled") - } - if c.Net.SASL.SCRAMClientGeneratorFunc == nil { - return ConfigurationError("A SCRAMClientGeneratorFunc function must be provided to Net.SASL.SCRAMClientGeneratorFunc") - } - case SASLTypeGSSAPI: - if c.Net.SASL.GSSAPI.ServiceName == "" { - return ConfigurationError("Net.SASL.GSSAPI.ServiceName must not be empty when GSS-API mechanism is used") - } - - if c.Net.SASL.GSSAPI.AuthType == KRB5_USER_AUTH { - if c.Net.SASL.GSSAPI.Password == "" { - return ConfigurationError("Net.SASL.GSSAPI.Password must not be empty when GSS-API " + - "mechanism is used and Net.SASL.GSSAPI.AuthType = KRB5_USER_AUTH") - } - } else if c.Net.SASL.GSSAPI.AuthType == KRB5_KEYTAB_AUTH { - if c.Net.SASL.GSSAPI.KeyTabPath == "" { - return ConfigurationError("Net.SASL.GSSAPI.KeyTabPath must not be empty when GSS-API mechanism is used" + - " and Net.SASL.GSSAPI.AuthType = KRB5_KEYTAB_AUTH") - } - } else { - return ConfigurationError("Net.SASL.GSSAPI.AuthType is invalid. Possible values are KRB5_USER_AUTH and KRB5_KEYTAB_AUTH") - } - if c.Net.SASL.GSSAPI.KerberosConfigPath == "" { - return ConfigurationError("Net.SASL.GSSAPI.KerberosConfigPath must not be empty when GSS-API mechanism is used") - } - if c.Net.SASL.GSSAPI.Username == "" { - return ConfigurationError("Net.SASL.GSSAPI.Username must not be empty when GSS-API mechanism is used") - } - if c.Net.SASL.GSSAPI.Realm == "" { - return ConfigurationError("Net.SASL.GSSAPI.Realm must not be empty when GSS-API mechanism is used") - } - default: - msg := fmt.Sprintf("The SASL mechanism configuration is invalid. Possible values are `%s`, `%s`, `%s`, `%s` and `%s`", - SASLTypeOAuth, SASLTypePlaintext, SASLTypeSCRAMSHA256, SASLTypeSCRAMSHA512, SASLTypeGSSAPI) - return ConfigurationError(msg) - } - } - - // validate the Admin values - switch { - case c.Admin.Timeout <= 0: - return ConfigurationError("Admin.Timeout must be > 0") - } - - // validate the Metadata values - switch { - case c.Metadata.Retry.Max < 0: - return ConfigurationError("Metadata.Retry.Max must be >= 0") - case c.Metadata.Retry.Backoff < 0: - return ConfigurationError("Metadata.Retry.Backoff must be >= 0") - case c.Metadata.RefreshFrequency < 0: - return ConfigurationError("Metadata.RefreshFrequency must be >= 0") - } - - // validate the Producer values - switch { - case c.Producer.MaxMessageBytes <= 0: - return ConfigurationError("Producer.MaxMessageBytes must be > 0") - case c.Producer.RequiredAcks < -1: - return ConfigurationError("Producer.RequiredAcks must be >= -1") - case c.Producer.Timeout <= 0: - return ConfigurationError("Producer.Timeout must be > 0") - case c.Producer.Partitioner == nil: - return ConfigurationError("Producer.Partitioner must not be nil") - case c.Producer.Flush.Bytes < 0: - return ConfigurationError("Producer.Flush.Bytes must be >= 0") - case c.Producer.Flush.Messages < 0: - return ConfigurationError("Producer.Flush.Messages must be >= 0") - case c.Producer.Flush.Frequency < 0: - return ConfigurationError("Producer.Flush.Frequency must be >= 0") - case c.Producer.Flush.MaxMessages < 0: - return ConfigurationError("Producer.Flush.MaxMessages must be >= 0") - case c.Producer.Flush.MaxMessages > 0 && c.Producer.Flush.MaxMessages < c.Producer.Flush.Messages: - return ConfigurationError("Producer.Flush.MaxMessages must be >= Producer.Flush.Messages when set") - case c.Producer.Retry.Max < 0: - return ConfigurationError("Producer.Retry.Max must be >= 0") - case c.Producer.Retry.Backoff < 0: - return ConfigurationError("Producer.Retry.Backoff must be >= 0") - } - - if c.Producer.Compression == CompressionLZ4 && !c.Version.IsAtLeast(V0_10_0_0) { - return ConfigurationError("lz4 compression requires Version >= V0_10_0_0") - } - - if c.Producer.Compression == CompressionGZIP { - if c.Producer.CompressionLevel != CompressionLevelDefault { - if _, err := gzip.NewWriterLevel(ioutil.Discard, c.Producer.CompressionLevel); err != nil { - return ConfigurationError(fmt.Sprintf("gzip compression does not work with level %d: %v", c.Producer.CompressionLevel, err)) - } - } - } - - if c.Producer.Compression == CompressionZSTD && !c.Version.IsAtLeast(V2_1_0_0) { - return ConfigurationError("zstd compression requires Version >= V2_1_0_0") - } - - if c.Producer.Idempotent { - if !c.Version.IsAtLeast(V0_11_0_0) { - return ConfigurationError("Idempotent producer requires Version >= V0_11_0_0") - } - if c.Producer.Retry.Max == 0 { - return ConfigurationError("Idempotent producer requires Producer.Retry.Max >= 1") - } - if c.Producer.RequiredAcks != WaitForAll { - return ConfigurationError("Idempotent producer requires Producer.RequiredAcks to be WaitForAll") - } - if c.Net.MaxOpenRequests > 1 { - return ConfigurationError("Idempotent producer requires Net.MaxOpenRequests to be 1") - } - } - - // validate the Consumer values - switch { - case c.Consumer.Fetch.Min <= 0: - return ConfigurationError("Consumer.Fetch.Min must be > 0") - case c.Consumer.Fetch.Default <= 0: - return ConfigurationError("Consumer.Fetch.Default must be > 0") - case c.Consumer.Fetch.Max < 0: - return ConfigurationError("Consumer.Fetch.Max must be >= 0") - case c.Consumer.MaxWaitTime < 1*time.Millisecond: - return ConfigurationError("Consumer.MaxWaitTime must be >= 1ms") - case c.Consumer.MaxProcessingTime <= 0: - return ConfigurationError("Consumer.MaxProcessingTime must be > 0") - case c.Consumer.Retry.Backoff < 0: - return ConfigurationError("Consumer.Retry.Backoff must be >= 0") - case c.Consumer.Offsets.AutoCommit.Interval <= 0: - return ConfigurationError("Consumer.Offsets.AutoCommit.Interval must be > 0") - case c.Consumer.Offsets.Initial != OffsetOldest && c.Consumer.Offsets.Initial != OffsetNewest: - return ConfigurationError("Consumer.Offsets.Initial must be OffsetOldest or OffsetNewest") - case c.Consumer.Offsets.Retry.Max < 0: - return ConfigurationError("Consumer.Offsets.Retry.Max must be >= 0") - case c.Consumer.IsolationLevel != ReadUncommitted && c.Consumer.IsolationLevel != ReadCommitted: - return ConfigurationError("Consumer.IsolationLevel must be ReadUncommitted or ReadCommitted") - } - - if c.Consumer.Offsets.CommitInterval != 0 { - Logger.Println("Deprecation warning: Consumer.Offsets.CommitInterval exists for historical compatibility" + - " and should not be used. Please use Consumer.Offsets.AutoCommit, the current value will be ignored") - } - - // validate IsolationLevel - if c.Consumer.IsolationLevel == ReadCommitted && !c.Version.IsAtLeast(V0_11_0_0) { - return ConfigurationError("ReadCommitted requires Version >= V0_11_0_0") - } - - // validate the Consumer Group values - switch { - case c.Consumer.Group.Session.Timeout <= 2*time.Millisecond: - return ConfigurationError("Consumer.Group.Session.Timeout must be >= 2ms") - case c.Consumer.Group.Heartbeat.Interval < 1*time.Millisecond: - return ConfigurationError("Consumer.Group.Heartbeat.Interval must be >= 1ms") - case c.Consumer.Group.Heartbeat.Interval >= c.Consumer.Group.Session.Timeout: - return ConfigurationError("Consumer.Group.Heartbeat.Interval must be < Consumer.Group.Session.Timeout") - case c.Consumer.Group.Rebalance.Strategy == nil: - return ConfigurationError("Consumer.Group.Rebalance.Strategy must not be empty") - case c.Consumer.Group.Rebalance.Timeout <= time.Millisecond: - return ConfigurationError("Consumer.Group.Rebalance.Timeout must be >= 1ms") - case c.Consumer.Group.Rebalance.Retry.Max < 0: - return ConfigurationError("Consumer.Group.Rebalance.Retry.Max must be >= 0") - case c.Consumer.Group.Rebalance.Retry.Backoff < 0: - return ConfigurationError("Consumer.Group.Rebalance.Retry.Backoff must be >= 0") - } - - // validate misc shared values - switch { - case c.ChannelBufferSize < 0: - return ConfigurationError("ChannelBufferSize must be >= 0") - case !validID.MatchString(c.ClientID): - return ConfigurationError("ClientID is invalid") - } - - return nil -} - -func (c *Config) getDialer() proxy.Dialer { - if c.Net.Proxy.Enable { - Logger.Printf("using proxy %s", c.Net.Proxy.Dialer) - return c.Net.Proxy.Dialer - } else { - return &net.Dialer{ - Timeout: c.Net.DialTimeout, - KeepAlive: c.Net.KeepAlive, - LocalAddr: c.Net.LocalAddr, - } - } -} diff --git a/vendor/github.com/Shopify/sarama/config_resource_type.go b/vendor/github.com/Shopify/sarama/config_resource_type.go deleted file mode 100644 index bef1053a..00000000 --- a/vendor/github.com/Shopify/sarama/config_resource_type.go +++ /dev/null @@ -1,18 +0,0 @@ -package sarama - -// ConfigResourceType is a type for resources that have configs. -type ConfigResourceType int8 - -// Taken from: -// https://github.com/apache/kafka/blob/ed7c071e07f1f90e4c2895582f61ca090ced3c42/clients/src/main/java/org/apache/kafka/common/config/ConfigResource.java#L32-L55 - -const ( - // UnknownResource constant type - UnknownResource ConfigResourceType = 0 - // TopicResource constant type - TopicResource ConfigResourceType = 2 - // BrokerResource constant type - BrokerResource ConfigResourceType = 4 - // BrokerLoggerResource constant type - BrokerLoggerResource ConfigResourceType = 8 -) diff --git a/vendor/github.com/Shopify/sarama/consumer.go b/vendor/github.com/Shopify/sarama/consumer.go deleted file mode 100644 index 5d992234..00000000 --- a/vendor/github.com/Shopify/sarama/consumer.go +++ /dev/null @@ -1,914 +0,0 @@ -package sarama - -import ( - "errors" - "fmt" - "math" - "sync" - "sync/atomic" - "time" - - "github.com/rcrowley/go-metrics" -) - -// ConsumerMessage encapsulates a Kafka message returned by the consumer. -type ConsumerMessage struct { - Headers []*RecordHeader // only set if kafka is version 0.11+ - Timestamp time.Time // only set if kafka is version 0.10+, inner message timestamp - BlockTimestamp time.Time // only set if kafka is version 0.10+, outer (compressed) block timestamp - - Key, Value []byte - Topic string - Partition int32 - Offset int64 -} - -// ConsumerError is what is provided to the user when an error occurs. -// It wraps an error and includes the topic and partition. -type ConsumerError struct { - Topic string - Partition int32 - Err error -} - -func (ce ConsumerError) Error() string { - return fmt.Sprintf("kafka: error while consuming %s/%d: %s", ce.Topic, ce.Partition, ce.Err) -} - -// ConsumerErrors is a type that wraps a batch of errors and implements the Error interface. -// It can be returned from the PartitionConsumer's Close methods to avoid the need to manually drain errors -// when stopping. -type ConsumerErrors []*ConsumerError - -func (ce ConsumerErrors) Error() string { - return fmt.Sprintf("kafka: %d errors while consuming", len(ce)) -} - -// Consumer manages PartitionConsumers which process Kafka messages from brokers. You MUST call Close() -// on a consumer to avoid leaks, it will not be garbage-collected automatically when it passes out of -// scope. -type Consumer interface { - // Topics returns the set of available topics as retrieved from the cluster - // metadata. This method is the same as Client.Topics(), and is provided for - // convenience. - Topics() ([]string, error) - - // Partitions returns the sorted list of all partition IDs for the given topic. - // This method is the same as Client.Partitions(), and is provided for convenience. - Partitions(topic string) ([]int32, error) - - // ConsumePartition creates a PartitionConsumer on the given topic/partition with - // the given offset. It will return an error if this Consumer is already consuming - // on the given topic/partition. Offset can be a literal offset, or OffsetNewest - // or OffsetOldest - ConsumePartition(topic string, partition int32, offset int64) (PartitionConsumer, error) - - // HighWaterMarks returns the current high water marks for each topic and partition. - // Consistency between partitions is not guaranteed since high water marks are updated separately. - HighWaterMarks() map[string]map[int32]int64 - - // Close shuts down the consumer. It must be called after all child - // PartitionConsumers have already been closed. - Close() error -} - -type consumer struct { - conf *Config - children map[string]map[int32]*partitionConsumer - brokerConsumers map[*Broker]*brokerConsumer - client Client - lock sync.Mutex -} - -// NewConsumer creates a new consumer using the given broker addresses and configuration. -func NewConsumer(addrs []string, config *Config) (Consumer, error) { - client, err := NewClient(addrs, config) - if err != nil { - return nil, err - } - return newConsumer(client) -} - -// NewConsumerFromClient creates a new consumer using the given client. It is still -// necessary to call Close() on the underlying client when shutting down this consumer. -func NewConsumerFromClient(client Client) (Consumer, error) { - // For clients passed in by the client, ensure we don't - // call Close() on it. - cli := &nopCloserClient{client} - return newConsumer(cli) -} - -func newConsumer(client Client) (Consumer, error) { - // Check that we are not dealing with a closed Client before processing any other arguments - if client.Closed() { - return nil, ErrClosedClient - } - - c := &consumer{ - client: client, - conf: client.Config(), - children: make(map[string]map[int32]*partitionConsumer), - brokerConsumers: make(map[*Broker]*brokerConsumer), - } - - return c, nil -} - -func (c *consumer) Close() error { - return c.client.Close() -} - -func (c *consumer) Topics() ([]string, error) { - return c.client.Topics() -} - -func (c *consumer) Partitions(topic string) ([]int32, error) { - return c.client.Partitions(topic) -} - -func (c *consumer) ConsumePartition(topic string, partition int32, offset int64) (PartitionConsumer, error) { - child := &partitionConsumer{ - consumer: c, - conf: c.conf, - topic: topic, - partition: partition, - messages: make(chan *ConsumerMessage, c.conf.ChannelBufferSize), - errors: make(chan *ConsumerError, c.conf.ChannelBufferSize), - feeder: make(chan *FetchResponse, 1), - trigger: make(chan none, 1), - dying: make(chan none), - fetchSize: c.conf.Consumer.Fetch.Default, - } - - if err := child.chooseStartingOffset(offset); err != nil { - return nil, err - } - - var leader *Broker - var err error - if leader, err = c.client.Leader(child.topic, child.partition); err != nil { - return nil, err - } - - if err := c.addChild(child); err != nil { - return nil, err - } - - go withRecover(child.dispatcher) - go withRecover(child.responseFeeder) - - child.broker = c.refBrokerConsumer(leader) - child.broker.input <- child - - return child, nil -} - -func (c *consumer) HighWaterMarks() map[string]map[int32]int64 { - c.lock.Lock() - defer c.lock.Unlock() - - hwms := make(map[string]map[int32]int64) - for topic, p := range c.children { - hwm := make(map[int32]int64, len(p)) - for partition, pc := range p { - hwm[partition] = pc.HighWaterMarkOffset() - } - hwms[topic] = hwm - } - - return hwms -} - -func (c *consumer) addChild(child *partitionConsumer) error { - c.lock.Lock() - defer c.lock.Unlock() - - topicChildren := c.children[child.topic] - if topicChildren == nil { - topicChildren = make(map[int32]*partitionConsumer) - c.children[child.topic] = topicChildren - } - - if topicChildren[child.partition] != nil { - return ConfigurationError("That topic/partition is already being consumed") - } - - topicChildren[child.partition] = child - return nil -} - -func (c *consumer) removeChild(child *partitionConsumer) { - c.lock.Lock() - defer c.lock.Unlock() - - delete(c.children[child.topic], child.partition) -} - -func (c *consumer) refBrokerConsumer(broker *Broker) *brokerConsumer { - c.lock.Lock() - defer c.lock.Unlock() - - bc := c.brokerConsumers[broker] - if bc == nil { - bc = c.newBrokerConsumer(broker) - c.brokerConsumers[broker] = bc - } - - bc.refs++ - - return bc -} - -func (c *consumer) unrefBrokerConsumer(brokerWorker *brokerConsumer) { - c.lock.Lock() - defer c.lock.Unlock() - - brokerWorker.refs-- - - if brokerWorker.refs == 0 { - close(brokerWorker.input) - if c.brokerConsumers[brokerWorker.broker] == brokerWorker { - delete(c.brokerConsumers, brokerWorker.broker) - } - } -} - -func (c *consumer) abandonBrokerConsumer(brokerWorker *brokerConsumer) { - c.lock.Lock() - defer c.lock.Unlock() - - delete(c.brokerConsumers, brokerWorker.broker) -} - -// PartitionConsumer - -// PartitionConsumer processes Kafka messages from a given topic and partition. You MUST call one of Close() or -// AsyncClose() on a PartitionConsumer to avoid leaks; it will not be garbage-collected automatically when it passes out -// of scope. -// -// The simplest way of using a PartitionConsumer is to loop over its Messages channel using a for/range -// loop. The PartitionConsumer will only stop itself in one case: when the offset being consumed is reported -// as out of range by the brokers. In this case you should decide what you want to do (try a different offset, -// notify a human, etc) and handle it appropriately. For all other error cases, it will just keep retrying. -// By default, it logs these errors to sarama.Logger; if you want to be notified directly of all errors, set -// your config's Consumer.Return.Errors to true and read from the Errors channel, using a select statement -// or a separate goroutine. Check out the Consumer examples to see implementations of these different approaches. -// -// To terminate such a for/range loop while the loop is executing, call AsyncClose. This will kick off the process of -// consumer tear-down & return immediately. Continue to loop, servicing the Messages channel until the teardown process -// AsyncClose initiated closes it (thus terminating the for/range loop). If you've already ceased reading Messages, call -// Close; this will signal the PartitionConsumer's goroutines to begin shutting down (just like AsyncClose), but will -// also drain the Messages channel, harvest all errors & return them once cleanup has completed. -type PartitionConsumer interface { - // AsyncClose initiates a shutdown of the PartitionConsumer. This method will return immediately, after which you - // should continue to service the 'Messages' and 'Errors' channels until they are empty. It is required to call this - // function, or Close before a consumer object passes out of scope, as it will otherwise leak memory. You must call - // this before calling Close on the underlying client. - AsyncClose() - - // Close stops the PartitionConsumer from fetching messages. It will initiate a shutdown just like AsyncClose, drain - // the Messages channel, harvest any errors & return them to the caller. Note that if you are continuing to service - // the Messages channel when this function is called, you will be competing with Close for messages; consider - // calling AsyncClose, instead. It is required to call this function (or AsyncClose) before a consumer object passes - // out of scope, as it will otherwise leak memory. You must call this before calling Close on the underlying client. - Close() error - - // Messages returns the read channel for the messages that are returned by - // the broker. - Messages() <-chan *ConsumerMessage - - // Errors returns a read channel of errors that occurred during consuming, if - // enabled. By default, errors are logged and not returned over this channel. - // If you want to implement any custom error handling, set your config's - // Consumer.Return.Errors setting to true, and read from this channel. - Errors() <-chan *ConsumerError - - // HighWaterMarkOffset returns the high water mark offset of the partition, - // i.e. the offset that will be used for the next message that will be produced. - // You can use this to determine how far behind the processing is. - HighWaterMarkOffset() int64 -} - -type partitionConsumer struct { - highWaterMarkOffset int64 // must be at the top of the struct because https://golang.org/pkg/sync/atomic/#pkg-note-BUG - - consumer *consumer - conf *Config - broker *brokerConsumer - messages chan *ConsumerMessage - errors chan *ConsumerError - feeder chan *FetchResponse - - trigger, dying chan none - closeOnce sync.Once - topic string - partition int32 - responseResult error - fetchSize int32 - offset int64 - retries int32 -} - -var errTimedOut = errors.New("timed out feeding messages to the user") // not user-facing - -func (child *partitionConsumer) sendError(err error) { - cErr := &ConsumerError{ - Topic: child.topic, - Partition: child.partition, - Err: err, - } - - if child.conf.Consumer.Return.Errors { - child.errors <- cErr - } else { - Logger.Println(cErr) - } -} - -func (child *partitionConsumer) computeBackoff() time.Duration { - if child.conf.Consumer.Retry.BackoffFunc != nil { - retries := atomic.AddInt32(&child.retries, 1) - return child.conf.Consumer.Retry.BackoffFunc(int(retries)) - } - return child.conf.Consumer.Retry.Backoff -} - -func (child *partitionConsumer) dispatcher() { - for range child.trigger { - select { - case <-child.dying: - close(child.trigger) - case <-time.After(child.computeBackoff()): - if child.broker != nil { - child.consumer.unrefBrokerConsumer(child.broker) - child.broker = nil - } - - Logger.Printf("consumer/%s/%d finding new broker\n", child.topic, child.partition) - if err := child.dispatch(); err != nil { - child.sendError(err) - child.trigger <- none{} - } - } - } - - if child.broker != nil { - child.consumer.unrefBrokerConsumer(child.broker) - } - child.consumer.removeChild(child) - close(child.feeder) -} - -func (child *partitionConsumer) dispatch() error { - if err := child.consumer.client.RefreshMetadata(child.topic); err != nil { - return err - } - - var leader *Broker - var err error - if leader, err = child.consumer.client.Leader(child.topic, child.partition); err != nil { - return err - } - - child.broker = child.consumer.refBrokerConsumer(leader) - - child.broker.input <- child - - return nil -} - -func (child *partitionConsumer) chooseStartingOffset(offset int64) error { - newestOffset, err := child.consumer.client.GetOffset(child.topic, child.partition, OffsetNewest) - if err != nil { - return err - } - oldestOffset, err := child.consumer.client.GetOffset(child.topic, child.partition, OffsetOldest) - if err != nil { - return err - } - - switch { - case offset == OffsetNewest: - child.offset = newestOffset - case offset == OffsetOldest: - child.offset = oldestOffset - case offset >= oldestOffset && offset <= newestOffset: - child.offset = offset - default: - return ErrOffsetOutOfRange - } - - return nil -} - -func (child *partitionConsumer) Messages() <-chan *ConsumerMessage { - return child.messages -} - -func (child *partitionConsumer) Errors() <-chan *ConsumerError { - return child.errors -} - -func (child *partitionConsumer) AsyncClose() { - // this triggers whatever broker owns this child to abandon it and close its trigger channel, which causes - // the dispatcher to exit its loop, which removes it from the consumer then closes its 'messages' and - // 'errors' channel (alternatively, if the child is already at the dispatcher for some reason, that will - // also just close itself) - child.closeOnce.Do(func() { - close(child.dying) - }) -} - -func (child *partitionConsumer) Close() error { - child.AsyncClose() - - var consumerErrors ConsumerErrors - for err := range child.errors { - consumerErrors = append(consumerErrors, err) - } - - if len(consumerErrors) > 0 { - return consumerErrors - } - return nil -} - -func (child *partitionConsumer) HighWaterMarkOffset() int64 { - return atomic.LoadInt64(&child.highWaterMarkOffset) -} - -func (child *partitionConsumer) responseFeeder() { - var msgs []*ConsumerMessage - expiryTicker := time.NewTicker(child.conf.Consumer.MaxProcessingTime) - firstAttempt := true - -feederLoop: - for response := range child.feeder { - msgs, child.responseResult = child.parseResponse(response) - - if child.responseResult == nil { - atomic.StoreInt32(&child.retries, 0) - } - - for i, msg := range msgs { - for _, interceptor := range child.conf.Consumer.Interceptors { - msg.safelyApplyInterceptor(interceptor) - } - messageSelect: - select { - case <-child.dying: - child.broker.acks.Done() - continue feederLoop - case child.messages <- msg: - firstAttempt = true - case <-expiryTicker.C: - if !firstAttempt { - child.responseResult = errTimedOut - child.broker.acks.Done() - remainingLoop: - for _, msg = range msgs[i:] { - select { - case child.messages <- msg: - case <-child.dying: - break remainingLoop - } - } - child.broker.input <- child - continue feederLoop - } else { - // current message has not been sent, return to select - // statement - firstAttempt = false - goto messageSelect - } - } - } - - child.broker.acks.Done() - } - - expiryTicker.Stop() - close(child.messages) - close(child.errors) -} - -func (child *partitionConsumer) parseMessages(msgSet *MessageSet) ([]*ConsumerMessage, error) { - var messages []*ConsumerMessage - for _, msgBlock := range msgSet.Messages { - for _, msg := range msgBlock.Messages() { - offset := msg.Offset - timestamp := msg.Msg.Timestamp - if msg.Msg.Version >= 1 { - baseOffset := msgBlock.Offset - msgBlock.Messages()[len(msgBlock.Messages())-1].Offset - offset += baseOffset - if msg.Msg.LogAppendTime { - timestamp = msgBlock.Msg.Timestamp - } - } - if offset < child.offset { - continue - } - messages = append(messages, &ConsumerMessage{ - Topic: child.topic, - Partition: child.partition, - Key: msg.Msg.Key, - Value: msg.Msg.Value, - Offset: offset, - Timestamp: timestamp, - BlockTimestamp: msgBlock.Msg.Timestamp, - }) - child.offset = offset + 1 - } - } - if len(messages) == 0 { - child.offset++ - } - return messages, nil -} - -func (child *partitionConsumer) parseRecords(batch *RecordBatch) ([]*ConsumerMessage, error) { - messages := make([]*ConsumerMessage, 0, len(batch.Records)) - - for _, rec := range batch.Records { - offset := batch.FirstOffset + rec.OffsetDelta - if offset < child.offset { - continue - } - timestamp := batch.FirstTimestamp.Add(rec.TimestampDelta) - if batch.LogAppendTime { - timestamp = batch.MaxTimestamp - } - messages = append(messages, &ConsumerMessage{ - Topic: child.topic, - Partition: child.partition, - Key: rec.Key, - Value: rec.Value, - Offset: offset, - Timestamp: timestamp, - Headers: rec.Headers, - }) - child.offset = offset + 1 - } - if len(messages) == 0 { - child.offset++ - } - return messages, nil -} - -func (child *partitionConsumer) parseResponse(response *FetchResponse) ([]*ConsumerMessage, error) { - var ( - metricRegistry = child.conf.MetricRegistry - consumerBatchSizeMetric metrics.Histogram - ) - - if metricRegistry != nil { - consumerBatchSizeMetric = getOrRegisterHistogram("consumer-batch-size", metricRegistry) - } - - // If request was throttled and empty we log and return without error - if response.ThrottleTime != time.Duration(0) && len(response.Blocks) == 0 { - Logger.Printf( - "consumer/broker/%d FetchResponse throttled %v\n", - child.broker.broker.ID(), response.ThrottleTime) - return nil, nil - } - - block := response.GetBlock(child.topic, child.partition) - if block == nil { - return nil, ErrIncompleteResponse - } - - if block.Err != ErrNoError { - return nil, block.Err - } - - nRecs, err := block.numRecords() - if err != nil { - return nil, err - } - - consumerBatchSizeMetric.Update(int64(nRecs)) - - if nRecs == 0 { - partialTrailingMessage, err := block.isPartial() - if err != nil { - return nil, err - } - // We got no messages. If we got a trailing one then we need to ask for more data. - // Otherwise we just poll again and wait for one to be produced... - if partialTrailingMessage { - if child.conf.Consumer.Fetch.Max > 0 && child.fetchSize == child.conf.Consumer.Fetch.Max { - // we can't ask for more data, we've hit the configured limit - child.sendError(ErrMessageTooLarge) - child.offset++ // skip this one so we can keep processing future messages - } else { - child.fetchSize *= 2 - // check int32 overflow - if child.fetchSize < 0 { - child.fetchSize = math.MaxInt32 - } - if child.conf.Consumer.Fetch.Max > 0 && child.fetchSize > child.conf.Consumer.Fetch.Max { - child.fetchSize = child.conf.Consumer.Fetch.Max - } - } - } - - return nil, nil - } - - // we got messages, reset our fetch size in case it was increased for a previous request - child.fetchSize = child.conf.Consumer.Fetch.Default - atomic.StoreInt64(&child.highWaterMarkOffset, block.HighWaterMarkOffset) - - // abortedProducerIDs contains producerID which message should be ignored as uncommitted - // - producerID are added when the partitionConsumer iterate over the offset at which an aborted transaction begins (abortedTransaction.FirstOffset) - // - producerID are removed when partitionConsumer iterate over an aborted controlRecord, meaning the aborted transaction for this producer is over - abortedProducerIDs := make(map[int64]struct{}, len(block.AbortedTransactions)) - abortedTransactions := block.getAbortedTransactions() - - var messages []*ConsumerMessage - for _, records := range block.RecordsSet { - switch records.recordsType { - case legacyRecords: - messageSetMessages, err := child.parseMessages(records.MsgSet) - if err != nil { - return nil, err - } - - messages = append(messages, messageSetMessages...) - case defaultRecords: - // Consume remaining abortedTransaction up to last offset of current batch - for _, txn := range abortedTransactions { - if txn.FirstOffset > records.RecordBatch.LastOffset() { - break - } - abortedProducerIDs[txn.ProducerID] = struct{}{} - // Pop abortedTransactions so that we never add it again - abortedTransactions = abortedTransactions[1:] - } - - recordBatchMessages, err := child.parseRecords(records.RecordBatch) - if err != nil { - return nil, err - } - - // Parse and commit offset but do not expose messages that are: - // - control records - // - part of an aborted transaction when set to `ReadCommitted` - - // control record - isControl, err := records.isControl() - if err != nil { - // I don't know why there is this continue in case of error to begin with - // Safe bet is to ignore control messages if ReadUncommitted - // and block on them in case of error and ReadCommitted - if child.conf.Consumer.IsolationLevel == ReadCommitted { - return nil, err - } - continue - } - if isControl { - controlRecord, err := records.getControlRecord() - if err != nil { - return nil, err - } - - if controlRecord.Type == ControlRecordAbort { - delete(abortedProducerIDs, records.RecordBatch.ProducerID) - } - continue - } - - // filter aborted transactions - if child.conf.Consumer.IsolationLevel == ReadCommitted { - _, isAborted := abortedProducerIDs[records.RecordBatch.ProducerID] - if records.RecordBatch.IsTransactional && isAborted { - continue - } - } - - messages = append(messages, recordBatchMessages...) - default: - return nil, fmt.Errorf("unknown records type: %v", records.recordsType) - } - } - - return messages, nil -} - -type brokerConsumer struct { - consumer *consumer - broker *Broker - input chan *partitionConsumer - newSubscriptions chan []*partitionConsumer - subscriptions map[*partitionConsumer]none - wait chan none - acks sync.WaitGroup - refs int -} - -func (c *consumer) newBrokerConsumer(broker *Broker) *brokerConsumer { - bc := &brokerConsumer{ - consumer: c, - broker: broker, - input: make(chan *partitionConsumer), - newSubscriptions: make(chan []*partitionConsumer), - wait: make(chan none), - subscriptions: make(map[*partitionConsumer]none), - refs: 0, - } - - go withRecover(bc.subscriptionManager) - go withRecover(bc.subscriptionConsumer) - - return bc -} - -// The subscriptionManager constantly accepts new subscriptions on `input` (even when the main subscriptionConsumer -// goroutine is in the middle of a network request) and batches it up. The main worker goroutine picks -// up a batch of new subscriptions between every network request by reading from `newSubscriptions`, so we give -// it nil if no new subscriptions are available. We also write to `wait` only when new subscriptions is available, -// so the main goroutine can block waiting for work if it has none. -func (bc *brokerConsumer) subscriptionManager() { - var buffer []*partitionConsumer - - for { - if len(buffer) > 0 { - select { - case event, ok := <-bc.input: - if !ok { - goto done - } - buffer = append(buffer, event) - case bc.newSubscriptions <- buffer: - buffer = nil - case bc.wait <- none{}: - } - } else { - select { - case event, ok := <-bc.input: - if !ok { - goto done - } - buffer = append(buffer, event) - case bc.newSubscriptions <- nil: - } - } - } - -done: - close(bc.wait) - if len(buffer) > 0 { - bc.newSubscriptions <- buffer - } - close(bc.newSubscriptions) -} - -//subscriptionConsumer ensures we will get nil right away if no new subscriptions is available -func (bc *brokerConsumer) subscriptionConsumer() { - <-bc.wait // wait for our first piece of work - - for newSubscriptions := range bc.newSubscriptions { - bc.updateSubscriptions(newSubscriptions) - - if len(bc.subscriptions) == 0 { - // We're about to be shut down or we're about to receive more subscriptions. - // Either way, the signal just hasn't propagated to our goroutine yet. - <-bc.wait - continue - } - - response, err := bc.fetchNewMessages() - - if err != nil { - Logger.Printf("consumer/broker/%d disconnecting due to error processing FetchRequest: %s\n", bc.broker.ID(), err) - bc.abort(err) - return - } - - bc.acks.Add(len(bc.subscriptions)) - for child := range bc.subscriptions { - child.feeder <- response - } - bc.acks.Wait() - bc.handleResponses() - } -} - -func (bc *brokerConsumer) updateSubscriptions(newSubscriptions []*partitionConsumer) { - for _, child := range newSubscriptions { - bc.subscriptions[child] = none{} - Logger.Printf("consumer/broker/%d added subscription to %s/%d\n", bc.broker.ID(), child.topic, child.partition) - } - - for child := range bc.subscriptions { - select { - case <-child.dying: - Logger.Printf("consumer/broker/%d closed dead subscription to %s/%d\n", bc.broker.ID(), child.topic, child.partition) - close(child.trigger) - delete(bc.subscriptions, child) - default: - // no-op - } - } -} - -//handleResponses handles the response codes left for us by our subscriptions, and abandons ones that have been closed -func (bc *brokerConsumer) handleResponses() { - for child := range bc.subscriptions { - result := child.responseResult - child.responseResult = nil - - switch result { - case nil: - // no-op - case errTimedOut: - Logger.Printf("consumer/broker/%d abandoned subscription to %s/%d because consuming was taking too long\n", - bc.broker.ID(), child.topic, child.partition) - delete(bc.subscriptions, child) - case ErrOffsetOutOfRange: - // there's no point in retrying this it will just fail the same way again - // shut it down and force the user to choose what to do - child.sendError(result) - Logger.Printf("consumer/%s/%d shutting down because %s\n", child.topic, child.partition, result) - close(child.trigger) - delete(bc.subscriptions, child) - case ErrUnknownTopicOrPartition, ErrNotLeaderForPartition, ErrLeaderNotAvailable, ErrReplicaNotAvailable: - // not an error, but does need redispatching - Logger.Printf("consumer/broker/%d abandoned subscription to %s/%d because %s\n", - bc.broker.ID(), child.topic, child.partition, result) - child.trigger <- none{} - delete(bc.subscriptions, child) - default: - // dunno, tell the user and try redispatching - child.sendError(result) - Logger.Printf("consumer/broker/%d abandoned subscription to %s/%d because %s\n", - bc.broker.ID(), child.topic, child.partition, result) - child.trigger <- none{} - delete(bc.subscriptions, child) - } - } -} - -func (bc *brokerConsumer) abort(err error) { - bc.consumer.abandonBrokerConsumer(bc) - _ = bc.broker.Close() // we don't care about the error this might return, we already have one - - for child := range bc.subscriptions { - child.sendError(err) - child.trigger <- none{} - } - - for newSubscriptions := range bc.newSubscriptions { - if len(newSubscriptions) == 0 { - <-bc.wait - continue - } - for _, child := range newSubscriptions { - child.sendError(err) - child.trigger <- none{} - } - } -} - -func (bc *brokerConsumer) fetchNewMessages() (*FetchResponse, error) { - request := &FetchRequest{ - MinBytes: bc.consumer.conf.Consumer.Fetch.Min, - MaxWaitTime: int32(bc.consumer.conf.Consumer.MaxWaitTime / time.Millisecond), - } - if bc.consumer.conf.Version.IsAtLeast(V0_9_0_0) { - request.Version = 1 - } - if bc.consumer.conf.Version.IsAtLeast(V0_10_0_0) { - request.Version = 2 - } - if bc.consumer.conf.Version.IsAtLeast(V0_10_1_0) { - request.Version = 3 - request.MaxBytes = MaxResponseSize - } - if bc.consumer.conf.Version.IsAtLeast(V0_11_0_0) { - request.Version = 4 - request.Isolation = bc.consumer.conf.Consumer.IsolationLevel - } - if bc.consumer.conf.Version.IsAtLeast(V1_1_0_0) { - request.Version = 7 - // We do not currently implement KIP-227 FetchSessions. Setting the id to 0 - // and the epoch to -1 tells the broker not to generate as session ID we're going - // to just ignore anyway. - request.SessionID = 0 - request.SessionEpoch = -1 - } - if bc.consumer.conf.Version.IsAtLeast(V2_1_0_0) { - request.Version = 10 - } - if bc.consumer.conf.Version.IsAtLeast(V2_3_0_0) { - request.Version = 11 - request.RackID = bc.consumer.conf.RackID - } - - for child := range bc.subscriptions { - request.AddBlock(child.topic, child.partition, child.offset, child.fetchSize) - } - - return bc.broker.Fetch(request) -} diff --git a/vendor/github.com/Shopify/sarama/consumer_group.go b/vendor/github.com/Shopify/sarama/consumer_group.go deleted file mode 100644 index fcc5792e..00000000 --- a/vendor/github.com/Shopify/sarama/consumer_group.go +++ /dev/null @@ -1,876 +0,0 @@ -package sarama - -import ( - "context" - "errors" - "fmt" - "sort" - "sync" - "time" -) - -// ErrClosedConsumerGroup is the error returned when a method is called on a consumer group that has been closed. -var ErrClosedConsumerGroup = errors.New("kafka: tried to use a consumer group that was closed") - -// ConsumerGroup is responsible for dividing up processing of topics and partitions -// over a collection of processes (the members of the consumer group). -type ConsumerGroup interface { - // Consume joins a cluster of consumers for a given list of topics and - // starts a blocking ConsumerGroupSession through the ConsumerGroupHandler. - // - // The life-cycle of a session is represented by the following steps: - // - // 1. The consumers join the group (as explained in https://kafka.apache.org/documentation/#intro_consumers) - // and is assigned their "fair share" of partitions, aka 'claims'. - // 2. Before processing starts, the handler's Setup() hook is called to notify the user - // of the claims and allow any necessary preparation or alteration of state. - // 3. For each of the assigned claims the handler's ConsumeClaim() function is then called - // in a separate goroutine which requires it to be thread-safe. Any state must be carefully protected - // from concurrent reads/writes. - // 4. The session will persist until one of the ConsumeClaim() functions exits. This can be either when the - // parent context is cancelled or when a server-side rebalance cycle is initiated. - // 5. Once all the ConsumeClaim() loops have exited, the handler's Cleanup() hook is called - // to allow the user to perform any final tasks before a rebalance. - // 6. Finally, marked offsets are committed one last time before claims are released. - // - // Please note, that once a rebalance is triggered, sessions must be completed within - // Config.Consumer.Group.Rebalance.Timeout. This means that ConsumeClaim() functions must exit - // as quickly as possible to allow time for Cleanup() and the final offset commit. If the timeout - // is exceeded, the consumer will be removed from the group by Kafka, which will cause offset - // commit failures. - // This method should be called inside an infinite loop, when a - // server-side rebalance happens, the consumer session will need to be - // recreated to get the new claims. - Consume(ctx context.Context, topics []string, handler ConsumerGroupHandler) error - - // Errors returns a read channel of errors that occurred during the consumer life-cycle. - // By default, errors are logged and not returned over this channel. - // If you want to implement any custom error handling, set your config's - // Consumer.Return.Errors setting to true, and read from this channel. - Errors() <-chan error - - // Close stops the ConsumerGroup and detaches any running sessions. It is required to call - // this function before the object passes out of scope, as it will otherwise leak memory. - Close() error -} - -type consumerGroup struct { - client Client - - config *Config - consumer Consumer - groupID string - memberID string - errors chan error - - lock sync.Mutex - closed chan none - closeOnce sync.Once - - userData []byte -} - -// NewConsumerGroup creates a new consumer group the given broker addresses and configuration. -func NewConsumerGroup(addrs []string, groupID string, config *Config) (ConsumerGroup, error) { - client, err := NewClient(addrs, config) - if err != nil { - return nil, err - } - - c, err := newConsumerGroup(groupID, client) - if err != nil { - _ = client.Close() - } - return c, err -} - -// NewConsumerGroupFromClient creates a new consumer group using the given client. It is still -// necessary to call Close() on the underlying client when shutting down this consumer. -// PLEASE NOTE: consumer groups can only re-use but not share clients. -func NewConsumerGroupFromClient(groupID string, client Client) (ConsumerGroup, error) { - // For clients passed in by the client, ensure we don't - // call Close() on it. - cli := &nopCloserClient{client} - return newConsumerGroup(groupID, cli) -} - -func newConsumerGroup(groupID string, client Client) (ConsumerGroup, error) { - config := client.Config() - if !config.Version.IsAtLeast(V0_10_2_0) { - return nil, ConfigurationError("consumer groups require Version to be >= V0_10_2_0") - } - - consumer, err := NewConsumerFromClient(client) - if err != nil { - return nil, err - } - - return &consumerGroup{ - client: client, - consumer: consumer, - config: config, - groupID: groupID, - errors: make(chan error, config.ChannelBufferSize), - closed: make(chan none), - }, nil -} - -// Errors implements ConsumerGroup. -func (c *consumerGroup) Errors() <-chan error { return c.errors } - -// Close implements ConsumerGroup. -func (c *consumerGroup) Close() (err error) { - c.closeOnce.Do(func() { - close(c.closed) - - // leave group - if e := c.leave(); e != nil { - err = e - } - - // drain errors - go func() { - close(c.errors) - }() - for e := range c.errors { - err = e - } - - if e := c.client.Close(); e != nil { - err = e - } - }) - return -} - -// Consume implements ConsumerGroup. -func (c *consumerGroup) Consume(ctx context.Context, topics []string, handler ConsumerGroupHandler) error { - // Ensure group is not closed - select { - case <-c.closed: - return ErrClosedConsumerGroup - default: - } - - c.lock.Lock() - defer c.lock.Unlock() - - // Quick exit when no topics are provided - if len(topics) == 0 { - return fmt.Errorf("no topics provided") - } - - // Refresh metadata for requested topics - if err := c.client.RefreshMetadata(topics...); err != nil { - return err - } - - // Init session - sess, err := c.newSession(ctx, topics, handler, c.config.Consumer.Group.Rebalance.Retry.Max) - if err == ErrClosedClient { - return ErrClosedConsumerGroup - } else if err != nil { - return err - } - - // loop check topic partition numbers changed - // will trigger rebalance when any topic partitions number had changed - // avoid Consume function called again that will generate more than loopCheckPartitionNumbers coroutine - go c.loopCheckPartitionNumbers(topics, sess) - - // Wait for session exit signal - <-sess.ctx.Done() - - // Gracefully release session claims - return sess.release(true) -} - -func (c *consumerGroup) retryNewSession(ctx context.Context, topics []string, handler ConsumerGroupHandler, retries int, refreshCoordinator bool) (*consumerGroupSession, error) { - select { - case <-c.closed: - return nil, ErrClosedConsumerGroup - case <-time.After(c.config.Consumer.Group.Rebalance.Retry.Backoff): - } - - if refreshCoordinator { - err := c.client.RefreshCoordinator(c.groupID) - if err != nil { - return c.retryNewSession(ctx, topics, handler, retries, true) - } - } - - return c.newSession(ctx, topics, handler, retries-1) -} - -func (c *consumerGroup) newSession(ctx context.Context, topics []string, handler ConsumerGroupHandler, retries int) (*consumerGroupSession, error) { - coordinator, err := c.client.Coordinator(c.groupID) - if err != nil { - if retries <= 0 { - return nil, err - } - - return c.retryNewSession(ctx, topics, handler, retries, true) - } - - // Join consumer group - join, err := c.joinGroupRequest(coordinator, topics) - if err != nil { - _ = coordinator.Close() - return nil, err - } - switch join.Err { - case ErrNoError: - c.memberID = join.MemberId - case ErrUnknownMemberId, ErrIllegalGeneration: // reset member ID and retry immediately - c.memberID = "" - return c.newSession(ctx, topics, handler, retries) - case ErrNotCoordinatorForConsumer: // retry after backoff with coordinator refresh - if retries <= 0 { - return nil, join.Err - } - - return c.retryNewSession(ctx, topics, handler, retries, true) - case ErrRebalanceInProgress: // retry after backoff - if retries <= 0 { - return nil, join.Err - } - - return c.retryNewSession(ctx, topics, handler, retries, false) - default: - return nil, join.Err - } - - // Prepare distribution plan if we joined as the leader - var plan BalanceStrategyPlan - if join.LeaderId == join.MemberId { - members, err := join.GetMembers() - if err != nil { - return nil, err - } - - plan, err = c.balance(members) - if err != nil { - return nil, err - } - } - - // Sync consumer group - groupRequest, err := c.syncGroupRequest(coordinator, plan, join.GenerationId) - if err != nil { - _ = coordinator.Close() - return nil, err - } - switch groupRequest.Err { - case ErrNoError: - case ErrUnknownMemberId, ErrIllegalGeneration: // reset member ID and retry immediately - c.memberID = "" - return c.newSession(ctx, topics, handler, retries) - case ErrNotCoordinatorForConsumer: // retry after backoff with coordinator refresh - if retries <= 0 { - return nil, groupRequest.Err - } - - return c.retryNewSession(ctx, topics, handler, retries, true) - case ErrRebalanceInProgress: // retry after backoff - if retries <= 0 { - return nil, groupRequest.Err - } - - return c.retryNewSession(ctx, topics, handler, retries, false) - default: - return nil, groupRequest.Err - } - - // Retrieve and sort claims - var claims map[string][]int32 - if len(groupRequest.MemberAssignment) > 0 { - members, err := groupRequest.GetMemberAssignment() - if err != nil { - return nil, err - } - claims = members.Topics - c.userData = members.UserData - - for _, partitions := range claims { - sort.Sort(int32Slice(partitions)) - } - } - - return newConsumerGroupSession(ctx, c, claims, join.MemberId, join.GenerationId, handler) -} - -func (c *consumerGroup) joinGroupRequest(coordinator *Broker, topics []string) (*JoinGroupResponse, error) { - req := &JoinGroupRequest{ - GroupId: c.groupID, - MemberId: c.memberID, - SessionTimeout: int32(c.config.Consumer.Group.Session.Timeout / time.Millisecond), - ProtocolType: "consumer", - } - if c.config.Version.IsAtLeast(V0_10_1_0) { - req.Version = 1 - req.RebalanceTimeout = int32(c.config.Consumer.Group.Rebalance.Timeout / time.Millisecond) - } - - // use static user-data if configured, otherwise use consumer-group userdata from the last sync - userData := c.config.Consumer.Group.Member.UserData - if len(userData) == 0 { - userData = c.userData - } - meta := &ConsumerGroupMemberMetadata{ - Topics: topics, - UserData: userData, - } - strategy := c.config.Consumer.Group.Rebalance.Strategy - if err := req.AddGroupProtocolMetadata(strategy.Name(), meta); err != nil { - return nil, err - } - - return coordinator.JoinGroup(req) -} - -func (c *consumerGroup) syncGroupRequest(coordinator *Broker, plan BalanceStrategyPlan, generationID int32) (*SyncGroupResponse, error) { - req := &SyncGroupRequest{ - GroupId: c.groupID, - MemberId: c.memberID, - GenerationId: generationID, - } - strategy := c.config.Consumer.Group.Rebalance.Strategy - for memberID, topics := range plan { - assignment := &ConsumerGroupMemberAssignment{Topics: topics} - userDataBytes, err := strategy.AssignmentData(memberID, topics, generationID) - if err != nil { - return nil, err - } - assignment.UserData = userDataBytes - if err := req.AddGroupAssignmentMember(memberID, assignment); err != nil { - return nil, err - } - } - return coordinator.SyncGroup(req) -} - -func (c *consumerGroup) heartbeatRequest(coordinator *Broker, memberID string, generationID int32) (*HeartbeatResponse, error) { - req := &HeartbeatRequest{ - GroupId: c.groupID, - MemberId: memberID, - GenerationId: generationID, - } - - return coordinator.Heartbeat(req) -} - -func (c *consumerGroup) balance(members map[string]ConsumerGroupMemberMetadata) (BalanceStrategyPlan, error) { - topics := make(map[string][]int32) - for _, meta := range members { - for _, topic := range meta.Topics { - topics[topic] = nil - } - } - - for topic := range topics { - partitions, err := c.client.Partitions(topic) - if err != nil { - return nil, err - } - topics[topic] = partitions - } - - strategy := c.config.Consumer.Group.Rebalance.Strategy - return strategy.Plan(members, topics) -} - -// Leaves the cluster, called by Close. -func (c *consumerGroup) leave() error { - c.lock.Lock() - defer c.lock.Unlock() - if c.memberID == "" { - return nil - } - - coordinator, err := c.client.Coordinator(c.groupID) - if err != nil { - return err - } - - resp, err := coordinator.LeaveGroup(&LeaveGroupRequest{ - GroupId: c.groupID, - MemberId: c.memberID, - }) - if err != nil { - _ = coordinator.Close() - return err - } - - // Unset memberID - c.memberID = "" - - // Check response - switch resp.Err { - case ErrRebalanceInProgress, ErrUnknownMemberId, ErrNoError: - return nil - default: - return resp.Err - } -} - -func (c *consumerGroup) handleError(err error, topic string, partition int32) { - if _, ok := err.(*ConsumerError); !ok && topic != "" && partition > -1 { - err = &ConsumerError{ - Topic: topic, - Partition: partition, - Err: err, - } - } - - if !c.config.Consumer.Return.Errors { - Logger.Println(err) - return - } - - select { - case <-c.closed: - //consumer is closed - return - default: - } - - select { - case c.errors <- err: - default: - // no error listener - } -} - -func (c *consumerGroup) loopCheckPartitionNumbers(topics []string, session *consumerGroupSession) { - pause := time.NewTicker(c.config.Metadata.RefreshFrequency) - defer session.cancel() - defer pause.Stop() - var oldTopicToPartitionNum map[string]int - var err error - if oldTopicToPartitionNum, err = c.topicToPartitionNumbers(topics); err != nil { - return - } - for { - if newTopicToPartitionNum, err := c.topicToPartitionNumbers(topics); err != nil { - return - } else { - for topic, num := range oldTopicToPartitionNum { - if newTopicToPartitionNum[topic] != num { - return // trigger the end of the session on exit - } - } - } - select { - case <-pause.C: - case <-session.ctx.Done(): - Logger.Printf("loop check partition number coroutine will exit, topics %s", topics) - // if session closed by other, should be exited - return - case <-c.closed: - return - } - } -} - -func (c *consumerGroup) topicToPartitionNumbers(topics []string) (map[string]int, error) { - topicToPartitionNum := make(map[string]int, len(topics)) - for _, topic := range topics { - if partitionNum, err := c.client.Partitions(topic); err != nil { - Logger.Printf("Consumer Group topic %s get partition number failed %v", topic, err) - return nil, err - } else { - topicToPartitionNum[topic] = len(partitionNum) - } - } - return topicToPartitionNum, nil -} - -// -------------------------------------------------------------------- - -// ConsumerGroupSession represents a consumer group member session. -type ConsumerGroupSession interface { - // Claims returns information about the claimed partitions by topic. - Claims() map[string][]int32 - - // MemberID returns the cluster member ID. - MemberID() string - - // GenerationID returns the current generation ID. - GenerationID() int32 - - // MarkOffset marks the provided offset, alongside a metadata string - // that represents the state of the partition consumer at that point in time. The - // metadata string can be used by another consumer to restore that state, so it - // can resume consumption. - // - // To follow upstream conventions, you are expected to mark the offset of the - // next message to read, not the last message read. Thus, when calling `MarkOffset` - // you should typically add one to the offset of the last consumed message. - // - // Note: calling MarkOffset does not necessarily commit the offset to the backend - // store immediately for efficiency reasons, and it may never be committed if - // your application crashes. This means that you may end up processing the same - // message twice, and your processing should ideally be idempotent. - MarkOffset(topic string, partition int32, offset int64, metadata string) - - // Commit the offset to the backend - // - // Note: calling Commit performs a blocking synchronous operation. - Commit() - - // ResetOffset resets to the provided offset, alongside a metadata string that - // represents the state of the partition consumer at that point in time. Reset - // acts as a counterpart to MarkOffset, the difference being that it allows to - // reset an offset to an earlier or smaller value, where MarkOffset only - // allows incrementing the offset. cf MarkOffset for more details. - ResetOffset(topic string, partition int32, offset int64, metadata string) - - // MarkMessage marks a message as consumed. - MarkMessage(msg *ConsumerMessage, metadata string) - - // Context returns the session context. - Context() context.Context -} - -type consumerGroupSession struct { - parent *consumerGroup - memberID string - generationID int32 - handler ConsumerGroupHandler - - claims map[string][]int32 - offsets *offsetManager - ctx context.Context - cancel func() - - waitGroup sync.WaitGroup - releaseOnce sync.Once - hbDying, hbDead chan none -} - -func newConsumerGroupSession(ctx context.Context, parent *consumerGroup, claims map[string][]int32, memberID string, generationID int32, handler ConsumerGroupHandler) (*consumerGroupSession, error) { - // init offset manager - offsets, err := newOffsetManagerFromClient(parent.groupID, memberID, generationID, parent.client) - if err != nil { - return nil, err - } - - // init context - ctx, cancel := context.WithCancel(ctx) - - // init session - sess := &consumerGroupSession{ - parent: parent, - memberID: memberID, - generationID: generationID, - handler: handler, - offsets: offsets, - claims: claims, - ctx: ctx, - cancel: cancel, - hbDying: make(chan none), - hbDead: make(chan none), - } - - // start heartbeat loop - go sess.heartbeatLoop() - - // create a POM for each claim - for topic, partitions := range claims { - for _, partition := range partitions { - pom, err := offsets.ManagePartition(topic, partition) - if err != nil { - _ = sess.release(false) - return nil, err - } - - // handle POM errors - go func(topic string, partition int32) { - for err := range pom.Errors() { - sess.parent.handleError(err, topic, partition) - } - }(topic, partition) - } - } - - // perform setup - if err := handler.Setup(sess); err != nil { - _ = sess.release(true) - return nil, err - } - - // start consuming - for topic, partitions := range claims { - for _, partition := range partitions { - sess.waitGroup.Add(1) - - go func(topic string, partition int32) { - defer sess.waitGroup.Done() - - // cancel the as session as soon as the first - // goroutine exits - defer sess.cancel() - - // consume a single topic/partition, blocking - sess.consume(topic, partition) - }(topic, partition) - } - } - return sess, nil -} - -func (s *consumerGroupSession) Claims() map[string][]int32 { return s.claims } -func (s *consumerGroupSession) MemberID() string { return s.memberID } -func (s *consumerGroupSession) GenerationID() int32 { return s.generationID } - -func (s *consumerGroupSession) MarkOffset(topic string, partition int32, offset int64, metadata string) { - if pom := s.offsets.findPOM(topic, partition); pom != nil { - pom.MarkOffset(offset, metadata) - } -} - -func (s *consumerGroupSession) Commit() { - s.offsets.Commit() -} - -func (s *consumerGroupSession) ResetOffset(topic string, partition int32, offset int64, metadata string) { - if pom := s.offsets.findPOM(topic, partition); pom != nil { - pom.ResetOffset(offset, metadata) - } -} - -func (s *consumerGroupSession) MarkMessage(msg *ConsumerMessage, metadata string) { - s.MarkOffset(msg.Topic, msg.Partition, msg.Offset+1, metadata) -} - -func (s *consumerGroupSession) Context() context.Context { - return s.ctx -} - -func (s *consumerGroupSession) consume(topic string, partition int32) { - // quick exit if rebalance is due - select { - case <-s.ctx.Done(): - return - case <-s.parent.closed: - return - default: - } - - // get next offset - offset := s.parent.config.Consumer.Offsets.Initial - if pom := s.offsets.findPOM(topic, partition); pom != nil { - offset, _ = pom.NextOffset() - } - - // create new claim - claim, err := newConsumerGroupClaim(s, topic, partition, offset) - if err != nil { - s.parent.handleError(err, topic, partition) - return - } - - // handle errors - go func() { - for err := range claim.Errors() { - s.parent.handleError(err, topic, partition) - } - }() - - // trigger close when session is done - go func() { - select { - case <-s.ctx.Done(): - case <-s.parent.closed: - } - claim.AsyncClose() - }() - - // start processing - if err := s.handler.ConsumeClaim(s, claim); err != nil { - s.parent.handleError(err, topic, partition) - } - - // ensure consumer is closed & drained - claim.AsyncClose() - for _, err := range claim.waitClosed() { - s.parent.handleError(err, topic, partition) - } -} - -func (s *consumerGroupSession) release(withCleanup bool) (err error) { - // signal release, stop heartbeat - s.cancel() - - // wait for consumers to exit - s.waitGroup.Wait() - - // perform release - s.releaseOnce.Do(func() { - if withCleanup { - if e := s.handler.Cleanup(s); e != nil { - s.parent.handleError(e, "", -1) - err = e - } - } - - if e := s.offsets.Close(); e != nil { - err = e - } - - close(s.hbDying) - <-s.hbDead - }) - - return -} - -func (s *consumerGroupSession) heartbeatLoop() { - defer close(s.hbDead) - defer s.cancel() // trigger the end of the session on exit - - pause := time.NewTicker(s.parent.config.Consumer.Group.Heartbeat.Interval) - defer pause.Stop() - - retries := s.parent.config.Metadata.Retry.Max - for { - coordinator, err := s.parent.client.Coordinator(s.parent.groupID) - if err != nil { - if retries <= 0 { - s.parent.handleError(err, "", -1) - return - } - - select { - case <-s.hbDying: - return - case <-time.After(s.parent.config.Metadata.Retry.Backoff): - retries-- - } - continue - } - - resp, err := s.parent.heartbeatRequest(coordinator, s.memberID, s.generationID) - if err != nil { - _ = coordinator.Close() - - if retries <= 0 { - s.parent.handleError(err, "", -1) - return - } - - retries-- - continue - } - - switch resp.Err { - case ErrNoError: - retries = s.parent.config.Metadata.Retry.Max - case ErrRebalanceInProgress, ErrUnknownMemberId, ErrIllegalGeneration: - return - default: - s.parent.handleError(resp.Err, "", -1) - return - } - - select { - case <-pause.C: - case <-s.hbDying: - return - } - } -} - -// -------------------------------------------------------------------- - -// ConsumerGroupHandler instances are used to handle individual topic/partition claims. -// It also provides hooks for your consumer group session life-cycle and allow you to -// trigger logic before or after the consume loop(s). -// -// PLEASE NOTE that handlers are likely be called from several goroutines concurrently, -// ensure that all state is safely protected against race conditions. -type ConsumerGroupHandler interface { - // Setup is run at the beginning of a new session, before ConsumeClaim. - Setup(ConsumerGroupSession) error - - // Cleanup is run at the end of a session, once all ConsumeClaim goroutines have exited - // but before the offsets are committed for the very last time. - Cleanup(ConsumerGroupSession) error - - // ConsumeClaim must start a consumer loop of ConsumerGroupClaim's Messages(). - // Once the Messages() channel is closed, the Handler must finish its processing - // loop and exit. - ConsumeClaim(ConsumerGroupSession, ConsumerGroupClaim) error -} - -// ConsumerGroupClaim processes Kafka messages from a given topic and partition within a consumer group. -type ConsumerGroupClaim interface { - // Topic returns the consumed topic name. - Topic() string - - // Partition returns the consumed partition. - Partition() int32 - - // InitialOffset returns the initial offset that was used as a starting point for this claim. - InitialOffset() int64 - - // HighWaterMarkOffset returns the high water mark offset of the partition, - // i.e. the offset that will be used for the next message that will be produced. - // You can use this to determine how far behind the processing is. - HighWaterMarkOffset() int64 - - // Messages returns the read channel for the messages that are returned by - // the broker. The messages channel will be closed when a new rebalance cycle - // is due. You must finish processing and mark offsets within - // Config.Consumer.Group.Session.Timeout before the topic/partition is eventually - // re-assigned to another group member. - Messages() <-chan *ConsumerMessage -} - -type consumerGroupClaim struct { - topic string - partition int32 - offset int64 - PartitionConsumer -} - -func newConsumerGroupClaim(sess *consumerGroupSession, topic string, partition int32, offset int64) (*consumerGroupClaim, error) { - pcm, err := sess.parent.consumer.ConsumePartition(topic, partition, offset) - if err == ErrOffsetOutOfRange { - offset = sess.parent.config.Consumer.Offsets.Initial - pcm, err = sess.parent.consumer.ConsumePartition(topic, partition, offset) - } - if err != nil { - return nil, err - } - - go func() { - for err := range pcm.Errors() { - sess.parent.handleError(err, topic, partition) - } - }() - - return &consumerGroupClaim{ - topic: topic, - partition: partition, - offset: offset, - PartitionConsumer: pcm, - }, nil -} - -func (c *consumerGroupClaim) Topic() string { return c.topic } -func (c *consumerGroupClaim) Partition() int32 { return c.partition } -func (c *consumerGroupClaim) InitialOffset() int64 { return c.offset } - -// Drains messages and errors, ensures the claim is fully closed. -func (c *consumerGroupClaim) waitClosed() (errs ConsumerErrors) { - go func() { - for range c.Messages() { - } - }() - - for err := range c.Errors() { - errs = append(errs, err) - } - return -} diff --git a/vendor/github.com/Shopify/sarama/consumer_group_members.go b/vendor/github.com/Shopify/sarama/consumer_group_members.go deleted file mode 100644 index 2d02cc38..00000000 --- a/vendor/github.com/Shopify/sarama/consumer_group_members.go +++ /dev/null @@ -1,96 +0,0 @@ -package sarama - -//ConsumerGroupMemberMetadata holds the metadata for consumer group -type ConsumerGroupMemberMetadata struct { - Version int16 - Topics []string - UserData []byte -} - -func (m *ConsumerGroupMemberMetadata) encode(pe packetEncoder) error { - pe.putInt16(m.Version) - - if err := pe.putStringArray(m.Topics); err != nil { - return err - } - - if err := pe.putBytes(m.UserData); err != nil { - return err - } - - return nil -} - -func (m *ConsumerGroupMemberMetadata) decode(pd packetDecoder) (err error) { - if m.Version, err = pd.getInt16(); err != nil { - return - } - - if m.Topics, err = pd.getStringArray(); err != nil { - return - } - - if m.UserData, err = pd.getBytes(); err != nil { - return - } - - return nil -} - -//ConsumerGroupMemberAssignment holds the member assignment for a consume group -type ConsumerGroupMemberAssignment struct { - Version int16 - Topics map[string][]int32 - UserData []byte -} - -func (m *ConsumerGroupMemberAssignment) encode(pe packetEncoder) error { - pe.putInt16(m.Version) - - if err := pe.putArrayLength(len(m.Topics)); err != nil { - return err - } - - for topic, partitions := range m.Topics { - if err := pe.putString(topic); err != nil { - return err - } - if err := pe.putInt32Array(partitions); err != nil { - return err - } - } - - if err := pe.putBytes(m.UserData); err != nil { - return err - } - - return nil -} - -func (m *ConsumerGroupMemberAssignment) decode(pd packetDecoder) (err error) { - if m.Version, err = pd.getInt16(); err != nil { - return - } - - var topicLen int - if topicLen, err = pd.getArrayLength(); err != nil { - return - } - - m.Topics = make(map[string][]int32, topicLen) - for i := 0; i < topicLen; i++ { - var topic string - if topic, err = pd.getString(); err != nil { - return - } - if m.Topics[topic], err = pd.getInt32Array(); err != nil { - return - } - } - - if m.UserData, err = pd.getBytes(); err != nil { - return - } - - return nil -} diff --git a/vendor/github.com/Shopify/sarama/consumer_metadata_request.go b/vendor/github.com/Shopify/sarama/consumer_metadata_request.go deleted file mode 100644 index e5ebdaef..00000000 --- a/vendor/github.com/Shopify/sarama/consumer_metadata_request.go +++ /dev/null @@ -1,38 +0,0 @@ -package sarama - -//ConsumerMetadataRequest is used for metadata requests -type ConsumerMetadataRequest struct { - ConsumerGroup string -} - -func (r *ConsumerMetadataRequest) encode(pe packetEncoder) error { - tmp := new(FindCoordinatorRequest) - tmp.CoordinatorKey = r.ConsumerGroup - tmp.CoordinatorType = CoordinatorGroup - return tmp.encode(pe) -} - -func (r *ConsumerMetadataRequest) decode(pd packetDecoder, version int16) (err error) { - tmp := new(FindCoordinatorRequest) - if err := tmp.decode(pd, version); err != nil { - return err - } - r.ConsumerGroup = tmp.CoordinatorKey - return nil -} - -func (r *ConsumerMetadataRequest) key() int16 { - return 10 -} - -func (r *ConsumerMetadataRequest) version() int16 { - return 0 -} - -func (r *ConsumerMetadataRequest) headerVersion() int16 { - return 1 -} - -func (r *ConsumerMetadataRequest) requiredVersion() KafkaVersion { - return V0_8_2_0 -} diff --git a/vendor/github.com/Shopify/sarama/consumer_metadata_response.go b/vendor/github.com/Shopify/sarama/consumer_metadata_response.go deleted file mode 100644 index 1b5d00d2..00000000 --- a/vendor/github.com/Shopify/sarama/consumer_metadata_response.go +++ /dev/null @@ -1,82 +0,0 @@ -package sarama - -import ( - "net" - "strconv" -) - -//ConsumerMetadataResponse holds the response for a consumer group meta data requests -type ConsumerMetadataResponse struct { - Err KError - Coordinator *Broker - CoordinatorID int32 // deprecated: use Coordinator.ID() - CoordinatorHost string // deprecated: use Coordinator.Addr() - CoordinatorPort int32 // deprecated: use Coordinator.Addr() -} - -func (r *ConsumerMetadataResponse) decode(pd packetDecoder, version int16) (err error) { - tmp := new(FindCoordinatorResponse) - - if err := tmp.decode(pd, version); err != nil { - return err - } - - r.Err = tmp.Err - - r.Coordinator = tmp.Coordinator - if tmp.Coordinator == nil { - return nil - } - - // this can all go away in 2.0, but we have to fill in deprecated fields to maintain - // backwards compatibility - host, portstr, err := net.SplitHostPort(r.Coordinator.Addr()) - if err != nil { - return err - } - port, err := strconv.ParseInt(portstr, 10, 32) - if err != nil { - return err - } - r.CoordinatorID = r.Coordinator.ID() - r.CoordinatorHost = host - r.CoordinatorPort = int32(port) - - return nil -} - -func (r *ConsumerMetadataResponse) encode(pe packetEncoder) error { - if r.Coordinator == nil { - r.Coordinator = new(Broker) - r.Coordinator.id = r.CoordinatorID - r.Coordinator.addr = net.JoinHostPort(r.CoordinatorHost, strconv.Itoa(int(r.CoordinatorPort))) - } - - tmp := &FindCoordinatorResponse{ - Version: 0, - Err: r.Err, - Coordinator: r.Coordinator, - } - - if err := tmp.encode(pe); err != nil { - return err - } - - return nil -} - -func (r *ConsumerMetadataResponse) key() int16 { - return 10 -} - -func (r *ConsumerMetadataResponse) version() int16 { - return 0 -} - -func (r *ConsumerMetadataResponse) headerVersion() int16 { - return 0 -} - -func (r *ConsumerMetadataResponse) requiredVersion() KafkaVersion { - return V0_8_2_0 -} diff --git a/vendor/github.com/Shopify/sarama/control_record.go b/vendor/github.com/Shopify/sarama/control_record.go deleted file mode 100644 index 9b75ab53..00000000 --- a/vendor/github.com/Shopify/sarama/control_record.go +++ /dev/null @@ -1,72 +0,0 @@ -package sarama - -//ControlRecordType ... -type ControlRecordType int - -const ( - //ControlRecordAbort is a control record for abort - ControlRecordAbort ControlRecordType = iota - //ControlRecordCommit is a control record for commit - ControlRecordCommit - //ControlRecordUnknown is a control record of unknown type - ControlRecordUnknown -) - -// Control records are returned as a record by fetchRequest -// However unlike "normal" records, they mean nothing application wise. -// They only serve internal logic for supporting transactions. -type ControlRecord struct { - Version int16 - CoordinatorEpoch int32 - Type ControlRecordType -} - -func (cr *ControlRecord) decode(key, value packetDecoder) error { - var err error - cr.Version, err = value.getInt16() - if err != nil { - return err - } - - cr.CoordinatorEpoch, err = value.getInt32() - if err != nil { - return err - } - - // There a version for the value part AND the key part. And I have no idea if they are supposed to match or not - // Either way, all these version can only be 0 for now - cr.Version, err = key.getInt16() - if err != nil { - return err - } - - recordType, err := key.getInt16() - if err != nil { - return err - } - - switch recordType { - case 0: - cr.Type = ControlRecordAbort - case 1: - cr.Type = ControlRecordCommit - default: - // from JAVA implementation: - // UNKNOWN is used to indicate a control type which the client is not aware of and should be ignored - cr.Type = ControlRecordUnknown - } - return nil -} - -func (cr *ControlRecord) encode(key, value packetEncoder) { - value.putInt16(cr.Version) - value.putInt32(cr.CoordinatorEpoch) - key.putInt16(cr.Version) - - switch cr.Type { - case ControlRecordAbort: - key.putInt16(0) - case ControlRecordCommit: - key.putInt16(1) - } -} diff --git a/vendor/github.com/Shopify/sarama/crc32_field.go b/vendor/github.com/Shopify/sarama/crc32_field.go deleted file mode 100644 index 38189a3c..00000000 --- a/vendor/github.com/Shopify/sarama/crc32_field.go +++ /dev/null @@ -1,86 +0,0 @@ -package sarama - -import ( - "encoding/binary" - "fmt" - "hash/crc32" - "sync" -) - -type crcPolynomial int8 - -const ( - crcIEEE crcPolynomial = iota - crcCastagnoli -) - -var crc32FieldPool = sync.Pool{} - -func acquireCrc32Field(polynomial crcPolynomial) *crc32Field { - val := crc32FieldPool.Get() - if val != nil { - c := val.(*crc32Field) - c.polynomial = polynomial - return c - } - return newCRC32Field(polynomial) -} - -func releaseCrc32Field(c *crc32Field) { - crc32FieldPool.Put(c) -} - -var castagnoliTable = crc32.MakeTable(crc32.Castagnoli) - -// crc32Field implements the pushEncoder and pushDecoder interfaces for calculating CRC32s. -type crc32Field struct { - startOffset int - polynomial crcPolynomial -} - -func (c *crc32Field) saveOffset(in int) { - c.startOffset = in -} - -func (c *crc32Field) reserveLength() int { - return 4 -} - -func newCRC32Field(polynomial crcPolynomial) *crc32Field { - return &crc32Field{polynomial: polynomial} -} - -func (c *crc32Field) run(curOffset int, buf []byte) error { - crc, err := c.crc(curOffset, buf) - if err != nil { - return err - } - binary.BigEndian.PutUint32(buf[c.startOffset:], crc) - return nil -} - -func (c *crc32Field) check(curOffset int, buf []byte) error { - crc, err := c.crc(curOffset, buf) - if err != nil { - return err - } - - expected := binary.BigEndian.Uint32(buf[c.startOffset:]) - if crc != expected { - return PacketDecodingError{fmt.Sprintf("CRC didn't match expected %#x got %#x", expected, crc)} - } - - return nil -} -func (c *crc32Field) crc(curOffset int, buf []byte) (uint32, error) { - var tab *crc32.Table - switch c.polynomial { - case crcIEEE: - tab = crc32.IEEETable - case crcCastagnoli: - tab = castagnoliTable - default: - return 0, PacketDecodingError{"invalid CRC type"} - } - return crc32.Checksum(buf[c.startOffset+4:curOffset], tab), nil -} diff --git a/vendor/github.com/Shopify/sarama/create_partitions_request.go b/vendor/github.com/Shopify/sarama/create_partitions_request.go deleted file mode 100644 index 46fb0440..00000000 --- a/vendor/github.com/Shopify/sarama/create_partitions_request.go +++ /dev/null @@ -1,125 +0,0 @@ -package sarama - -import "time" - -type CreatePartitionsRequest struct { - TopicPartitions map[string]*TopicPartition - Timeout time.Duration - ValidateOnly bool -} - -func (c *CreatePartitionsRequest) encode(pe packetEncoder) error { - if err := pe.putArrayLength(len(c.TopicPartitions)); err != nil { - return err - } - - for topic, partition := range c.TopicPartitions { - if err := pe.putString(topic); err != nil { - return err - } - if err := partition.encode(pe); err != nil { - return err - } - } - - pe.putInt32(int32(c.Timeout / time.Millisecond)) - - pe.putBool(c.ValidateOnly) - - return nil -} - -func (c *CreatePartitionsRequest) decode(pd packetDecoder, version int16) (err error) { - n, err := pd.getArrayLength() - if err != nil { - return err - } - c.TopicPartitions = make(map[string]*TopicPartition, n) - for i := 0; i < n; i++ { - topic, err := pd.getString() - if err != nil { - return err - } - c.TopicPartitions[topic] = new(TopicPartition) - if err := c.TopicPartitions[topic].decode(pd, version); err != nil { - return err - } - } - - timeout, err := pd.getInt32() - if err != nil { - return err - } - c.Timeout = time.Duration(timeout) * time.Millisecond - - if c.ValidateOnly, err = pd.getBool(); err != nil { - return err - } - - return nil -} - -func (r *CreatePartitionsRequest) key() int16 { - return 37 -} - -func (r *CreatePartitionsRequest) version() int16 { - return 0 -} - -func (r *CreatePartitionsRequest) headerVersion() int16 { - return 1 -} - -func (r *CreatePartitionsRequest) requiredVersion() KafkaVersion { - return V1_0_0_0 -} - -type TopicPartition struct { - Count int32 - Assignment [][]int32 -} - -func (t *TopicPartition) encode(pe packetEncoder) error { - pe.putInt32(t.Count) - - if len(t.Assignment) == 0 { - pe.putInt32(-1) - return nil - } - - if err := pe.putArrayLength(len(t.Assignment)); err != nil { - return err - } - - for _, assign := range t.Assignment { - if err := pe.putInt32Array(assign); err != nil { - return err - } - } - - return nil -} - -func (t *TopicPartition) decode(pd packetDecoder, version int16) (err error) { - if t.Count, err = pd.getInt32(); err != nil { - return err - } - - n, err := pd.getInt32() - if err != nil { - return err - } - if n <= 0 { - return nil - } - t.Assignment = make([][]int32, n) - - for i := 0; i < int(n); i++ { - if t.Assignment[i], err = pd.getInt32Array(); err != nil { - return err - } - } - - return nil -} diff --git a/vendor/github.com/Shopify/sarama/create_partitions_response.go b/vendor/github.com/Shopify/sarama/create_partitions_response.go deleted file mode 100644 index 12ce7885..00000000 --- a/vendor/github.com/Shopify/sarama/create_partitions_response.go +++ /dev/null @@ -1,109 +0,0 @@ -package sarama - -import ( - "fmt" - "time" -) - -type CreatePartitionsResponse struct { - ThrottleTime time.Duration - TopicPartitionErrors map[string]*TopicPartitionError -} - -func (c *CreatePartitionsResponse) encode(pe packetEncoder) error { - pe.putInt32(int32(c.ThrottleTime / time.Millisecond)) - if err := pe.putArrayLength(len(c.TopicPartitionErrors)); err != nil { - return err - } - - for topic, partitionError := range c.TopicPartitionErrors { - if err := pe.putString(topic); err != nil { - return err - } - if err := partitionError.encode(pe); err != nil { - return err - } - } - - return nil -} - -func (c *CreatePartitionsResponse) decode(pd packetDecoder, version int16) (err error) { - throttleTime, err := pd.getInt32() - if err != nil { - return err - } - c.ThrottleTime = time.Duration(throttleTime) * time.Millisecond - - n, err := pd.getArrayLength() - if err != nil { - return err - } - - c.TopicPartitionErrors = make(map[string]*TopicPartitionError, n) - for i := 0; i < n; i++ { - topic, err := pd.getString() - if err != nil { - return err - } - c.TopicPartitionErrors[topic] = new(TopicPartitionError) - if err := c.TopicPartitionErrors[topic].decode(pd, version); err != nil { - return err - } - } - - return nil -} - -func (r *CreatePartitionsResponse) key() int16 { - return 37 -} - -func (r *CreatePartitionsResponse) version() int16 { - return 0 -} - -func (r *CreatePartitionsResponse) headerVersion() int16 { - return 0 -} - -func (r *CreatePartitionsResponse) requiredVersion() KafkaVersion { - return V1_0_0_0 -} - -type TopicPartitionError struct { - Err KError - ErrMsg *string -} - -func (t *TopicPartitionError) Error() string { - text := t.Err.Error() - if t.ErrMsg != nil { - text = fmt.Sprintf("%s - %s", text, *t.ErrMsg) - } - return text -} - -func (t *TopicPartitionError) encode(pe packetEncoder) error { - pe.putInt16(int16(t.Err)) - - if err := pe.putNullableString(t.ErrMsg); err != nil { - return err - } - - return nil -} - -func (t *TopicPartitionError) decode(pd packetDecoder, version int16) (err error) { - kerr, err := pd.getInt16() - if err != nil { - return err - } - t.Err = KError(kerr) - - if t.ErrMsg, err = pd.getNullableString(); err != nil { - return err - } - - return nil -} diff --git a/vendor/github.com/Shopify/sarama/create_topics_request.go b/vendor/github.com/Shopify/sarama/create_topics_request.go deleted file mode 100644 index 287acd06..00000000 --- a/vendor/github.com/Shopify/sarama/create_topics_request.go +++ /dev/null @@ -1,178 +0,0 @@ -package sarama - -import ( - "time" -) - -type CreateTopicsRequest struct { - Version int16 - - TopicDetails map[string]*TopicDetail - Timeout time.Duration - ValidateOnly bool -} - -func (c *CreateTopicsRequest) encode(pe packetEncoder) error { - if err := pe.putArrayLength(len(c.TopicDetails)); err != nil { - return err - } - for topic, detail := range c.TopicDetails { - if err := pe.putString(topic); err != nil { - return err - } - if err := detail.encode(pe); err != nil { - return err - } - } - - pe.putInt32(int32(c.Timeout / time.Millisecond)) - - if c.Version >= 1 { - pe.putBool(c.ValidateOnly) - } - - return nil -} - -func (c *CreateTopicsRequest) decode(pd packetDecoder, version int16) (err error) { - n, err := pd.getArrayLength() - if err != nil { - return err - } - - c.TopicDetails = make(map[string]*TopicDetail, n) - - for i := 0; i < n; i++ { - topic, err := pd.getString() - if err != nil { - return err - } - c.TopicDetails[topic] = new(TopicDetail) - if err = c.TopicDetails[topic].decode(pd, version); err != nil { - return err - } - } - - timeout, err := pd.getInt32() - if err != nil { - return err - } - c.Timeout = time.Duration(timeout) * time.Millisecond - - if version >= 1 { - c.ValidateOnly, err = pd.getBool() - if err != nil { - return err - } - - c.Version = version - } - - return nil -} - -func (c *CreateTopicsRequest) key() int16 { - return 19 -} - -func (c *CreateTopicsRequest) version() int16 { - return c.Version -} - -func (r *CreateTopicsRequest) headerVersion() int16 { - return 1 -} - -func (c *CreateTopicsRequest) requiredVersion() KafkaVersion { - switch c.Version { - case 2: - return V1_0_0_0 - case 1: - return V0_11_0_0 - default: - return V0_10_1_0 - } -} - -type TopicDetail struct { - NumPartitions int32 - ReplicationFactor int16 - ReplicaAssignment map[int32][]int32 - ConfigEntries map[string]*string -} - -func (t *TopicDetail) encode(pe packetEncoder) error { - pe.putInt32(t.NumPartitions) - pe.putInt16(t.ReplicationFactor) - - if err := pe.putArrayLength(len(t.ReplicaAssignment)); err != nil { - return err - } - for partition, assignment := range t.ReplicaAssignment { - pe.putInt32(partition) - if err := pe.putInt32Array(assignment); err != nil { - return err - } - } - - if err := pe.putArrayLength(len(t.ConfigEntries)); err != nil { - return err - } - for configKey, configValue := range t.ConfigEntries { - if err := pe.putString(configKey); err != nil { - return err - } - if err := pe.putNullableString(configValue); err != nil { - return err - } - } - - return nil -} - -func (t *TopicDetail) decode(pd packetDecoder, version int16) (err error) { - if t.NumPartitions, err = pd.getInt32(); err != nil { - return err - } - if t.ReplicationFactor, err = pd.getInt16(); err != nil { - return err - } - - n, err := pd.getArrayLength() - if err != nil { - return err - } - - if n > 0 { - t.ReplicaAssignment = make(map[int32][]int32, n) - for i := 0; i < n; i++ { - replica, err := pd.getInt32() - if err != nil { - return err - } - if t.ReplicaAssignment[replica], err = pd.getInt32Array(); err != nil { - return err - } - } - } - - n, err = pd.getArrayLength() - if err != nil { - return err - } - - if n > 0 { - t.ConfigEntries = make(map[string]*string, n) - for i := 0; i < n; i++ { - configKey, err := pd.getString() - if err != nil { - return err - } - if t.ConfigEntries[configKey], err = pd.getNullableString(); err != nil { - return err - } - } - } - - return nil -} diff --git a/vendor/github.com/Shopify/sarama/create_topics_response.go b/vendor/github.com/Shopify/sarama/create_topics_response.go deleted file mode 100644 index 7e1448a6..00000000 --- a/vendor/github.com/Shopify/sarama/create_topics_response.go +++ /dev/null @@ -1,127 +0,0 @@ -package sarama - -import ( - "fmt" - "time" -) - -type CreateTopicsResponse struct { - Version int16 - ThrottleTime time.Duration - TopicErrors map[string]*TopicError -} - -func (c *CreateTopicsResponse) encode(pe packetEncoder) error { - if c.Version >= 2 { - pe.putInt32(int32(c.ThrottleTime / time.Millisecond)) - } - - if err := pe.putArrayLength(len(c.TopicErrors)); err != nil { - return err - } - for topic, topicError := range c.TopicErrors { - if err := pe.putString(topic); err != nil { - return err - } - if err := topicError.encode(pe, c.Version); err != nil { - return err - } - } - - return nil -} - -func (c *CreateTopicsResponse) decode(pd packetDecoder, version int16) (err error) { - c.Version = version - - if version >= 2 { - throttleTime, err := pd.getInt32() - if err != nil { - return err - } - c.ThrottleTime = time.Duration(throttleTime) * time.Millisecond - } - - n, err := pd.getArrayLength() - if err != nil { - return err - } - - c.TopicErrors = make(map[string]*TopicError, n) - for i := 0; i < n; i++ { - topic, err := pd.getString() - if err != nil { - return err - } - c.TopicErrors[topic] = new(TopicError) - if err := c.TopicErrors[topic].decode(pd, version); err != nil { - return err - } - } - - return nil -} - -func (c *CreateTopicsResponse) key() int16 { - return 19 -} - -func (c *CreateTopicsResponse) version() int16 { - return c.Version -} - -func (c *CreateTopicsResponse) headerVersion() int16 { - return 0 -} - -func (c *CreateTopicsResponse) requiredVersion() KafkaVersion { - switch c.Version { - case 2: - return V1_0_0_0 - case 1: - return V0_11_0_0 - default: - return V0_10_1_0 - } -} - -type TopicError struct { - Err KError - ErrMsg *string -} - -func (t *TopicError) Error() string { - text := t.Err.Error() - if t.ErrMsg != nil { - text = fmt.Sprintf("%s - %s", text, *t.ErrMsg) - } - return text -} - -func (t *TopicError) encode(pe packetEncoder, version int16) error { - pe.putInt16(int16(t.Err)) - - if version >= 1 { - if err := pe.putNullableString(t.ErrMsg); err != nil { - return err - } - } - - return nil -} - -func (t *TopicError) decode(pd packetDecoder, version int16) (err error) { - kErr, err := pd.getInt16() - if err != nil { - return err - } - t.Err = KError(kErr) - - if version >= 1 { - if t.ErrMsg, err = pd.getNullableString(); err != nil { - return err - } - } - - return nil -} diff --git a/vendor/github.com/Shopify/sarama/decompress.go b/vendor/github.com/Shopify/sarama/decompress.go deleted file mode 100644 index e4dc3c18..00000000 --- a/vendor/github.com/Shopify/sarama/decompress.go +++ /dev/null @@ -1,63 +0,0 @@ -package sarama - -import ( - "bytes" - "compress/gzip" - "fmt" - "io/ioutil" - "sync" - - snappy "github.com/eapache/go-xerial-snappy" - "github.com/pierrec/lz4" -) - -var ( - lz4ReaderPool = sync.Pool{ - New: func() interface{} { - return lz4.NewReader(nil) - }, - } - - gzipReaderPool sync.Pool -) - -func decompress(cc CompressionCodec, data []byte) ([]byte, error) { - switch cc { - case CompressionNone: - return data, nil - case CompressionGZIP: - var ( - err error - reader *gzip.Reader - readerIntf = gzipReaderPool.Get() - ) - if readerIntf != nil { - reader = readerIntf.(*gzip.Reader) - } else { - reader, err = gzip.NewReader(bytes.NewReader(data)) - if err != nil { - return nil, err - } - } - - defer gzipReaderPool.Put(reader) - - if err := reader.Reset(bytes.NewReader(data)); err != nil { - return nil, err - } - - return ioutil.ReadAll(reader) - case CompressionSnappy: - return snappy.Decode(data) - case CompressionLZ4: - reader := lz4ReaderPool.Get().(*lz4.Reader) - defer lz4ReaderPool.Put(reader) - - reader.Reset(bytes.NewReader(data)) - return ioutil.ReadAll(reader) - case CompressionZSTD: - return zstdDecompress(nil, data) - default: - return nil, PacketDecodingError{fmt.Sprintf("invalid compression specified (%d)", cc)} - } -} diff --git a/vendor/github.com/Shopify/sarama/delete_groups_request.go b/vendor/github.com/Shopify/sarama/delete_groups_request.go deleted file mode 100644 index 4ac8bbee..00000000 --- a/vendor/github.com/Shopify/sarama/delete_groups_request.go +++ /dev/null @@ -1,34 +0,0 @@ -package sarama - -type DeleteGroupsRequest struct { - Groups []string -} - -func (r *DeleteGroupsRequest) encode(pe packetEncoder) error { - return pe.putStringArray(r.Groups) -} - -func (r *DeleteGroupsRequest) decode(pd packetDecoder, version int16) (err error) { - r.Groups, err = pd.getStringArray() - return -} - -func (r *DeleteGroupsRequest) key() int16 { - return 42 -} - -func (r *DeleteGroupsRequest) version() int16 { - return 0 -} - -func (r *DeleteGroupsRequest) headerVersion() int16 { - return 1 -} - -func (r *DeleteGroupsRequest) requiredVersion() KafkaVersion { - return V1_1_0_0 -} - -func (r *DeleteGroupsRequest) AddGroup(group string) { - r.Groups = append(r.Groups, group) -} diff --git a/vendor/github.com/Shopify/sarama/delete_groups_response.go b/vendor/github.com/Shopify/sarama/delete_groups_response.go deleted file mode 100644 index 5e7b1ed3..00000000 --- a/vendor/github.com/Shopify/sarama/delete_groups_response.go +++ /dev/null @@ -1,74 +0,0 @@ -package sarama - -import ( - "time" -) - -type DeleteGroupsResponse struct { - ThrottleTime time.Duration - GroupErrorCodes map[string]KError -} - -func (r *DeleteGroupsResponse) encode(pe packetEncoder) error { - pe.putInt32(int32(r.ThrottleTime / time.Millisecond)) - - if err := pe.putArrayLength(len(r.GroupErrorCodes)); err != nil { - return err - } - for groupID, errorCode := range r.GroupErrorCodes { - if err := pe.putString(groupID); err != nil { - return err - } - pe.putInt16(int16(errorCode)) - } - - return nil -} - -func (r *DeleteGroupsResponse) decode(pd packetDecoder, version int16) error { - throttleTime, err := pd.getInt32() - if err != nil { - return err - } - r.ThrottleTime = time.Duration(throttleTime) * time.Millisecond - - n, err := pd.getArrayLength() - if err != nil { - return err - } - if n == 0 { - return nil - } - - r.GroupErrorCodes = make(map[string]KError, n) - for i := 0; i < n; i++ { - groupID, err := pd.getString() - if err != nil { - return err - } - errorCode, err := pd.getInt16() - if err != nil { - return err - } - - r.GroupErrorCodes[groupID] = KError(errorCode) - } - - return nil -} - -func (r *DeleteGroupsResponse) key() int16 { - return 42 -} - -func (r *DeleteGroupsResponse) version() int16 { - return 0 -} - -func (r *DeleteGroupsResponse) headerVersion() int16 { - return 0 -} - -func (r *DeleteGroupsResponse) requiredVersion() KafkaVersion { - return V1_1_0_0 -} diff --git a/vendor/github.com/Shopify/sarama/delete_records_request.go b/vendor/github.com/Shopify/sarama/delete_records_request.go deleted file mode 100644 index dc106b17..00000000 --- a/vendor/github.com/Shopify/sarama/delete_records_request.go +++ /dev/null @@ -1,130 +0,0 @@ -package sarama - -import ( - "sort" - "time" -) - -// request message format is: -// [topic] timeout(int32) -// where topic is: -// name(string) [partition] -// where partition is: -// id(int32) offset(int64) - -type DeleteRecordsRequest struct { - Topics map[string]*DeleteRecordsRequestTopic - Timeout time.Duration -} - -func (d *DeleteRecordsRequest) encode(pe packetEncoder) error { - if err := pe.putArrayLength(len(d.Topics)); err != nil { - return err - } - keys := make([]string, 0, len(d.Topics)) - for topic := range d.Topics { - keys = append(keys, topic) - } - sort.Strings(keys) - for _, topic := range keys { - if err := pe.putString(topic); err != nil { - return err - } - if err := d.Topics[topic].encode(pe); err != nil { - return err - } - } - pe.putInt32(int32(d.Timeout / time.Millisecond)) - - return nil -} - -func (d *DeleteRecordsRequest) decode(pd packetDecoder, version int16) error { - n, err := pd.getArrayLength() - if err != nil { - return err - } - - if n > 0 { - d.Topics = make(map[string]*DeleteRecordsRequestTopic, n) - for i := 0; i < n; i++ { - topic, err := pd.getString() - if err != nil { - return err - } - details := new(DeleteRecordsRequestTopic) - if err = details.decode(pd, version); err != nil { - return err - } - d.Topics[topic] = details - } - } - - timeout, err := pd.getInt32() - if err != nil { - return err - } - d.Timeout = time.Duration(timeout) * time.Millisecond - - return nil -} - -func (d *DeleteRecordsRequest) key() int16 { - return 21 -} - -func (d *DeleteRecordsRequest) version() int16 { - return 0 -} - -func (d *DeleteRecordsRequest) headerVersion() int16 { - return 1 -} - -func (d *DeleteRecordsRequest) requiredVersion() KafkaVersion { - return V0_11_0_0 -} - -type DeleteRecordsRequestTopic struct { - PartitionOffsets map[int32]int64 // partition => offset -} - -func (t *DeleteRecordsRequestTopic) encode(pe packetEncoder) error { - if err := pe.putArrayLength(len(t.PartitionOffsets)); err != nil { - return err - } - keys := make([]int32, 0, len(t.PartitionOffsets)) - for partition := range t.PartitionOffsets { - keys = append(keys, partition) - } - sort.Slice(keys, func(i, j int) bool { return keys[i] < keys[j] }) - for _, partition := range keys { - pe.putInt32(partition) - pe.putInt64(t.PartitionOffsets[partition]) - } - return nil -} - -func (t *DeleteRecordsRequestTopic) decode(pd packetDecoder, version int16) error { - n, err := pd.getArrayLength() - if err != nil { - return err - } - - if n > 0 { - t.PartitionOffsets = make(map[int32]int64, n) - for i := 0; i < n; i++ { - partition, err := pd.getInt32() - if err != nil { - return err - } - offset, err := pd.getInt64() - if err != nil { - return err - } - t.PartitionOffsets[partition] = offset - } - } - - return nil -} diff --git a/vendor/github.com/Shopify/sarama/delete_records_response.go b/vendor/github.com/Shopify/sarama/delete_records_response.go deleted file mode 100644 index d530b4c7..00000000 --- a/vendor/github.com/Shopify/sarama/delete_records_response.go +++ /dev/null @@ -1,162 +0,0 @@ -package sarama - -import ( - "sort" - "time" -) - -// response message format is: -// throttleMs(int32) [topic] -// where topic is: -// name(string) [partition] -// where partition is: -// id(int32) low_watermark(int64) error_code(int16) - -type DeleteRecordsResponse struct { - Version int16 - ThrottleTime time.Duration - Topics map[string]*DeleteRecordsResponseTopic -} - -func (d *DeleteRecordsResponse) encode(pe packetEncoder) error { - pe.putInt32(int32(d.ThrottleTime / time.Millisecond)) - - if err := pe.putArrayLength(len(d.Topics)); err != nil { - return err - } - keys := make([]string, 0, len(d.Topics)) - for topic := range d.Topics { - keys = append(keys, topic) - } - sort.Strings(keys) - for _, topic := range keys { - if err := pe.putString(topic); err != nil { - return err - } - if err := d.Topics[topic].encode(pe); err != nil { - return err - } - } - return nil -} - -func (d *DeleteRecordsResponse) decode(pd packetDecoder, version int16) error { - d.Version = version - - throttleTime, err := pd.getInt32() - if err != nil { - return err - } - d.ThrottleTime = time.Duration(throttleTime) * time.Millisecond - - n, err := pd.getArrayLength() - if err != nil { - return err - } - - if n > 0 { - d.Topics = make(map[string]*DeleteRecordsResponseTopic, n) - for i := 0; i < n; i++ { - topic, err := pd.getString() - if err != nil { - return err - } - details := new(DeleteRecordsResponseTopic) - if err = details.decode(pd, version); err != nil { - return err - } - d.Topics[topic] = details - } - } - - return nil -} - -func (d *DeleteRecordsResponse) key() int16 { - return 21 -} - -func (d *DeleteRecordsResponse) version() int16 { - return 0 -} - -func (d *DeleteRecordsResponse) headerVersion() int16 { - return 0 -} - -func (d *DeleteRecordsResponse) requiredVersion() KafkaVersion { - return V0_11_0_0 -} - -type DeleteRecordsResponseTopic struct { - Partitions map[int32]*DeleteRecordsResponsePartition -} - -func (t *DeleteRecordsResponseTopic) encode(pe packetEncoder) error { - if err := pe.putArrayLength(len(t.Partitions)); err != nil { - return err - } - keys := make([]int32, 0, len(t.Partitions)) - for partition := range t.Partitions { - keys = append(keys, partition) - } - sort.Slice(keys, func(i, j int) bool { return keys[i] < keys[j] }) - for _, partition := range keys { - pe.putInt32(partition) - if err := t.Partitions[partition].encode(pe); err != nil { - return err - } - } - return nil -} - -func (t *DeleteRecordsResponseTopic) decode(pd packetDecoder, version int16) error { - n, err := pd.getArrayLength() - if err != nil { - return err - } - - if n > 0 { - t.Partitions = make(map[int32]*DeleteRecordsResponsePartition, n) - for i := 0; i < n; i++ { - partition, err := pd.getInt32() - if err != nil { - return err - } - details := new(DeleteRecordsResponsePartition) - if err = details.decode(pd, version); err != nil { - return err - } - t.Partitions[partition] = details - } - } - - return nil -} - -type DeleteRecordsResponsePartition struct { - LowWatermark int64 - Err KError -} - -func (t *DeleteRecordsResponsePartition) encode(pe packetEncoder) error { - pe.putInt64(t.LowWatermark) - pe.putInt16(int16(t.Err)) - return nil -} - -func (t *DeleteRecordsResponsePartition) decode(pd packetDecoder, version int16) error { - lowWatermark, err := pd.getInt64() - if err != nil { - return err - } - t.LowWatermark = lowWatermark - - kErr, err := pd.getInt16() - if err != nil { - return err - } - t.Err = KError(kErr) - - return nil -} diff --git a/vendor/github.com/Shopify/sarama/delete_topics_request.go b/vendor/github.com/Shopify/sarama/delete_topics_request.go deleted file mode 100644 index ba6780a8..00000000 --- a/vendor/github.com/Shopify/sarama/delete_topics_request.go +++ /dev/null @@ -1,52 +0,0 @@ -package sarama - -import "time" - -type DeleteTopicsRequest struct { - Version int16 - Topics []string - Timeout time.Duration -} - -func (d *DeleteTopicsRequest) encode(pe packetEncoder) error { - if err := pe.putStringArray(d.Topics); err != nil { - return err - } - pe.putInt32(int32(d.Timeout / time.Millisecond)) - - return nil -} - -func (d *DeleteTopicsRequest) decode(pd packetDecoder, version int16) (err error) { - if d.Topics, err = pd.getStringArray(); err != nil { - return err - } - timeout, err := pd.getInt32() - if err != nil { - return err - } - d.Timeout = time.Duration(timeout) * time.Millisecond - d.Version = version - return nil -} - -func (d *DeleteTopicsRequest) key() int16 { - return 20 -} - -func (d *DeleteTopicsRequest) version() int16 { - return d.Version -} - -func (d *DeleteTopicsRequest) headerVersion() int16 { - return 1 -} - -func (d *DeleteTopicsRequest) requiredVersion() KafkaVersion { - switch d.Version { - case 1: - return V0_11_0_0 - default: - return V0_10_1_0 - } -} diff --git a/vendor/github.com/Shopify/sarama/delete_topics_response.go b/vendor/github.com/Shopify/sarama/delete_topics_response.go deleted file mode 100644 index 733961a8..00000000 --- a/vendor/github.com/Shopify/sarama/delete_topics_response.go +++ /dev/null @@ -1,82 +0,0 @@ -package sarama - -import "time" - -type DeleteTopicsResponse struct { - Version int16 - ThrottleTime time.Duration - TopicErrorCodes map[string]KError -} - -func (d *DeleteTopicsResponse) encode(pe packetEncoder) error { - if d.Version >= 1 { - pe.putInt32(int32(d.ThrottleTime / time.Millisecond)) - } - - if err := pe.putArrayLength(len(d.TopicErrorCodes)); err != nil { - return err - } - for topic, errorCode := range d.TopicErrorCodes { - if err := pe.putString(topic); err != nil { - return err - } - pe.putInt16(int16(errorCode)) - } - - return nil -} - -func (d *DeleteTopicsResponse) decode(pd packetDecoder, version int16) (err error) { - if version >= 1 { - throttleTime, err := pd.getInt32() - if err != nil { - return err - } - d.ThrottleTime = time.Duration(throttleTime) * time.Millisecond - - d.Version = version - } - - n, err := pd.getArrayLength() - if err != nil { - return err - } - - d.TopicErrorCodes = make(map[string]KError, n) - - for i := 0; i < n; i++ { - topic, err := pd.getString() - if err != nil { - return err - } - errorCode, err := pd.getInt16() - if err != nil { - return err - } - - d.TopicErrorCodes[topic] = KError(errorCode) - } - - return nil -} - -func (d *DeleteTopicsResponse) key() int16 { - return 20 -} - -func (d *DeleteTopicsResponse) version() int16 { - return d.Version -} - -func (d *DeleteTopicsResponse) headerVersion() int16 { - return 0 -} - -func (d *DeleteTopicsResponse) requiredVersion() KafkaVersion { - switch d.Version { - case 1: - return V0_11_0_0 - default: - return V0_10_1_0 - } -} diff --git a/vendor/github.com/Shopify/sarama/describe_configs_request.go b/vendor/github.com/Shopify/sarama/describe_configs_request.go deleted file mode 100644 index d0c73528..00000000 --- a/vendor/github.com/Shopify/sarama/describe_configs_request.go +++ /dev/null @@ -1,116 +0,0 @@ -package sarama - -type DescribeConfigsRequest struct { - Version int16 - Resources []*ConfigResource - IncludeSynonyms bool -} - -type ConfigResource struct { - Type ConfigResourceType - Name string - ConfigNames []string -} - -func (r *DescribeConfigsRequest) encode(pe packetEncoder) error { - if err := pe.putArrayLength(len(r.Resources)); err != nil { - return err - } - - for _, c := range r.Resources { - pe.putInt8(int8(c.Type)) - if err := pe.putString(c.Name); err != nil { - return err - } - - if len(c.ConfigNames) == 0 { - pe.putInt32(-1) - continue - } - if err := pe.putStringArray(c.ConfigNames); err != nil { - return err - } - } - - if r.Version >= 1 { - pe.putBool(r.IncludeSynonyms) - } - - return nil -} - -func (r *DescribeConfigsRequest) decode(pd packetDecoder, version int16) (err error) { - n, err := pd.getArrayLength() - if err != nil { - return err - } - - r.Resources = make([]*ConfigResource, n) - - for i := 0; i < n; i++ { - r.Resources[i] = &ConfigResource{} - t, err := pd.getInt8() - if err != nil { - return err - } - r.Resources[i].Type = ConfigResourceType(t) - name, err := pd.getString() - if err != nil { - return err - } - r.Resources[i].Name = name - - confLength, err := pd.getArrayLength() - - if err != nil { - return err - } - - if confLength == -1 { - continue - } - - cfnames := make([]string, confLength) - for i := 0; i < confLength; i++ { - s, err := pd.getString() - if err != nil { - return err - } - cfnames[i] = s - } - r.Resources[i].ConfigNames = cfnames - } - r.Version = version - if r.Version >= 1 { - b, err := pd.getBool() - if err != nil { - return err - } - r.IncludeSynonyms = b - } - - return nil -} - -func (r *DescribeConfigsRequest) key() int16 { - return 32 -} - -func (r *DescribeConfigsRequest) version() int16 { - return r.Version -} - -func (r *DescribeConfigsRequest) headerVersion() int16 { - return 1 -} - -func (r *DescribeConfigsRequest) requiredVersion() KafkaVersion { - switch r.Version { - case 1: - return V1_1_0_0 - case 2: - return V2_0_0_0 - default: - return V0_11_0_0 - } -} diff --git a/vendor/github.com/Shopify/sarama/describe_configs_response.go b/vendor/github.com/Shopify/sarama/describe_configs_response.go deleted file mode 100644 index 063ae911..00000000 --- a/vendor/github.com/Shopify/sarama/describe_configs_response.go +++ /dev/null @@ -1,327 +0,0 @@ -package sarama - -import ( - "fmt" - "time" -) - -type ConfigSource int8 - -func (s ConfigSource) String() string { - switch s { - case SourceUnknown: - return "Unknown" - case SourceTopic: - return "Topic" - case SourceDynamicBroker: - return "DynamicBroker" - case SourceDynamicDefaultBroker: - return "DynamicDefaultBroker" - case SourceStaticBroker: - return "StaticBroker" - case SourceDefault: - return "Default" - } - return fmt.Sprintf("Source Invalid: %d", int(s)) -} - -const ( - SourceUnknown ConfigSource = iota - SourceTopic - SourceDynamicBroker - SourceDynamicDefaultBroker - SourceStaticBroker - SourceDefault -) - -type DescribeConfigsResponse struct { - Version int16 - ThrottleTime time.Duration - Resources []*ResourceResponse -} - -type ResourceResponse struct { - ErrorCode int16 - ErrorMsg string - Type ConfigResourceType - Name string - Configs []*ConfigEntry -} - -type ConfigEntry struct { - Name string - Value string - ReadOnly bool - Default bool - Source ConfigSource - Sensitive bool - Synonyms []*ConfigSynonym -} - -type ConfigSynonym struct { - ConfigName string - ConfigValue string - Source ConfigSource -} - -func (r *DescribeConfigsResponse) encode(pe packetEncoder) (err error) { - pe.putInt32(int32(r.ThrottleTime / time.Millisecond)) - if err = pe.putArrayLength(len(r.Resources)); err != nil { - return err - } - - for _, c := range r.Resources { - if err = c.encode(pe, r.Version); err != nil { - return err - } - } - - return nil -} - -func (r *DescribeConfigsResponse) decode(pd packetDecoder, version int16) (err error) { - r.Version = version - throttleTime, err := pd.getInt32() - if err != nil { - return err - } - r.ThrottleTime = time.Duration(throttleTime) * time.Millisecond - - n, err := pd.getArrayLength() - if err != nil { - return err - } - - r.Resources = make([]*ResourceResponse, n) - for i := 0; i < n; i++ { - rr := &ResourceResponse{} - if err := rr.decode(pd, version); err != nil { - return err - } - r.Resources[i] = rr - } - - return nil -} - -func (r *DescribeConfigsResponse) key() int16 { - return 32 -} - -func (r *DescribeConfigsResponse) version() int16 { - return r.Version -} - -func (r *DescribeConfigsResponse) headerVersion() int16 { - return 0 -} - -func (r *DescribeConfigsResponse) requiredVersion() KafkaVersion { - switch r.Version { - case 1: - return V1_0_0_0 - case 2: - return V2_0_0_0 - default: - return V0_11_0_0 - } -} - -func (r *ResourceResponse) encode(pe packetEncoder, version int16) (err error) { - pe.putInt16(r.ErrorCode) - - if err = pe.putString(r.ErrorMsg); err != nil { - return err - } - - pe.putInt8(int8(r.Type)) - - if err = pe.putString(r.Name); err != nil { - return err - } - - if err = pe.putArrayLength(len(r.Configs)); err != nil { - return err - } - - for _, c := range r.Configs { - if err = c.encode(pe, version); err != nil { - return err - } - } - return nil -} - -func (r *ResourceResponse) decode(pd packetDecoder, version int16) (err error) { - ec, err := pd.getInt16() - if err != nil { - return err - } - r.ErrorCode = ec - - em, err := pd.getString() - if err != nil { - return err - } - r.ErrorMsg = em - - t, err := pd.getInt8() - if err != nil { - return err - } - r.Type = ConfigResourceType(t) - - name, err := pd.getString() - if err != nil { - return err - } - r.Name = name - - n, err := pd.getArrayLength() - if err != nil { - return err - } - - r.Configs = make([]*ConfigEntry, n) - for i := 0; i < n; i++ { - c := &ConfigEntry{} - if err := c.decode(pd, version); err != nil { - return err - } - r.Configs[i] = c - } - return nil -} - -func (r *ConfigEntry) encode(pe packetEncoder, version int16) (err error) { - if err = pe.putString(r.Name); err != nil { - return err - } - - if err = pe.putString(r.Value); err != nil { - return err - } - - pe.putBool(r.ReadOnly) - - if version <= 0 { - pe.putBool(r.Default) - pe.putBool(r.Sensitive) - } else { - pe.putInt8(int8(r.Source)) - pe.putBool(r.Sensitive) - - if err := pe.putArrayLength(len(r.Synonyms)); err != nil { - return err - } - for _, c := range r.Synonyms { - if err = c.encode(pe, version); err != nil { - return err - } - } - } - - return nil -} - -//https://cwiki.apache.org/confluence/display/KAFKA/KIP-226+-+Dynamic+Broker+Configuration -func (r *ConfigEntry) decode(pd packetDecoder, version int16) (err error) { - if version == 0 { - r.Source = SourceUnknown - } - name, err := pd.getString() - if err != nil { - return err - } - r.Name = name - - value, err := pd.getString() - if err != nil { - return err - } - r.Value = value - - read, err := pd.getBool() - if err != nil { - return err - } - r.ReadOnly = read - - if version == 0 { - defaultB, err := pd.getBool() - if err != nil { - return err - } - r.Default = defaultB - if defaultB { - r.Source = SourceDefault - } - } else { - source, err := pd.getInt8() - if err != nil { - return err - } - r.Source = ConfigSource(source) - r.Default = r.Source == SourceDefault - } - - sensitive, err := pd.getBool() - if err != nil { - return err - } - r.Sensitive = sensitive - - if version > 0 { - n, err := pd.getArrayLength() - if err != nil { - return err - } - r.Synonyms = make([]*ConfigSynonym, n) - - for i := 0; i < n; i++ { - s := &ConfigSynonym{} - if err := s.decode(pd, version); err != nil { - return err - } - r.Synonyms[i] = s - } - } - return nil -} - -func (c *ConfigSynonym) encode(pe packetEncoder, version int16) (err error) { - err = pe.putString(c.ConfigName) - if err != nil { - return err - } - - err = pe.putString(c.ConfigValue) - if err != nil { - return err - } - - pe.putInt8(int8(c.Source)) - - return nil -} - -func (c *ConfigSynonym) decode(pd packetDecoder, version int16) error { - name, err := pd.getString() - if err != nil { - return nil - } - c.ConfigName = name - - value, err := pd.getString() - if err != nil { - return nil - } - c.ConfigValue = value - - source, err := pd.getInt8() - if err != nil { - return nil - } - c.Source = ConfigSource(source) - return nil -} diff --git a/vendor/github.com/Shopify/sarama/describe_groups_request.go b/vendor/github.com/Shopify/sarama/describe_groups_request.go deleted file mode 100644 index f8962da5..00000000 --- a/vendor/github.com/Shopify/sarama/describe_groups_request.go +++ /dev/null @@ -1,34 +0,0 @@ -package sarama - -type DescribeGroupsRequest struct { - Groups []string -} - -func (r *DescribeGroupsRequest) encode(pe packetEncoder) error { - return pe.putStringArray(r.Groups) -} - -func (r *DescribeGroupsRequest) decode(pd packetDecoder, version int16) (err error) { - r.Groups, err = pd.getStringArray() - return -} - -func (r *DescribeGroupsRequest) key() int16 { - return 15 -} - -func (r *DescribeGroupsRequest) version() int16 { - return 0 -} - -func (r *DescribeGroupsRequest) headerVersion() int16 { - return 1 -} - -func (r *DescribeGroupsRequest) requiredVersion() KafkaVersion { - return V0_9_0_0 -} - -func (r *DescribeGroupsRequest) AddGroup(group string) { - r.Groups = append(r.Groups, group) -} diff --git a/vendor/github.com/Shopify/sarama/describe_groups_response.go b/vendor/github.com/Shopify/sarama/describe_groups_response.go deleted file mode 100644 index bc242e42..00000000 --- a/vendor/github.com/Shopify/sarama/describe_groups_response.go +++ /dev/null @@ -1,191 +0,0 @@ -package sarama - -type DescribeGroupsResponse struct { - Groups []*GroupDescription -} - -func (r *DescribeGroupsResponse) encode(pe packetEncoder) error { - if err := pe.putArrayLength(len(r.Groups)); err != nil { - return err - } - - for _, groupDescription := range r.Groups { - if err := groupDescription.encode(pe); err != nil { - return err - } - } - - return nil -} - -func (r *DescribeGroupsResponse) decode(pd packetDecoder, version int16) (err error) { - n, err := pd.getArrayLength() - if err != nil { - return err - } - - r.Groups = make([]*GroupDescription, n) - for i := 0; i < n; i++ { - r.Groups[i] = new(GroupDescription) - if err := r.Groups[i].decode(pd); err != nil { - return err - } - } - - return nil -} - -func (r *DescribeGroupsResponse) key() int16 { - return 15 -} - -func (r *DescribeGroupsResponse) version() int16 { - return 0 -} - -func (r *DescribeGroupsResponse) headerVersion() int16 { - return 0 -} - -func (r *DescribeGroupsResponse) requiredVersion() KafkaVersion { - return V0_9_0_0 -} - -type GroupDescription struct { - Err KError - GroupId string - State string - ProtocolType string - Protocol string - Members map[string]*GroupMemberDescription -} - -func (gd *GroupDescription) encode(pe packetEncoder) error { - pe.putInt16(int16(gd.Err)) - - if err := pe.putString(gd.GroupId); err != nil { - return err - } - if err := pe.putString(gd.State); err != nil { - return err - } - if err := pe.putString(gd.ProtocolType); err != nil { - return err - } - if err := pe.putString(gd.Protocol); err != nil { - return err - } - - if err := pe.putArrayLength(len(gd.Members)); err != nil { - return err - } - - for memberId, groupMemberDescription := range gd.Members { - if err := pe.putString(memberId); err != nil { - return err - } - if err := groupMemberDescription.encode(pe); err != nil { - return err - } - } - - return nil -} - -func (gd *GroupDescription) decode(pd packetDecoder) (err error) { - kerr, err := pd.getInt16() - if err != nil { - return err - } - - gd.Err = KError(kerr) - - if gd.GroupId, err = pd.getString(); err != nil { - return - } - if gd.State, err = pd.getString(); err != nil { - return - } - if gd.ProtocolType, err = pd.getString(); err != nil { - return - } - if gd.Protocol, err = pd.getString(); err != nil { - return - } - - n, err := pd.getArrayLength() - if err != nil { - return err - } - if n == 0 { - return nil - } - - gd.Members = make(map[string]*GroupMemberDescription) - for i := 0; i < n; i++ { - memberId, err := pd.getString() - if err != nil { - return err - } - - gd.Members[memberId] = new(GroupMemberDescription) - if err := gd.Members[memberId].decode(pd); err != nil { - return err - } - } - - return nil -} - -type GroupMemberDescription struct { - ClientId string - ClientHost string - MemberMetadata []byte - MemberAssignment []byte -} - -func (gmd *GroupMemberDescription) encode(pe packetEncoder) error { - if err := pe.putString(gmd.ClientId); err != nil { - return err - } - if err := pe.putString(gmd.ClientHost); err != nil { - return err - } - if err := pe.putBytes(gmd.MemberMetadata); err != nil { - return err - } - if err := pe.putBytes(gmd.MemberAssignment); err != nil { - return err - } - - return nil -} - -func (gmd *GroupMemberDescription) decode(pd packetDecoder) (err error) { - if gmd.ClientId, err = pd.getString(); err != nil { - return - } - if gmd.ClientHost, err = pd.getString(); err != nil { - return - } - if gmd.MemberMetadata, err = pd.getBytes(); err != nil { - return - } - if gmd.MemberAssignment, err = pd.getBytes(); err != nil { - return - } - - return nil -} - -func (gmd *GroupMemberDescription) GetMemberAssignment() (*ConsumerGroupMemberAssignment, error) { - assignment := new(ConsumerGroupMemberAssignment) - err := decode(gmd.MemberAssignment, assignment) - return assignment, err -} - -func (gmd *GroupMemberDescription) GetMemberMetadata() (*ConsumerGroupMemberMetadata, error) { - metadata := new(ConsumerGroupMemberMetadata) - err := decode(gmd.MemberMetadata, metadata) - return metadata, err -} diff --git a/vendor/github.com/Shopify/sarama/describe_log_dirs_request.go b/vendor/github.com/Shopify/sarama/describe_log_dirs_request.go deleted file mode 100644 index c0bf04e0..00000000 --- a/vendor/github.com/Shopify/sarama/describe_log_dirs_request.go +++ /dev/null @@ -1,87 +0,0 @@ -package sarama - -// DescribeLogDirsRequest is a describe request to get partitions' log size -type DescribeLogDirsRequest struct { - // Version 0 and 1 are equal - // The version number is bumped to indicate that on quota violation brokers send out responses before throttling. - Version int16 - - // If this is an empty array, all topics will be queried - DescribeTopics []DescribeLogDirsRequestTopic -} - -// DescribeLogDirsRequestTopic is a describe request about the log dir of one or more partitions within a Topic -type DescribeLogDirsRequestTopic struct { - Topic string - PartitionIDs []int32 -} - -func (r *DescribeLogDirsRequest) encode(pe packetEncoder) error { - length := len(r.DescribeTopics) - if length == 0 { - // In order to query all topics we must send null - length = -1 - } - - if err := pe.putArrayLength(length); err != nil { - return err - } - - for _, d := range r.DescribeTopics { - if err := pe.putString(d.Topic); err != nil { - return err - } - - if err := pe.putInt32Array(d.PartitionIDs); err != nil { - return err - } - } - - return nil -} - -func (r *DescribeLogDirsRequest) decode(pd packetDecoder, version int16) error { - n, err := pd.getArrayLength() - if err != nil { - return err - } - if n == -1 { - n = 0 - } - - topics := make([]DescribeLogDirsRequestTopic, n) - for i := 0; i < n; i++ { - topics[i] = DescribeLogDirsRequestTopic{} - - topic, err := pd.getString() - if err != nil { - return err - } - topics[i].Topic = topic - - pIDs, err := pd.getInt32Array() - if err != nil { - return err - } - topics[i].PartitionIDs = pIDs - } - r.DescribeTopics = topics - - return nil -} - -func (r *DescribeLogDirsRequest) key() int16 { - return 35 -} - -func (r *DescribeLogDirsRequest) version() int16 { - return r.Version -} - -func (r *DescribeLogDirsRequest) headerVersion() int16 { - return 1 -} - -func (r *DescribeLogDirsRequest) requiredVersion() KafkaVersion { - return V1_0_0_0 -} diff --git a/vendor/github.com/Shopify/sarama/describe_log_dirs_response.go b/vendor/github.com/Shopify/sarama/describe_log_dirs_response.go deleted file mode 100644 index 411da38a..00000000 --- a/vendor/github.com/Shopify/sarama/describe_log_dirs_response.go +++ /dev/null @@ -1,229 +0,0 @@ -package sarama - -import "time" - -type DescribeLogDirsResponse struct { - ThrottleTime time.Duration - - // Version 0 and 1 are equal - // The version number is bumped to indicate that on quota violation brokers send out responses before throttling. - Version int16 - - LogDirs []DescribeLogDirsResponseDirMetadata -} - -func (r *DescribeLogDirsResponse) encode(pe packetEncoder) error { - pe.putInt32(int32(r.ThrottleTime / time.Millisecond)) - - if err := pe.putArrayLength(len(r.LogDirs)); err != nil { - return err - } - - for _, dir := range r.LogDirs { - if err := dir.encode(pe); err != nil { - return err - } - } - - return nil -} - -func (r *DescribeLogDirsResponse) decode(pd packetDecoder, version int16) error { - throttleTime, err := pd.getInt32() - if err != nil { - return err - } - r.ThrottleTime = time.Duration(throttleTime) * time.Millisecond - - // Decode array of DescribeLogDirsResponseDirMetadata - n, err := pd.getArrayLength() - if err != nil { - return err - } - - r.LogDirs = make([]DescribeLogDirsResponseDirMetadata, n) - for i := 0; i < n; i++ { - dir := DescribeLogDirsResponseDirMetadata{} - if err := dir.decode(pd, version); err != nil { - return err - } - r.LogDirs[i] = dir - } - - return nil -} - -func (r *DescribeLogDirsResponse) key() int16 { - return 35 -} - -func (r *DescribeLogDirsResponse) version() int16 { - return r.Version -} - -func (r *DescribeLogDirsResponse) headerVersion() int16 { - return 0 -} - -func (r *DescribeLogDirsResponse) requiredVersion() KafkaVersion { - return V1_0_0_0 -} - -type DescribeLogDirsResponseDirMetadata struct { - ErrorCode KError - - // The absolute log directory path - Path string - Topics []DescribeLogDirsResponseTopic -} - -func (r *DescribeLogDirsResponseDirMetadata) encode(pe packetEncoder) error { - pe.putInt16(int16(r.ErrorCode)) - - if err := pe.putString(r.Path); err != nil { - return err - } - - if err := pe.putArrayLength(len(r.Topics)); err != nil { - return err - } - for _, topic := range r.Topics { - if err := topic.encode(pe); err != nil { - return err - } - } - - return nil -} - -func (r *DescribeLogDirsResponseDirMetadata) decode(pd packetDecoder, version int16) error { - errCode, err := pd.getInt16() - if err != nil { - return err - } - r.ErrorCode = KError(errCode) - - path, err := pd.getString() - if err != nil { - return err - } - r.Path = path - - // Decode array of DescribeLogDirsResponseTopic - n, err := pd.getArrayLength() - if err != nil { - return err - } - - r.Topics = make([]DescribeLogDirsResponseTopic, n) - for i := 0; i < n; i++ { - t := DescribeLogDirsResponseTopic{} - - if err := t.decode(pd, version); err != nil { - return err - } - - r.Topics[i] = t - } - - return nil -} - -// DescribeLogDirsResponseTopic contains a topic's partitions descriptions -type DescribeLogDirsResponseTopic struct { - Topic string - Partitions []DescribeLogDirsResponsePartition -} - -func (r *DescribeLogDirsResponseTopic) encode(pe packetEncoder) error { - if err := pe.putString(r.Topic); err != nil { - return err - } - - if err := pe.putArrayLength(len(r.Partitions)); err != nil { - return err - } - for _, partition := range r.Partitions { - if err := partition.encode(pe); err != nil { - return err - } - } - - return nil -} - -func (r *DescribeLogDirsResponseTopic) decode(pd packetDecoder, version int16) error { - t, err := pd.getString() - if err != nil { - return err - } - r.Topic = t - - n, err := pd.getArrayLength() - if err != nil { - return err - } - r.Partitions = make([]DescribeLogDirsResponsePartition, n) - for i := 0; i < n; i++ { - p := DescribeLogDirsResponsePartition{} - if err := p.decode(pd, version); err != nil { - return err - } - r.Partitions[i] = p - } - - return nil -} - -// DescribeLogDirsResponsePartition describes a partition's log directory -type DescribeLogDirsResponsePartition struct { - PartitionID int32 - - // The size of the log segments of the partition in bytes. - Size int64 - - // The lag of the log's LEO w.r.t. partition's HW (if it is the current log for the partition) or - // current replica's LEO (if it is the future log for the partition) - OffsetLag int64 - - // True if this log is created by AlterReplicaLogDirsRequest and will replace the current log of - // the replica in the future. - IsTemporary bool -} - -func (r *DescribeLogDirsResponsePartition) encode(pe packetEncoder) error { - pe.putInt32(r.PartitionID) - pe.putInt64(r.Size) - pe.putInt64(r.OffsetLag) - pe.putBool(r.IsTemporary) - - return nil -} - -func (r *DescribeLogDirsResponsePartition) decode(pd packetDecoder, version int16) error { - pID, err := pd.getInt32() - if err != nil { - return err - } - r.PartitionID = pID - - size, err := pd.getInt64() - if err != nil { - return err - } - r.Size = size - - lag, err := pd.getInt64() - if err != nil { - return err - } - r.OffsetLag = lag - - isTemp, err := pd.getBool() - if err != nil { - return err - } - r.IsTemporary = isTemp - - return nil -} diff --git a/vendor/github.com/Shopify/sarama/dev.yml b/vendor/github.com/Shopify/sarama/dev.yml deleted file mode 100644 index 6468f154..00000000 --- a/vendor/github.com/Shopify/sarama/dev.yml +++ /dev/null @@ -1,10 +0,0 @@ -name: sarama - -up: - - go: - version: '1.14.3' - -commands: - test: - run: make test - desc: 'run unit tests' diff --git a/vendor/github.com/Shopify/sarama/docker-compose.yml b/vendor/github.com/Shopify/sarama/docker-compose.yml deleted file mode 100644 index 25593fd3..00000000 --- a/vendor/github.com/Shopify/sarama/docker-compose.yml +++ /dev/null @@ -1,134 +0,0 @@ -version: '3.7' -services: - zookeeper-1: - image: 'confluentinc/cp-zookeeper:${CONFLUENT_PLATFORM_VERSION:-5.5.0}' - restart: always - environment: - ZOOKEEPER_SERVER_ID: '1' - ZOOKEEPER_SERVERS: 'zookeeper-1:2888:3888;zookeeper-2:2888:3888;zookeeper-3:2888:3888' - ZOOKEEPER_CLIENT_PORT: '2181' - ZOOKEEPER_PEER_PORT: '2888' - ZOOKEEPER_LEADER_PORT: '3888' - ZOOKEEPER_INIT_LIMIT: '10' - ZOOKEEPER_SYNC_LIMIT: '5' - ZOOKEEPER_MAX_CLIENT_CONNS: '0' - zookeeper-2: - image: 'confluentinc/cp-zookeeper:${CONFLUENT_PLATFORM_VERSION:-5.5.0}' - restart: always - environment: - ZOOKEEPER_SERVER_ID: '2' - ZOOKEEPER_SERVERS: 'zookeeper-1:2888:3888;zookeeper-2:2888:3888;zookeeper-3:2888:3888' - ZOOKEEPER_CLIENT_PORT: '2181' - ZOOKEEPER_PEER_PORT: '2888' - ZOOKEEPER_LEADER_PORT: '3888' - ZOOKEEPER_INIT_LIMIT: '10' - ZOOKEEPER_SYNC_LIMIT: '5' - ZOOKEEPER_MAX_CLIENT_CONNS: '0' - zookeeper-3: - image: 'confluentinc/cp-zookeeper:${CONFLUENT_PLATFORM_VERSION:-5.5.0}' - restart: always - environment: - ZOOKEEPER_SERVER_ID: '3' - ZOOKEEPER_SERVERS: 'zookeeper-1:2888:3888;zookeeper-2:2888:3888;zookeeper-3:2888:3888' - ZOOKEEPER_CLIENT_PORT: '2181' - ZOOKEEPER_PEER_PORT: '2888' - ZOOKEEPER_LEADER_PORT: '3888' - ZOOKEEPER_INIT_LIMIT: '10' - ZOOKEEPER_SYNC_LIMIT: '5' - ZOOKEEPER_MAX_CLIENT_CONNS: '0' - kafka-1: - image: 'confluentinc/cp-kafka:${CONFLUENT_PLATFORM_VERSION:-5.5.0}' - restart: always - environment: - KAFKA_ZOOKEEPER_CONNECT: 'zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181' - KAFKA_LISTENERS: 'LISTENER_INTERNAL://:9091,LISTENER_LOCAL://:29091' - KAFKA_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-1:9091,LISTENER_LOCAL://localhost:29091' - KAFKA_INTER_BROKER_LISTENER_NAME: 'LISTENER_INTERNAL' - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'LISTENER_INTERNAL:PLAINTEXT,LISTENER_LOCAL:PLAINTEXT' - KAFKA_DEFAULT_REPLICATION_FACTOR: '2' - KAFKA_BROKER_ID: '1' - KAFKA_BROKER_RACK: '1' - KAFKA_ZOOKEEPER_SESSION_TIMEOUT_MS: '3000' - KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: '3000' - KAFKA_REPLICA_SELECTOR_CLASS: 'org.apache.kafka.common.replica.RackAwareReplicaSelector' - KAFKA_DELETE_TOPIC_ENABLE: 'true' - KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'false' - kafka-2: - image: 'confluentinc/cp-kafka:${CONFLUENT_PLATFORM_VERSION:-5.5.0}' - restart: always - environment: - KAFKA_ZOOKEEPER_CONNECT: 'zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181' - KAFKA_LISTENERS: 'LISTENER_INTERNAL://:9091,LISTENER_LOCAL://:29092' - KAFKA_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-2:9091,LISTENER_LOCAL://localhost:29092' - KAFKA_INTER_BROKER_LISTENER_NAME: 'LISTENER_INTERNAL' - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'LISTENER_INTERNAL:PLAINTEXT,LISTENER_LOCAL:PLAINTEXT' - KAFKA_DEFAULT_REPLICATION_FACTOR: '2' - KAFKA_BROKER_ID: '2' - KAFKA_BROKER_RACK: '2' - KAFKA_ZOOKEEPER_SESSION_TIMEOUT_MS: '3000' - KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: '3000' - KAFKA_REPLICA_SELECTOR_CLASS: 'org.apache.kafka.common.replica.RackAwareReplicaSelector' - KAFKA_DELETE_TOPIC_ENABLE: 'true' - KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'false' - kafka-3: - image: 'confluentinc/cp-kafka:${CONFLUENT_PLATFORM_VERSION:-5.5.0}' - restart: always - environment: - KAFKA_ZOOKEEPER_CONNECT: 'zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181' - KAFKA_LISTENERS: 'LISTENER_INTERNAL://:9091,LISTENER_LOCAL://:29093' - KAFKA_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-3:9091,LISTENER_LOCAL://localhost:29093' - KAFKA_INTER_BROKER_LISTENER_NAME: 'LISTENER_INTERNAL' - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'LISTENER_INTERNAL:PLAINTEXT,LISTENER_LOCAL:PLAINTEXT' - KAFKA_DEFAULT_REPLICATION_FACTOR: '2' - KAFKA_BROKER_ID: '3' - KAFKA_BROKER_RACK: '3' - KAFKA_ZOOKEEPER_SESSION_TIMEOUT_MS: '3000' - KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: '3000' - KAFKA_REPLICA_SELECTOR_CLASS: 'org.apache.kafka.common.replica.RackAwareReplicaSelector' - KAFKA_DELETE_TOPIC_ENABLE: 'true' - KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'false' - kafka-4: - image: 'confluentinc/cp-kafka:${CONFLUENT_PLATFORM_VERSION:-5.5.0}' - restart: always - environment: - KAFKA_ZOOKEEPER_CONNECT: 'zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181' - KAFKA_LISTENERS: 'LISTENER_INTERNAL://:9091,LISTENER_LOCAL://:29094' - KAFKA_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-4:9091,LISTENER_LOCAL://localhost:29094' - KAFKA_INTER_BROKER_LISTENER_NAME: 'LISTENER_INTERNAL' - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'LISTENER_INTERNAL:PLAINTEXT,LISTENER_LOCAL:PLAINTEXT' - KAFKA_DEFAULT_REPLICATION_FACTOR: '2' - KAFKA_BROKER_ID: '4' - KAFKA_BROKER_RACK: '4' - KAFKA_ZOOKEEPER_SESSION_TIMEOUT_MS: '3000' - KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: '3000' - KAFKA_REPLICA_SELECTOR_CLASS: 'org.apache.kafka.common.replica.RackAwareReplicaSelector' - KAFKA_DELETE_TOPIC_ENABLE: 'true' - KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'false' - kafka-5: - image: 'confluentinc/cp-kafka:${CONFLUENT_PLATFORM_VERSION:-5.5.0}' - restart: always - environment: - KAFKA_ZOOKEEPER_CONNECT: 'zookeeper-1:2181,zookeeper-2:2181,zookeeper-3:2181' - KAFKA_LISTENERS: 'LISTENER_INTERNAL://:9091,LISTENER_LOCAL://:29095' - KAFKA_ADVERTISED_LISTENERS: 'LISTENER_INTERNAL://kafka-5:9091,LISTENER_LOCAL://localhost:29095' - KAFKA_INTER_BROKER_LISTENER_NAME: 'LISTENER_INTERNAL' - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'LISTENER_INTERNAL:PLAINTEXT,LISTENER_LOCAL:PLAINTEXT' - KAFKA_DEFAULT_REPLICATION_FACTOR: '2' - KAFKA_BROKER_ID: '5' - KAFKA_BROKER_RACK: '5' - KAFKA_ZOOKEEPER_SESSION_TIMEOUT_MS: '3000' - KAFKA_ZOOKEEPER_CONNECTION_TIMEOUT_MS: '3000' - KAFKA_REPLICA_SELECTOR_CLASS: 'org.apache.kafka.common.replica.RackAwareReplicaSelector' - KAFKA_DELETE_TOPIC_ENABLE: 'true' - KAFKA_AUTO_CREATE_TOPICS_ENABLE: 'false' - toxiproxy: - image: 'shopify/toxiproxy:2.1.4' - ports: - # The tests themselves actually start the proies on these ports - - '29091:29091' - - '29092:29092' - - '29093:29093' - - '29094:29094' - - '29095:29095' - # This is the toxiproxy API port - - '8474:8474' diff --git a/vendor/github.com/Shopify/sarama/encoder_decoder.go b/vendor/github.com/Shopify/sarama/encoder_decoder.go deleted file mode 100644 index 025bad61..00000000 --- a/vendor/github.com/Shopify/sarama/encoder_decoder.go +++ /dev/null @@ -1,94 +0,0 @@ -package sarama - -import ( - "fmt" - - "github.com/rcrowley/go-metrics" -) - -// Encoder is the interface that wraps the basic Encode method. -// Anything implementing Encoder can be turned into bytes using Kafka's encoding rules. -type encoder interface { - encode(pe packetEncoder) error -} - -type encoderWithHeader interface { - encoder - headerVersion() int16 -} - -// Encode takes an Encoder and turns it into bytes while potentially recording metrics. -func encode(e encoder, metricRegistry metrics.Registry) ([]byte, error) { - if e == nil { - return nil, nil - } - - var prepEnc prepEncoder - var realEnc realEncoder - - err := e.encode(&prepEnc) - if err != nil { - return nil, err - } - - if prepEnc.length < 0 || prepEnc.length > int(MaxRequestSize) { - return nil, PacketEncodingError{fmt.Sprintf("invalid request size (%d)", prepEnc.length)} - } - - realEnc.raw = make([]byte, prepEnc.length) - realEnc.registry = metricRegistry - err = e.encode(&realEnc) - if err != nil { - return nil, err - } - - return realEnc.raw, nil -} - -// Decoder is the interface that wraps the basic Decode method. -// Anything implementing Decoder can be extracted from bytes using Kafka's encoding rules. -type decoder interface { - decode(pd packetDecoder) error -} - -type versionedDecoder interface { - decode(pd packetDecoder, version int16) error -} - -// Decode takes bytes and a Decoder and fills the fields of the decoder from the bytes, -// interpreted using Kafka's encoding rules. -func decode(buf []byte, in decoder) error { - if buf == nil { - return nil - } - - helper := realDecoder{raw: buf} - err := in.decode(&helper) - if err != nil { - return err - } - - if helper.off != len(buf) { - return PacketDecodingError{"invalid length"} - } - - return nil -} - -func versionedDecode(buf []byte, in versionedDecoder, version int16) error { - if buf == nil { - return nil - } - - helper := realDecoder{raw: buf} - err := in.decode(&helper, version) - if err != nil { - return err - } - - if helper.off != len(buf) { - return PacketDecodingError{"invalid length"} - } - - return nil -} diff --git a/vendor/github.com/Shopify/sarama/end_txn_request.go b/vendor/github.com/Shopify/sarama/end_txn_request.go deleted file mode 100644 index 6635425d..00000000 --- a/vendor/github.com/Shopify/sarama/end_txn_request.go +++ /dev/null @@ -1,54 +0,0 @@ -package sarama - -type EndTxnRequest struct { - TransactionalID string - ProducerID int64 - ProducerEpoch int16 - TransactionResult bool -} - -func (a *EndTxnRequest) encode(pe packetEncoder) error { - if err := pe.putString(a.TransactionalID); err != nil { - return err - } - - pe.putInt64(a.ProducerID) - - pe.putInt16(a.ProducerEpoch) - - pe.putBool(a.TransactionResult) - - return nil -} - -func (a *EndTxnRequest) decode(pd packetDecoder, version int16) (err error) { - if a.TransactionalID, err = pd.getString(); err != nil { - return err - } - if a.ProducerID, err = pd.getInt64(); err != nil { - return err - } - if a.ProducerEpoch, err = pd.getInt16(); err != nil { - return err - } - if a.TransactionResult, err = pd.getBool(); err != nil { - return err - } - return nil -} - -func (a *EndTxnRequest) key() int16 { - return 26 -} - -func (a *EndTxnRequest) version() int16 { - return 0 -} - -func (r *EndTxnRequest) headerVersion() int16 { - return 1 -} - -func (a *EndTxnRequest) requiredVersion() KafkaVersion { - return V0_11_0_0 -} diff --git a/vendor/github.com/Shopify/sarama/end_txn_response.go b/vendor/github.com/Shopify/sarama/end_txn_response.go deleted file mode 100644 index 76397672..00000000 --- a/vendor/github.com/Shopify/sarama/end_txn_response.go +++ /dev/null @@ -1,48 +0,0 @@ -package sarama - -import ( - "time" -) - -type EndTxnResponse struct { - ThrottleTime time.Duration - Err KError -} - -func (e *EndTxnResponse) encode(pe packetEncoder) error { - pe.putInt32(int32(e.ThrottleTime / time.Millisecond)) - pe.putInt16(int16(e.Err)) - return nil -} - -func (e *EndTxnResponse) decode(pd packetDecoder, version int16) (err error) { - throttleTime, err := pd.getInt32() - if err != nil { - return err - } - e.ThrottleTime = time.Duration(throttleTime) * time.Millisecond - - kerr, err := pd.getInt16() - if err != nil { - return err - } - e.Err = KError(kerr) - - return nil -} - -func (e *EndTxnResponse) key() int16 { - return 25 -} - -func (e *EndTxnResponse) version() int16 { - return 0 -} - -func (r *EndTxnResponse) headerVersion() int16 { - return 0 -} - -func (e *EndTxnResponse) requiredVersion() KafkaVersion { - return V0_11_0_0 -} diff --git a/vendor/github.com/Shopify/sarama/errors.go b/vendor/github.com/Shopify/sarama/errors.go deleted file mode 100644 index ca621b09..00000000 --- a/vendor/github.com/Shopify/sarama/errors.go +++ /dev/null @@ -1,385 +0,0 @@ -package sarama - -import ( - "errors" - "fmt" -) - -// ErrOutOfBrokers is the error returned when the client has run out of brokers to talk to because all of them errored -// or otherwise failed to respond. -var ErrOutOfBrokers = errors.New("kafka: client has run out of available brokers to talk to (Is your cluster reachable?)") - -// ErrClosedClient is the error returned when a method is called on a client that has been closed. -var ErrClosedClient = errors.New("kafka: tried to use a client that was closed") - -// ErrIncompleteResponse is the error returned when the server returns a syntactically valid response, but it does -// not contain the expected information. -var ErrIncompleteResponse = errors.New("kafka: response did not contain all the expected topic/partition blocks") - -// ErrInvalidPartition is the error returned when a partitioner returns an invalid partition index -// (meaning one outside of the range [0...numPartitions-1]). -var ErrInvalidPartition = errors.New("kafka: partitioner returned an invalid partition index") - -// ErrAlreadyConnected is the error returned when calling Open() on a Broker that is already connected or connecting. -var ErrAlreadyConnected = errors.New("kafka: broker connection already initiated") - -// ErrNotConnected is the error returned when trying to send or call Close() on a Broker that is not connected. -var ErrNotConnected = errors.New("kafka: broker not connected") - -// ErrInsufficientData is returned when decoding and the packet is truncated. This can be expected -// when requesting messages, since as an optimization the server is allowed to return a partial message at the end -// of the message set. -var ErrInsufficientData = errors.New("kafka: insufficient data to decode packet, more bytes expected") - -// ErrShuttingDown is returned when a producer receives a message during shutdown. -var ErrShuttingDown = errors.New("kafka: message received by producer in process of shutting down") - -// ErrMessageTooLarge is returned when the next message to consume is larger than the configured Consumer.Fetch.Max -var ErrMessageTooLarge = errors.New("kafka: message is larger than Consumer.Fetch.Max") - -// ErrConsumerOffsetNotAdvanced is returned when a partition consumer didn't advance its offset after parsing -// a RecordBatch. -var ErrConsumerOffsetNotAdvanced = errors.New("kafka: consumer offset was not advanced after a RecordBatch") - -// ErrControllerNotAvailable is returned when server didn't give correct controller id. May be kafka server's version -// is lower than 0.10.0.0. -var ErrControllerNotAvailable = errors.New("kafka: controller is not available") - -// ErrNoTopicsToUpdateMetadata is returned when Meta.Full is set to false but no specific topics were found to update -// the metadata. -var ErrNoTopicsToUpdateMetadata = errors.New("kafka: no specific topics to update metadata") - -// PacketEncodingError is returned from a failure while encoding a Kafka packet. This can happen, for example, -// if you try to encode a string over 2^15 characters in length, since Kafka's encoding rules do not permit that. -type PacketEncodingError struct { - Info string -} - -func (err PacketEncodingError) Error() string { - return fmt.Sprintf("kafka: error encoding packet: %s", err.Info) -} - -// PacketDecodingError is returned when there was an error (other than truncated data) decoding the Kafka broker's response. -// This can be a bad CRC or length field, or any other invalid value. -type PacketDecodingError struct { - Info string -} - -func (err PacketDecodingError) Error() string { - return fmt.Sprintf("kafka: error decoding packet: %s", err.Info) -} - -// ConfigurationError is the type of error returned from a constructor (e.g. NewClient, or NewConsumer) -// when the specified configuration is invalid. -type ConfigurationError string - -func (err ConfigurationError) Error() string { - return "kafka: invalid configuration (" + string(err) + ")" -} - -// KError is the type of error that can be returned directly by the Kafka broker. -// See https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-ErrorCodes -type KError int16 - -// MultiError is used to contain multi error. -type MultiError struct { - Errors *[]error -} - -func (mErr MultiError) Error() string { - var errString = "" - for _, err := range *mErr.Errors { - errString += err.Error() + "," - } - return errString -} - -func (mErr MultiError) PrettyError() string { - var errString = "" - for _, err := range *mErr.Errors { - errString += err.Error() + "\n" - } - return errString -} - -// ErrDeleteRecords is the type of error returned when fail to delete the required records -type ErrDeleteRecords struct { - MultiError -} - -func (err ErrDeleteRecords) Error() string { - return "kafka server: failed to delete records " + err.MultiError.Error() -} - -type ErrReassignPartitions struct { - MultiError -} - -func (err ErrReassignPartitions) Error() string { - return fmt.Sprintf("failed to reassign partitions for topic: \n%s", err.MultiError.PrettyError()) -} - -// Numeric error codes returned by the Kafka server. -const ( - ErrNoError KError = 0 - ErrUnknown KError = -1 - ErrOffsetOutOfRange KError = 1 - ErrInvalidMessage KError = 2 - ErrUnknownTopicOrPartition KError = 3 - ErrInvalidMessageSize KError = 4 - ErrLeaderNotAvailable KError = 5 - ErrNotLeaderForPartition KError = 6 - ErrRequestTimedOut KError = 7 - ErrBrokerNotAvailable KError = 8 - ErrReplicaNotAvailable KError = 9 - ErrMessageSizeTooLarge KError = 10 - ErrStaleControllerEpochCode KError = 11 - ErrOffsetMetadataTooLarge KError = 12 - ErrNetworkException KError = 13 - ErrOffsetsLoadInProgress KError = 14 - ErrConsumerCoordinatorNotAvailable KError = 15 - ErrNotCoordinatorForConsumer KError = 16 - ErrInvalidTopic KError = 17 - ErrMessageSetSizeTooLarge KError = 18 - ErrNotEnoughReplicas KError = 19 - ErrNotEnoughReplicasAfterAppend KError = 20 - ErrInvalidRequiredAcks KError = 21 - ErrIllegalGeneration KError = 22 - ErrInconsistentGroupProtocol KError = 23 - ErrInvalidGroupId KError = 24 - ErrUnknownMemberId KError = 25 - ErrInvalidSessionTimeout KError = 26 - ErrRebalanceInProgress KError = 27 - ErrInvalidCommitOffsetSize KError = 28 - ErrTopicAuthorizationFailed KError = 29 - ErrGroupAuthorizationFailed KError = 30 - ErrClusterAuthorizationFailed KError = 31 - ErrInvalidTimestamp KError = 32 - ErrUnsupportedSASLMechanism KError = 33 - ErrIllegalSASLState KError = 34 - ErrUnsupportedVersion KError = 35 - ErrTopicAlreadyExists KError = 36 - ErrInvalidPartitions KError = 37 - ErrInvalidReplicationFactor KError = 38 - ErrInvalidReplicaAssignment KError = 39 - ErrInvalidConfig KError = 40 - ErrNotController KError = 41 - ErrInvalidRequest KError = 42 - ErrUnsupportedForMessageFormat KError = 43 - ErrPolicyViolation KError = 44 - ErrOutOfOrderSequenceNumber KError = 45 - ErrDuplicateSequenceNumber KError = 46 - ErrInvalidProducerEpoch KError = 47 - ErrInvalidTxnState KError = 48 - ErrInvalidProducerIDMapping KError = 49 - ErrInvalidTransactionTimeout KError = 50 - ErrConcurrentTransactions KError = 51 - ErrTransactionCoordinatorFenced KError = 52 - ErrTransactionalIDAuthorizationFailed KError = 53 - ErrSecurityDisabled KError = 54 - ErrOperationNotAttempted KError = 55 - ErrKafkaStorageError KError = 56 - ErrLogDirNotFound KError = 57 - ErrSASLAuthenticationFailed KError = 58 - ErrUnknownProducerID KError = 59 - ErrReassignmentInProgress KError = 60 - ErrDelegationTokenAuthDisabled KError = 61 - ErrDelegationTokenNotFound KError = 62 - ErrDelegationTokenOwnerMismatch KError = 63 - ErrDelegationTokenRequestNotAllowed KError = 64 - ErrDelegationTokenAuthorizationFailed KError = 65 - ErrDelegationTokenExpired KError = 66 - ErrInvalidPrincipalType KError = 67 - ErrNonEmptyGroup KError = 68 - ErrGroupIDNotFound KError = 69 - ErrFetchSessionIDNotFound KError = 70 - ErrInvalidFetchSessionEpoch KError = 71 - ErrListenerNotFound KError = 72 - ErrTopicDeletionDisabled KError = 73 - ErrFencedLeaderEpoch KError = 74 - ErrUnknownLeaderEpoch KError = 75 - ErrUnsupportedCompressionType KError = 76 - ErrStaleBrokerEpoch KError = 77 - ErrOffsetNotAvailable KError = 78 - ErrMemberIdRequired KError = 79 - ErrPreferredLeaderNotAvailable KError = 80 - ErrGroupMaxSizeReached KError = 81 - ErrFencedInstancedId KError = 82 -) - -func (err KError) Error() string { - // Error messages stolen/adapted from - // https://kafka.apache.org/protocol#protocol_error_codes - switch err { - case ErrNoError: - return "kafka server: Not an error, why are you printing me?" - case ErrUnknown: - return "kafka server: Unexpected (unknown?) server error." - case ErrOffsetOutOfRange: - return "kafka server: The requested offset is outside the range of offsets maintained by the server for the given topic/partition." - case ErrInvalidMessage: - return "kafka server: Message contents does not match its CRC." - case ErrUnknownTopicOrPartition: - return "kafka server: Request was for a topic or partition that does not exist on this broker." - case ErrInvalidMessageSize: - return "kafka server: The message has a negative size." - case ErrLeaderNotAvailable: - return "kafka server: In the middle of a leadership election, there is currently no leader for this partition and hence it is unavailable for writes." - case ErrNotLeaderForPartition: - return "kafka server: Tried to send a message to a replica that is not the leader for some partition. Your metadata is out of date." - case ErrRequestTimedOut: - return "kafka server: Request exceeded the user-specified time limit in the request." - case ErrBrokerNotAvailable: - return "kafka server: Broker not available. Not a client facing error, we should never receive this!!!" - case ErrReplicaNotAvailable: - return "kafka server: Replica information not available, one or more brokers are down." - case ErrMessageSizeTooLarge: - return "kafka server: Message was too large, server rejected it to avoid allocation error." - case ErrStaleControllerEpochCode: - return "kafka server: StaleControllerEpochCode (internal error code for broker-to-broker communication)." - case ErrOffsetMetadataTooLarge: - return "kafka server: Specified a string larger than the configured maximum for offset metadata." - case ErrNetworkException: - return "kafka server: The server disconnected before a response was received." - case ErrOffsetsLoadInProgress: - return "kafka server: The broker is still loading offsets after a leader change for that offset's topic partition." - case ErrConsumerCoordinatorNotAvailable: - return "kafka server: Offset's topic has not yet been created." - case ErrNotCoordinatorForConsumer: - return "kafka server: Request was for a consumer group that is not coordinated by this broker." - case ErrInvalidTopic: - return "kafka server: The request attempted to perform an operation on an invalid topic." - case ErrMessageSetSizeTooLarge: - return "kafka server: The request included message batch larger than the configured segment size on the server." - case ErrNotEnoughReplicas: - return "kafka server: Messages are rejected since there are fewer in-sync replicas than required." - case ErrNotEnoughReplicasAfterAppend: - return "kafka server: Messages are written to the log, but to fewer in-sync replicas than required." - case ErrInvalidRequiredAcks: - return "kafka server: The number of required acks is invalid (should be either -1, 0, or 1)." - case ErrIllegalGeneration: - return "kafka server: The provided generation id is not the current generation." - case ErrInconsistentGroupProtocol: - return "kafka server: The provider group protocol type is incompatible with the other members." - case ErrInvalidGroupId: - return "kafka server: The provided group id was empty." - case ErrUnknownMemberId: - return "kafka server: The provided member is not known in the current generation." - case ErrInvalidSessionTimeout: - return "kafka server: The provided session timeout is outside the allowed range." - case ErrRebalanceInProgress: - return "kafka server: A rebalance for the group is in progress. Please re-join the group." - case ErrInvalidCommitOffsetSize: - return "kafka server: The provided commit metadata was too large." - case ErrTopicAuthorizationFailed: - return "kafka server: The client is not authorized to access this topic." - case ErrGroupAuthorizationFailed: - return "kafka server: The client is not authorized to access this group." - case ErrClusterAuthorizationFailed: - return "kafka server: The client is not authorized to send this request type." - case ErrInvalidTimestamp: - return "kafka server: The timestamp of the message is out of acceptable range." - case ErrUnsupportedSASLMechanism: - return "kafka server: The broker does not support the requested SASL mechanism." - case ErrIllegalSASLState: - return "kafka server: Request is not valid given the current SASL state." - case ErrUnsupportedVersion: - return "kafka server: The version of API is not supported." - case ErrTopicAlreadyExists: - return "kafka server: Topic with this name already exists." - case ErrInvalidPartitions: - return "kafka server: Number of partitions is invalid." - case ErrInvalidReplicationFactor: - return "kafka server: Replication-factor is invalid." - case ErrInvalidReplicaAssignment: - return "kafka server: Replica assignment is invalid." - case ErrInvalidConfig: - return "kafka server: Configuration is invalid." - case ErrNotController: - return "kafka server: This is not the correct controller for this cluster." - case ErrInvalidRequest: - return "kafka server: This most likely occurs because of a request being malformed by the client library or the message was sent to an incompatible broker. See the broker logs for more details." - case ErrUnsupportedForMessageFormat: - return "kafka server: The requested operation is not supported by the message format version." - case ErrPolicyViolation: - return "kafka server: Request parameters do not satisfy the configured policy." - case ErrOutOfOrderSequenceNumber: - return "kafka server: The broker received an out of order sequence number." - case ErrDuplicateSequenceNumber: - return "kafka server: The broker received a duplicate sequence number." - case ErrInvalidProducerEpoch: - return "kafka server: Producer attempted an operation with an old epoch." - case ErrInvalidTxnState: - return "kafka server: The producer attempted a transactional operation in an invalid state." - case ErrInvalidProducerIDMapping: - return "kafka server: The producer attempted to use a producer id which is not currently assigned to its transactional id." - case ErrInvalidTransactionTimeout: - return "kafka server: The transaction timeout is larger than the maximum value allowed by the broker (as configured by max.transaction.timeout.ms)." - case ErrConcurrentTransactions: - return "kafka server: The producer attempted to update a transaction while another concurrent operation on the same transaction was ongoing." - case ErrTransactionCoordinatorFenced: - return "kafka server: The transaction coordinator sending a WriteTxnMarker is no longer the current coordinator for a given producer." - case ErrTransactionalIDAuthorizationFailed: - return "kafka server: Transactional ID authorization failed." - case ErrSecurityDisabled: - return "kafka server: Security features are disabled." - case ErrOperationNotAttempted: - return "kafka server: The broker did not attempt to execute this operation." - case ErrKafkaStorageError: - return "kafka server: Disk error when trying to access log file on the disk." - case ErrLogDirNotFound: - return "kafka server: The specified log directory is not found in the broker config." - case ErrSASLAuthenticationFailed: - return "kafka server: SASL Authentication failed." - case ErrUnknownProducerID: - return "kafka server: The broker could not locate the producer metadata associated with the Producer ID." - case ErrReassignmentInProgress: - return "kafka server: A partition reassignment is in progress." - case ErrDelegationTokenAuthDisabled: - return "kafka server: Delegation Token feature is not enabled." - case ErrDelegationTokenNotFound: - return "kafka server: Delegation Token is not found on server." - case ErrDelegationTokenOwnerMismatch: - return "kafka server: Specified Principal is not valid Owner/Renewer." - case ErrDelegationTokenRequestNotAllowed: - return "kafka server: Delegation Token requests are not allowed on PLAINTEXT/1-way SSL channels and on delegation token authenticated channels." - case ErrDelegationTokenAuthorizationFailed: - return "kafka server: Delegation Token authorization failed." - case ErrDelegationTokenExpired: - return "kafka server: Delegation Token is expired." - case ErrInvalidPrincipalType: - return "kafka server: Supplied principalType is not supported." - case ErrNonEmptyGroup: - return "kafka server: The group is not empty." - case ErrGroupIDNotFound: - return "kafka server: The group id does not exist." - case ErrFetchSessionIDNotFound: - return "kafka server: The fetch session ID was not found." - case ErrInvalidFetchSessionEpoch: - return "kafka server: The fetch session epoch is invalid." - case ErrListenerNotFound: - return "kafka server: There is no listener on the leader broker that matches the listener on which metadata request was processed." - case ErrTopicDeletionDisabled: - return "kafka server: Topic deletion is disabled." - case ErrFencedLeaderEpoch: - return "kafka server: The leader epoch in the request is older than the epoch on the broker." - case ErrUnknownLeaderEpoch: - return "kafka server: The leader epoch in the request is newer than the epoch on the broker." - case ErrUnsupportedCompressionType: - return "kafka server: The requesting client does not support the compression type of given partition." - case ErrStaleBrokerEpoch: - return "kafka server: Broker epoch has changed" - case ErrOffsetNotAvailable: - return "kafka server: The leader high watermark has not caught up from a recent leader election so the offsets cannot be guaranteed to be monotonically increasing" - case ErrMemberIdRequired: - return "kafka server: The group member needs to have a valid member id before actually entering a consumer group" - case ErrPreferredLeaderNotAvailable: - return "kafka server: The preferred leader was not available" - case ErrGroupMaxSizeReached: - return "kafka server: Consumer group The consumer group has reached its max size. already has the configured maximum number of members." - case ErrFencedInstancedId: - return "kafka server: The broker rejected this static consumer since another consumer with the same group.instance.id has registered with a different member.id." - } - - return fmt.Sprintf("Unknown error, how did this happen? Error code = %d", err) -} diff --git a/vendor/github.com/Shopify/sarama/fetch_request.go b/vendor/github.com/Shopify/sarama/fetch_request.go deleted file mode 100644 index f893aeff..00000000 --- a/vendor/github.com/Shopify/sarama/fetch_request.go +++ /dev/null @@ -1,295 +0,0 @@ -package sarama - -type fetchRequestBlock struct { - Version int16 - currentLeaderEpoch int32 - fetchOffset int64 - logStartOffset int64 - maxBytes int32 -} - -func (b *fetchRequestBlock) encode(pe packetEncoder, version int16) error { - b.Version = version - if b.Version >= 9 { - pe.putInt32(b.currentLeaderEpoch) - } - pe.putInt64(b.fetchOffset) - if b.Version >= 5 { - pe.putInt64(b.logStartOffset) - } - pe.putInt32(b.maxBytes) - return nil -} - -func (b *fetchRequestBlock) decode(pd packetDecoder, version int16) (err error) { - b.Version = version - if b.Version >= 9 { - if b.currentLeaderEpoch, err = pd.getInt32(); err != nil { - return err - } - } - if b.fetchOffset, err = pd.getInt64(); err != nil { - return err - } - if b.Version >= 5 { - if b.logStartOffset, err = pd.getInt64(); err != nil { - return err - } - } - if b.maxBytes, err = pd.getInt32(); err != nil { - return err - } - return nil -} - -// FetchRequest (API key 1) will fetch Kafka messages. Version 3 introduced the MaxBytes field. See -// https://issues.apache.org/jira/browse/KAFKA-2063 for a discussion of the issues leading up to that. The KIP is at -// https://cwiki.apache.org/confluence/display/KAFKA/KIP-74%3A+Add+Fetch+Response+Size+Limit+in+Bytes -type FetchRequest struct { - MaxWaitTime int32 - MinBytes int32 - MaxBytes int32 - Version int16 - Isolation IsolationLevel - SessionID int32 - SessionEpoch int32 - blocks map[string]map[int32]*fetchRequestBlock - forgotten map[string][]int32 - RackID string -} - -type IsolationLevel int8 - -const ( - ReadUncommitted IsolationLevel = iota - ReadCommitted -) - -func (r *FetchRequest) encode(pe packetEncoder) (err error) { - pe.putInt32(-1) // replica ID is always -1 for clients - pe.putInt32(r.MaxWaitTime) - pe.putInt32(r.MinBytes) - if r.Version >= 3 { - pe.putInt32(r.MaxBytes) - } - if r.Version >= 4 { - pe.putInt8(int8(r.Isolation)) - } - if r.Version >= 7 { - pe.putInt32(r.SessionID) - pe.putInt32(r.SessionEpoch) - } - err = pe.putArrayLength(len(r.blocks)) - if err != nil { - return err - } - for topic, blocks := range r.blocks { - err = pe.putString(topic) - if err != nil { - return err - } - err = pe.putArrayLength(len(blocks)) - if err != nil { - return err - } - for partition, block := range blocks { - pe.putInt32(partition) - err = block.encode(pe, r.Version) - if err != nil { - return err - } - } - } - if r.Version >= 7 { - err = pe.putArrayLength(len(r.forgotten)) - if err != nil { - return err - } - for topic, partitions := range r.forgotten { - err = pe.putString(topic) - if err != nil { - return err - } - err = pe.putArrayLength(len(partitions)) - if err != nil { - return err - } - for _, partition := range partitions { - pe.putInt32(partition) - } - } - } - if r.Version >= 11 { - err = pe.putString(r.RackID) - if err != nil { - return err - } - } - - return nil -} - -func (r *FetchRequest) decode(pd packetDecoder, version int16) (err error) { - r.Version = version - - if _, err = pd.getInt32(); err != nil { - return err - } - if r.MaxWaitTime, err = pd.getInt32(); err != nil { - return err - } - if r.MinBytes, err = pd.getInt32(); err != nil { - return err - } - if r.Version >= 3 { - if r.MaxBytes, err = pd.getInt32(); err != nil { - return err - } - } - if r.Version >= 4 { - isolation, err := pd.getInt8() - if err != nil { - return err - } - r.Isolation = IsolationLevel(isolation) - } - if r.Version >= 7 { - r.SessionID, err = pd.getInt32() - if err != nil { - return err - } - r.SessionEpoch, err = pd.getInt32() - if err != nil { - return err - } - } - topicCount, err := pd.getArrayLength() - if err != nil { - return err - } - if topicCount == 0 { - return nil - } - r.blocks = make(map[string]map[int32]*fetchRequestBlock) - for i := 0; i < topicCount; i++ { - topic, err := pd.getString() - if err != nil { - return err - } - partitionCount, err := pd.getArrayLength() - if err != nil { - return err - } - r.blocks[topic] = make(map[int32]*fetchRequestBlock) - for j := 0; j < partitionCount; j++ { - partition, err := pd.getInt32() - if err != nil { - return err - } - fetchBlock := &fetchRequestBlock{} - if err = fetchBlock.decode(pd, r.Version); err != nil { - return err - } - r.blocks[topic][partition] = fetchBlock - } - } - - if r.Version >= 7 { - forgottenCount, err := pd.getArrayLength() - if err != nil { - return err - } - r.forgotten = make(map[string][]int32) - for i := 0; i < forgottenCount; i++ { - topic, err := pd.getString() - if err != nil { - return err - } - partitionCount, err := pd.getArrayLength() - if err != nil { - return err - } - r.forgotten[topic] = make([]int32, partitionCount) - - for j := 0; j < partitionCount; j++ { - partition, err := pd.getInt32() - if err != nil { - return err - } - r.forgotten[topic][j] = partition - } - } - } - - if r.Version >= 11 { - r.RackID, err = pd.getString() - if err != nil { - return err - } - } - - return nil -} - -func (r *FetchRequest) key() int16 { - return 1 -} - -func (r *FetchRequest) version() int16 { - return r.Version -} - -func (r *FetchRequest) headerVersion() int16 { - return 1 -} - -func (r *FetchRequest) requiredVersion() KafkaVersion { - switch r.Version { - case 0: - return MinVersion - case 1: - return V0_9_0_0 - case 2: - return V0_10_0_0 - case 3: - return V0_10_1_0 - case 4, 5: - return V0_11_0_0 - case 6: - return V1_0_0_0 - case 7: - return V1_1_0_0 - case 8: - return V2_0_0_0 - case 9, 10: - return V2_1_0_0 - case 11: - return V2_3_0_0 - default: - return MaxVersion - } -} - -func (r *FetchRequest) AddBlock(topic string, partitionID int32, fetchOffset int64, maxBytes int32) { - if r.blocks == nil { - r.blocks = make(map[string]map[int32]*fetchRequestBlock) - } - - if r.Version >= 7 && r.forgotten == nil { - r.forgotten = make(map[string][]int32) - } - - if r.blocks[topic] == nil { - r.blocks[topic] = make(map[int32]*fetchRequestBlock) - } - - tmp := new(fetchRequestBlock) - tmp.Version = r.Version - tmp.maxBytes = maxBytes - tmp.fetchOffset = fetchOffset - if r.Version >= 9 { - tmp.currentLeaderEpoch = int32(-1) - } - - r.blocks[topic][partitionID] = tmp -} diff --git a/vendor/github.com/Shopify/sarama/fetch_response.go b/vendor/github.com/Shopify/sarama/fetch_response.go deleted file mode 100644 index ca6d7883..00000000 --- a/vendor/github.com/Shopify/sarama/fetch_response.go +++ /dev/null @@ -1,546 +0,0 @@ -package sarama - -import ( - "sort" - "time" -) - -type AbortedTransaction struct { - ProducerID int64 - FirstOffset int64 -} - -func (t *AbortedTransaction) decode(pd packetDecoder) (err error) { - if t.ProducerID, err = pd.getInt64(); err != nil { - return err - } - - if t.FirstOffset, err = pd.getInt64(); err != nil { - return err - } - - return nil -} - -func (t *AbortedTransaction) encode(pe packetEncoder) (err error) { - pe.putInt64(t.ProducerID) - pe.putInt64(t.FirstOffset) - - return nil -} - -type FetchResponseBlock struct { - Err KError - HighWaterMarkOffset int64 - LastStableOffset int64 - LogStartOffset int64 - AbortedTransactions []*AbortedTransaction - PreferredReadReplica int32 - Records *Records // deprecated: use FetchResponseBlock.RecordsSet - RecordsSet []*Records - Partial bool -} - -func (b *FetchResponseBlock) decode(pd packetDecoder, version int16) (err error) { - tmp, err := pd.getInt16() - if err != nil { - return err - } - b.Err = KError(tmp) - - b.HighWaterMarkOffset, err = pd.getInt64() - if err != nil { - return err - } - - if version >= 4 { - b.LastStableOffset, err = pd.getInt64() - if err != nil { - return err - } - - if version >= 5 { - b.LogStartOffset, err = pd.getInt64() - if err != nil { - return err - } - } - - numTransact, err := pd.getArrayLength() - if err != nil { - return err - } - - if numTransact >= 0 { - b.AbortedTransactions = make([]*AbortedTransaction, numTransact) - } - - for i := 0; i < numTransact; i++ { - transact := new(AbortedTransaction) - if err = transact.decode(pd); err != nil { - return err - } - b.AbortedTransactions[i] = transact - } - } - - if version >= 11 { - b.PreferredReadReplica, err = pd.getInt32() - if err != nil { - return err - } - } - - recordsSize, err := pd.getInt32() - if err != nil { - return err - } - - recordsDecoder, err := pd.getSubset(int(recordsSize)) - if err != nil { - return err - } - - b.RecordsSet = []*Records{} - - for recordsDecoder.remaining() > 0 { - records := &Records{} - if err := records.decode(recordsDecoder); err != nil { - // If we have at least one decoded records, this is not an error - if err == ErrInsufficientData { - if len(b.RecordsSet) == 0 { - b.Partial = true - } - break - } - return err - } - - partial, err := records.isPartial() - if err != nil { - return err - } - - n, err := records.numRecords() - if err != nil { - return err - } - - if n > 0 || (partial && len(b.RecordsSet) == 0) { - b.RecordsSet = append(b.RecordsSet, records) - - if b.Records == nil { - b.Records = records - } - } - - overflow, err := records.isOverflow() - if err != nil { - return err - } - - if partial || overflow { - break - } - } - - return nil -} - -func (b *FetchResponseBlock) numRecords() (int, error) { - sum := 0 - - for _, records := range b.RecordsSet { - count, err := records.numRecords() - if err != nil { - return 0, err - } - - sum += count - } - - return sum, nil -} - -func (b *FetchResponseBlock) isPartial() (bool, error) { - if b.Partial { - return true, nil - } - - if len(b.RecordsSet) == 1 { - return b.RecordsSet[0].isPartial() - } - - return false, nil -} - -func (b *FetchResponseBlock) encode(pe packetEncoder, version int16) (err error) { - pe.putInt16(int16(b.Err)) - - pe.putInt64(b.HighWaterMarkOffset) - - if version >= 4 { - pe.putInt64(b.LastStableOffset) - - if version >= 5 { - pe.putInt64(b.LogStartOffset) - } - - if err = pe.putArrayLength(len(b.AbortedTransactions)); err != nil { - return err - } - for _, transact := range b.AbortedTransactions { - if err = transact.encode(pe); err != nil { - return err - } - } - } - - if version >= 11 { - pe.putInt32(b.PreferredReadReplica) - } - - pe.push(&lengthField{}) - for _, records := range b.RecordsSet { - err = records.encode(pe) - if err != nil { - return err - } - } - return pe.pop() -} - -func (b *FetchResponseBlock) getAbortedTransactions() []*AbortedTransaction { - // I can't find any doc that guarantee the field `fetchResponse.AbortedTransactions` is ordered - // plus Java implementation use a PriorityQueue based on `FirstOffset`. I guess we have to order it ourself - at := b.AbortedTransactions - sort.Slice( - at, - func(i, j int) bool { return at[i].FirstOffset < at[j].FirstOffset }, - ) - return at -} - -type FetchResponse struct { - Blocks map[string]map[int32]*FetchResponseBlock - ThrottleTime time.Duration - ErrorCode int16 - SessionID int32 - Version int16 - LogAppendTime bool - Timestamp time.Time -} - -func (r *FetchResponse) decode(pd packetDecoder, version int16) (err error) { - r.Version = version - - if r.Version >= 1 { - throttle, err := pd.getInt32() - if err != nil { - return err - } - r.ThrottleTime = time.Duration(throttle) * time.Millisecond - } - - if r.Version >= 7 { - r.ErrorCode, err = pd.getInt16() - if err != nil { - return err - } - r.SessionID, err = pd.getInt32() - if err != nil { - return err - } - } - - numTopics, err := pd.getArrayLength() - if err != nil { - return err - } - - r.Blocks = make(map[string]map[int32]*FetchResponseBlock, numTopics) - for i := 0; i < numTopics; i++ { - name, err := pd.getString() - if err != nil { - return err - } - - numBlocks, err := pd.getArrayLength() - if err != nil { - return err - } - - r.Blocks[name] = make(map[int32]*FetchResponseBlock, numBlocks) - - for j := 0; j < numBlocks; j++ { - id, err := pd.getInt32() - if err != nil { - return err - } - - block := new(FetchResponseBlock) - err = block.decode(pd, version) - if err != nil { - return err - } - r.Blocks[name][id] = block - } - } - - return nil -} - -func (r *FetchResponse) encode(pe packetEncoder) (err error) { - if r.Version >= 1 { - pe.putInt32(int32(r.ThrottleTime / time.Millisecond)) - } - - if r.Version >= 7 { - pe.putInt16(r.ErrorCode) - pe.putInt32(r.SessionID) - } - - err = pe.putArrayLength(len(r.Blocks)) - if err != nil { - return err - } - - for topic, partitions := range r.Blocks { - err = pe.putString(topic) - if err != nil { - return err - } - - err = pe.putArrayLength(len(partitions)) - if err != nil { - return err - } - - for id, block := range partitions { - pe.putInt32(id) - err = block.encode(pe, r.Version) - if err != nil { - return err - } - } - } - return nil -} - -func (r *FetchResponse) key() int16 { - return 1 -} - -func (r *FetchResponse) version() int16 { - return r.Version -} - -func (r *FetchResponse) headerVersion() int16 { - return 0 -} - -func (r *FetchResponse) requiredVersion() KafkaVersion { - switch r.Version { - case 0: - return MinVersion - case 1: - return V0_9_0_0 - case 2: - return V0_10_0_0 - case 3: - return V0_10_1_0 - case 4, 5: - return V0_11_0_0 - case 6: - return V1_0_0_0 - case 7: - return V1_1_0_0 - case 8: - return V2_0_0_0 - case 9, 10: - return V2_1_0_0 - case 11: - return V2_3_0_0 - default: - return MaxVersion - } -} - -func (r *FetchResponse) GetBlock(topic string, partition int32) *FetchResponseBlock { - if r.Blocks == nil { - return nil - } - - if r.Blocks[topic] == nil { - return nil - } - - return r.Blocks[topic][partition] -} - -func (r *FetchResponse) AddError(topic string, partition int32, err KError) { - if r.Blocks == nil { - r.Blocks = make(map[string]map[int32]*FetchResponseBlock) - } - partitions, ok := r.Blocks[topic] - if !ok { - partitions = make(map[int32]*FetchResponseBlock) - r.Blocks[topic] = partitions - } - frb, ok := partitions[partition] - if !ok { - frb = new(FetchResponseBlock) - partitions[partition] = frb - } - frb.Err = err -} - -func (r *FetchResponse) getOrCreateBlock(topic string, partition int32) *FetchResponseBlock { - if r.Blocks == nil { - r.Blocks = make(map[string]map[int32]*FetchResponseBlock) - } - partitions, ok := r.Blocks[topic] - if !ok { - partitions = make(map[int32]*FetchResponseBlock) - r.Blocks[topic] = partitions - } - frb, ok := partitions[partition] - if !ok { - frb = new(FetchResponseBlock) - partitions[partition] = frb - } - - return frb -} - -func encodeKV(key, value Encoder) ([]byte, []byte) { - var kb []byte - var vb []byte - if key != nil { - kb, _ = key.Encode() - } - if value != nil { - vb, _ = value.Encode() - } - - return kb, vb -} - -func (r *FetchResponse) AddMessageWithTimestamp(topic string, partition int32, key, value Encoder, offset int64, timestamp time.Time, version int8) { - frb := r.getOrCreateBlock(topic, partition) - kb, vb := encodeKV(key, value) - if r.LogAppendTime { - timestamp = r.Timestamp - } - msg := &Message{Key: kb, Value: vb, LogAppendTime: r.LogAppendTime, Timestamp: timestamp, Version: version} - msgBlock := &MessageBlock{Msg: msg, Offset: offset} - if len(frb.RecordsSet) == 0 { - records := newLegacyRecords(&MessageSet{}) - frb.RecordsSet = []*Records{&records} - } - set := frb.RecordsSet[0].MsgSet - set.Messages = append(set.Messages, msgBlock) -} - -func (r *FetchResponse) AddRecordWithTimestamp(topic string, partition int32, key, value Encoder, offset int64, timestamp time.Time) { - frb := r.getOrCreateBlock(topic, partition) - kb, vb := encodeKV(key, value) - if len(frb.RecordsSet) == 0 { - records := newDefaultRecords(&RecordBatch{Version: 2, LogAppendTime: r.LogAppendTime, FirstTimestamp: timestamp, MaxTimestamp: r.Timestamp}) - frb.RecordsSet = []*Records{&records} - } - batch := frb.RecordsSet[0].RecordBatch - rec := &Record{Key: kb, Value: vb, OffsetDelta: offset, TimestampDelta: timestamp.Sub(batch.FirstTimestamp)} - batch.addRecord(rec) -} - -// AddRecordBatchWithTimestamp is similar to AddRecordWithTimestamp -// But instead of appending 1 record to a batch, it append a new batch containing 1 record to the fetchResponse -// Since transaction are handled on batch level (the whole batch is either committed or aborted), use this to test transactions -func (r *FetchResponse) AddRecordBatchWithTimestamp(topic string, partition int32, key, value Encoder, offset int64, producerID int64, isTransactional bool, timestamp time.Time) { - frb := r.getOrCreateBlock(topic, partition) - kb, vb := encodeKV(key, value) - - records := newDefaultRecords(&RecordBatch{Version: 2, LogAppendTime: r.LogAppendTime, FirstTimestamp: timestamp, MaxTimestamp: r.Timestamp}) - batch := &RecordBatch{ - Version: 2, - LogAppendTime: r.LogAppendTime, - FirstTimestamp: timestamp, - MaxTimestamp: r.Timestamp, - FirstOffset: offset, - LastOffsetDelta: 0, - ProducerID: producerID, - IsTransactional: isTransactional, - } - rec := &Record{Key: kb, Value: vb, OffsetDelta: 0, TimestampDelta: timestamp.Sub(batch.FirstTimestamp)} - batch.addRecord(rec) - records.RecordBatch = batch - - frb.RecordsSet = append(frb.RecordsSet, &records) -} - -func (r *FetchResponse) AddControlRecordWithTimestamp(topic string, partition int32, offset int64, producerID int64, recordType ControlRecordType, timestamp time.Time) { - frb := r.getOrCreateBlock(topic, partition) - - // batch - batch := &RecordBatch{ - Version: 2, - LogAppendTime: r.LogAppendTime, - FirstTimestamp: timestamp, - MaxTimestamp: r.Timestamp, - FirstOffset: offset, - LastOffsetDelta: 0, - ProducerID: producerID, - IsTransactional: true, - Control: true, - } - - // records - records := newDefaultRecords(nil) - records.RecordBatch = batch - - // record - crAbort := ControlRecord{ - Version: 0, - Type: recordType, - } - crKey := &realEncoder{raw: make([]byte, 4)} - crValue := &realEncoder{raw: make([]byte, 6)} - crAbort.encode(crKey, crValue) - rec := &Record{Key: ByteEncoder(crKey.raw), Value: ByteEncoder(crValue.raw), OffsetDelta: 0, TimestampDelta: timestamp.Sub(batch.FirstTimestamp)} - batch.addRecord(rec) - - frb.RecordsSet = append(frb.RecordsSet, &records) -} - -func (r *FetchResponse) AddMessage(topic string, partition int32, key, value Encoder, offset int64) { - r.AddMessageWithTimestamp(topic, partition, key, value, offset, time.Time{}, 0) -} - -func (r *FetchResponse) AddRecord(topic string, partition int32, key, value Encoder, offset int64) { - r.AddRecordWithTimestamp(topic, partition, key, value, offset, time.Time{}) -} - -func (r *FetchResponse) AddRecordBatch(topic string, partition int32, key, value Encoder, offset int64, producerID int64, isTransactional bool) { - r.AddRecordBatchWithTimestamp(topic, partition, key, value, offset, producerID, isTransactional, time.Time{}) -} - -func (r *FetchResponse) AddControlRecord(topic string, partition int32, offset int64, producerID int64, recordType ControlRecordType) { - // define controlRecord key and value - r.AddControlRecordWithTimestamp(topic, partition, offset, producerID, recordType, time.Time{}) -} - -func (r *FetchResponse) SetLastOffsetDelta(topic string, partition int32, offset int32) { - frb := r.getOrCreateBlock(topic, partition) - if len(frb.RecordsSet) == 0 { - records := newDefaultRecords(&RecordBatch{Version: 2}) - frb.RecordsSet = []*Records{&records} - } - batch := frb.RecordsSet[0].RecordBatch - batch.LastOffsetDelta = offset -} - -func (r *FetchResponse) SetLastStableOffset(topic string, partition int32, offset int64) { - frb := r.getOrCreateBlock(topic, partition) - frb.LastStableOffset = offset -} diff --git a/vendor/github.com/Shopify/sarama/find_coordinator_request.go b/vendor/github.com/Shopify/sarama/find_coordinator_request.go deleted file mode 100644 index 597bcbf7..00000000 --- a/vendor/github.com/Shopify/sarama/find_coordinator_request.go +++ /dev/null @@ -1,65 +0,0 @@ -package sarama - -type CoordinatorType int8 - -const ( - CoordinatorGroup CoordinatorType = iota - CoordinatorTransaction -) - -type FindCoordinatorRequest struct { - Version int16 - CoordinatorKey string - CoordinatorType CoordinatorType -} - -func (f *FindCoordinatorRequest) encode(pe packetEncoder) error { - if err := pe.putString(f.CoordinatorKey); err != nil { - return err - } - - if f.Version >= 1 { - pe.putInt8(int8(f.CoordinatorType)) - } - - return nil -} - -func (f *FindCoordinatorRequest) decode(pd packetDecoder, version int16) (err error) { - if f.CoordinatorKey, err = pd.getString(); err != nil { - return err - } - - if version >= 1 { - f.Version = version - coordinatorType, err := pd.getInt8() - if err != nil { - return err - } - - f.CoordinatorType = CoordinatorType(coordinatorType) - } - - return nil -} - -func (f *FindCoordinatorRequest) key() int16 { - return 10 -} - -func (f *FindCoordinatorRequest) version() int16 { - return f.Version -} - -func (r *FindCoordinatorRequest) headerVersion() int16 { - return 1 -} - -func (f *FindCoordinatorRequest) requiredVersion() KafkaVersion { - switch f.Version { - case 1: - return V0_11_0_0 - default: - return V0_8_2_0 - } -} diff --git a/vendor/github.com/Shopify/sarama/find_coordinator_response.go b/vendor/github.com/Shopify/sarama/find_coordinator_response.go deleted file mode 100644 index 83a648ad..00000000 --- a/vendor/github.com/Shopify/sarama/find_coordinator_response.go +++ /dev/null @@ -1,96 +0,0 @@ -package sarama - -import ( - "time" -) - -var NoNode = &Broker{id: -1, addr: ":-1"} - -type FindCoordinatorResponse struct { - Version int16 - ThrottleTime time.Duration - Err KError - ErrMsg *string - Coordinator *Broker -} - -func (f *FindCoordinatorResponse) decode(pd packetDecoder, version int16) (err error) { - if version >= 1 { - f.Version = version - - throttleTime, err := pd.getInt32() - if err != nil { - return err - } - f.ThrottleTime = time.Duration(throttleTime) * time.Millisecond - } - - tmp, err := pd.getInt16() - if err != nil { - return err - } - f.Err = KError(tmp) - - if version >= 1 { - if f.ErrMsg, err = pd.getNullableString(); err != nil { - return err - } - } - - coordinator := new(Broker) - // The version is hardcoded to 0, as version 1 of the Broker-decode - // contains the rack-field which is not present in the FindCoordinatorResponse. - if err := coordinator.decode(pd, 0); err != nil { - return err - } - if coordinator.addr == ":0" { - return nil - } - f.Coordinator = coordinator - - return nil -} - -func (f *FindCoordinatorResponse) encode(pe packetEncoder) error { - if f.Version >= 1 { - pe.putInt32(int32(f.ThrottleTime / time.Millisecond)) - } - - pe.putInt16(int16(f.Err)) - - if f.Version >= 1 { - if err := pe.putNullableString(f.ErrMsg); err != nil { - return err - } - } - - coordinator := f.Coordinator - if coordinator == nil { - coordinator = NoNode - } - if err := coordinator.encode(pe, 0); err != nil { - return err - } - return nil -} - -func (f *FindCoordinatorResponse) key() int16 { - return 10 -} - -func (f *FindCoordinatorResponse) version() int16 { - return f.Version -} - -func (r *FindCoordinatorResponse) headerVersion() int16 { - return 0 -} - -func (f *FindCoordinatorResponse) requiredVersion() KafkaVersion { - switch f.Version { - case 1: - return V0_11_0_0 - default: - return V0_8_2_0 - } -} diff --git a/vendor/github.com/Shopify/sarama/go.mod b/vendor/github.com/Shopify/sarama/go.mod deleted file mode 100644 index d775035c..00000000 --- a/vendor/github.com/Shopify/sarama/go.mod +++ /dev/null @@ -1,35 +0,0 @@ -module github.com/Shopify/sarama - -go 1.13 - -require ( - github.com/Shopify/toxiproxy v2.1.4+incompatible - github.com/davecgh/go-spew v1.1.1 - github.com/eapache/go-resiliency v1.2.0 - github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 - github.com/eapache/queue v1.1.0 - github.com/fortytw2/leaktest v1.3.0 - github.com/frankban/quicktest v1.10.0 // indirect - github.com/golang/snappy v0.0.1 // indirect - github.com/google/go-cmp v0.4.1 // indirect - github.com/hashicorp/go-uuid v1.0.2 // indirect - github.com/jcmturner/gofork v1.0.0 - github.com/klauspost/compress v1.10.10 - github.com/kr/text v0.2.0 // indirect - github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect - github.com/pierrec/lz4 v2.5.2+incompatible - github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 - github.com/stretchr/testify v1.6.0 - github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c - github.com/xdg/stringprep v1.0.0 // indirect - golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37 // indirect - golang.org/x/net v0.0.0-20200528225125-3c3fba18258b - golang.org/x/text v0.3.2 // indirect - gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect - gopkg.in/jcmturner/aescts.v1 v1.0.1 // indirect - gopkg.in/jcmturner/dnsutils.v1 v1.0.1 // indirect - gopkg.in/jcmturner/goidentity.v3 v3.0.0 // indirect - gopkg.in/jcmturner/gokrb5.v7 v7.5.0 - gopkg.in/jcmturner/rpc.v1 v1.1.0 // indirect - gopkg.in/yaml.v3 v3.0.0-20200601152816-913338de1bd2 // indirect -) diff --git a/vendor/github.com/Shopify/sarama/go.sum b/vendor/github.com/Shopify/sarama/go.sum deleted file mode 100644 index ad0889d4..00000000 --- a/vendor/github.com/Shopify/sarama/go.sum +++ /dev/null @@ -1,88 +0,0 @@ -github.com/Shopify/toxiproxy v2.1.4+incompatible h1:TKdv8HiTLgE5wdJuEML90aBgNWsokNbMijUGhmcoBJc= -github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/eapache/go-resiliency v1.2.0 h1:v7g92e/KSN71Rq7vSThKaWIq68fL4YHvWyiUKorFR1Q= -github.com/eapache/go-resiliency v1.2.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= -github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 h1:YEetp8/yCZMuEPMUDHG0CW/brkkEp8mzqk2+ODEitlw= -github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= -github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= -github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= -github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= -github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= -github.com/frankban/quicktest v1.10.0 h1:Gfh+GAJZOAoKZsIZeZbdn2JF10kN1XHNvjsvQK8gVkE= -github.com/frankban/quicktest v1.10.0/go.mod h1:ui7WezCLWMWxVWr1GETZY3smRy0G4KWq9vcPtJmFl7Y= -github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= -github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= -github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.4.1 h1:/exdXoGamhu5ONeUJH0deniYLWYvQwW66yvlfiiKTu0= -github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/hashicorp/go-uuid v1.0.2 h1:cfejS+Tpcp13yd5nYHWDI6qVCny6wyX2Mt5SGur2IGE= -github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/jcmturner/gofork v1.0.0 h1:J7uCkflzTEhUZ64xqKnkDxq3kzc96ajM1Gli5ktUem8= -github.com/jcmturner/gofork v1.0.0/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o= -github.com/klauspost/compress v1.10.7 h1:7rix8v8GpI3ZBb0nSozFRgbtXKv+hOe+qfEpZqybrAg= -github.com/klauspost/compress v1.10.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= -github.com/klauspost/compress v1.10.10 h1:a/y8CglcM7gLGYmlbP/stPE5sR3hbhFRUjCBfd/0B3I= -github.com/klauspost/compress v1.10.10/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= -github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= -github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= -github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= -github.com/pierrec/lz4 v2.5.2+incompatible h1:WCjObylUIOlKy/+7Abdn34TLIkXiA4UWUMhxq9m9ZXI= -github.com/pierrec/lz4 v2.5.2+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 h1:MkV+77GLUNo5oJ0jf870itWm3D0Sjh7+Za9gazKc5LQ= -github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.6.0 h1:jlIyCplCJFULU/01vCkhKuTyc3OorI3bJFuw6obfgho= -github.com/stretchr/testify v1.6.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c h1:u40Z8hqBAAQyv+vATcGgV0YCnDjqSL7/q/JyPhhJSPk= -github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= -github.com/xdg/stringprep v1.0.0 h1:d9X0esnoa3dFsV0FG35rAT0RIhYFlPq7MiP+DW89La0= -github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37 h1:cg5LA/zNPRzIXIWSCxQW10Rvpy94aQh3LT/ShoCpkHw= -golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3 h1:0GoQqolDA55aaLxZyTzK/Y2ePZzZTUrRacwib7cNsYQ= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20200528225125-3c3fba18258b h1:IYiJPiJfzktmDAO1HQiwjMjwjlYKHAL7KzeD544RJPs= -golang.org/x/net v0.0.0-20200528225125-3c3fba18258b/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= -golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= -gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/jcmturner/aescts.v1 v1.0.1 h1:cVVZBK2b1zY26haWB4vbBiZrfFQnfbTVrE3xZq6hrEw= -gopkg.in/jcmturner/aescts.v1 v1.0.1/go.mod h1:nsR8qBOg+OucoIW+WMhB3GspUQXq9XorLnQb9XtvcOo= -gopkg.in/jcmturner/dnsutils.v1 v1.0.1 h1:cIuC1OLRGZrld+16ZJvvZxVJeKPsvd5eUIvxfoN5hSM= -gopkg.in/jcmturner/dnsutils.v1 v1.0.1/go.mod h1:m3v+5svpVOhtFAP/wSz+yzh4Mc0Fg7eRhxkJMWSIz9Q= -gopkg.in/jcmturner/goidentity.v3 v3.0.0 h1:1duIyWiTaYvVx3YX2CYtpJbUFd7/UuPYCfgXtQ3VTbI= -gopkg.in/jcmturner/goidentity.v3 v3.0.0/go.mod h1:oG2kH0IvSYNIu80dVAyu/yoefjq1mNfM5bm88whjWx4= -gopkg.in/jcmturner/gokrb5.v7 v7.5.0 h1:a9tsXlIDD9SKxotJMK3niV7rPZAJeX2aD/0yg3qlIrg= -gopkg.in/jcmturner/gokrb5.v7 v7.5.0/go.mod h1:l8VISx+WGYp+Fp7KRbsiUuXTTOnxIc3Tuvyavf11/WM= -gopkg.in/jcmturner/rpc.v1 v1.1.0 h1:QHIUxTX1ISuAv9dD2wJ9HWQVuWDX/Zc0PfeC2tjc4rU= -gopkg.in/jcmturner/rpc.v1 v1.1.0/go.mod h1:YIdkC4XfD6GXbzje11McwsDuOlZQSb9W4vfLvuNnlv8= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20200601152816-913338de1bd2 h1:VEmvx0P+GVTgkNu2EdTN988YCZPcD3lo9AoczZpucwc= -gopkg.in/yaml.v3 v3.0.0-20200601152816-913338de1bd2/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/vendor/github.com/Shopify/sarama/gssapi_kerberos.go b/vendor/github.com/Shopify/sarama/gssapi_kerberos.go deleted file mode 100644 index 1993fc08..00000000 --- a/vendor/github.com/Shopify/sarama/gssapi_kerberos.go +++ /dev/null @@ -1,258 +0,0 @@ -package sarama - -import ( - "encoding/binary" - "fmt" - "io" - "strings" - "time" - - "github.com/jcmturner/gofork/encoding/asn1" - "gopkg.in/jcmturner/gokrb5.v7/asn1tools" - "gopkg.in/jcmturner/gokrb5.v7/gssapi" - "gopkg.in/jcmturner/gokrb5.v7/iana/chksumtype" - "gopkg.in/jcmturner/gokrb5.v7/iana/keyusage" - "gopkg.in/jcmturner/gokrb5.v7/messages" - "gopkg.in/jcmturner/gokrb5.v7/types" -) - -const ( - TOK_ID_KRB_AP_REQ = 256 - GSS_API_GENERIC_TAG = 0x60 - KRB5_USER_AUTH = 1 - KRB5_KEYTAB_AUTH = 2 - GSS_API_INITIAL = 1 - GSS_API_VERIFY = 2 - GSS_API_FINISH = 3 -) - -type GSSAPIConfig struct { - AuthType int - KeyTabPath string - KerberosConfigPath string - ServiceName string - Username string - Password string - Realm string - DisablePAFXFAST bool -} - -type GSSAPIKerberosAuth struct { - Config *GSSAPIConfig - ticket messages.Ticket - encKey types.EncryptionKey - NewKerberosClientFunc func(config *GSSAPIConfig) (KerberosClient, error) - step int -} - -type KerberosClient interface { - Login() error - GetServiceTicket(spn string) (messages.Ticket, types.EncryptionKey, error) - Domain() string - CName() types.PrincipalName - Destroy() -} - -/* -* -* Appends length in big endian before payload, and send it to kafka -* - */ - -func (krbAuth *GSSAPIKerberosAuth) writePackage(broker *Broker, payload []byte) (int, error) { - length := len(payload) - finalPackage := make([]byte, length+4) //4 byte length header + payload - copy(finalPackage[4:], payload) - binary.BigEndian.PutUint32(finalPackage, uint32(length)) - bytes, err := broker.conn.Write(finalPackage) - if err != nil { - return bytes, err - } - return bytes, nil -} - -/* -* -* Read length (4 bytes) and then read the payload -* - */ - -func (krbAuth *GSSAPIKerberosAuth) readPackage(broker *Broker) ([]byte, int, error) { - bytesRead := 0 - lengthInBytes := make([]byte, 4) - bytes, err := io.ReadFull(broker.conn, lengthInBytes) - if err != nil { - return nil, bytesRead, err - } - bytesRead += bytes - payloadLength := binary.BigEndian.Uint32(lengthInBytes) - payloadBytes := make([]byte, payloadLength) // buffer for read.. - bytes, err = io.ReadFull(broker.conn, payloadBytes) // read bytes - if err != nil { - return payloadBytes, bytesRead, err - } - bytesRead += bytes - return payloadBytes, bytesRead, nil -} - -func (krbAuth *GSSAPIKerberosAuth) newAuthenticatorChecksum() []byte { - a := make([]byte, 24) - flags := []int{gssapi.ContextFlagInteg, gssapi.ContextFlagConf} - binary.LittleEndian.PutUint32(a[:4], 16) - for _, i := range flags { - f := binary.LittleEndian.Uint32(a[20:24]) - f |= uint32(i) - binary.LittleEndian.PutUint32(a[20:24], f) - } - return a -} - -/* -* -* Construct Kerberos AP_REQ package, conforming to RFC-4120 -* https://tools.ietf.org/html/rfc4120#page-84 -* - */ -func (krbAuth *GSSAPIKerberosAuth) createKrb5Token( - domain string, cname types.PrincipalName, - ticket messages.Ticket, - sessionKey types.EncryptionKey) ([]byte, error) { - auth, err := types.NewAuthenticator(domain, cname) - if err != nil { - return nil, err - } - auth.Cksum = types.Checksum{ - CksumType: chksumtype.GSSAPI, - Checksum: krbAuth.newAuthenticatorChecksum(), - } - APReq, err := messages.NewAPReq( - ticket, - sessionKey, - auth, - ) - if err != nil { - return nil, err - } - aprBytes := make([]byte, 2) - binary.BigEndian.PutUint16(aprBytes, TOK_ID_KRB_AP_REQ) - tb, err := APReq.Marshal() - if err != nil { - return nil, err - } - aprBytes = append(aprBytes, tb...) - return aprBytes, nil -} - -/* -* -* Append the GSS-API header to the payload, conforming to RFC-2743 -* Section 3.1, Mechanism-Independent Token Format -* -* https://tools.ietf.org/html/rfc2743#page-81 -* -* GSSAPIHeader + -* - */ -func (krbAuth *GSSAPIKerberosAuth) appendGSSAPIHeader(payload []byte) ([]byte, error) { - oidBytes, err := asn1.Marshal(gssapi.OID(gssapi.OIDKRB5)) - if err != nil { - return nil, err - } - tkoLengthBytes := asn1tools.MarshalLengthBytes(len(oidBytes) + len(payload)) - GSSHeader := append([]byte{GSS_API_GENERIC_TAG}, tkoLengthBytes...) - GSSHeader = append(GSSHeader, oidBytes...) - GSSPackage := append(GSSHeader, payload...) - return GSSPackage, nil -} - -func (krbAuth *GSSAPIKerberosAuth) initSecContext(bytes []byte, kerberosClient KerberosClient) ([]byte, error) { - switch krbAuth.step { - case GSS_API_INITIAL: - aprBytes, err := krbAuth.createKrb5Token( - kerberosClient.Domain(), - kerberosClient.CName(), - krbAuth.ticket, - krbAuth.encKey) - if err != nil { - return nil, err - } - krbAuth.step = GSS_API_VERIFY - return krbAuth.appendGSSAPIHeader(aprBytes) - case GSS_API_VERIFY: - wrapTokenReq := gssapi.WrapToken{} - if err := wrapTokenReq.Unmarshal(bytes, true); err != nil { - return nil, err - } - // Validate response. - isValid, err := wrapTokenReq.Verify(krbAuth.encKey, keyusage.GSSAPI_ACCEPTOR_SEAL) - if !isValid { - return nil, err - } - - wrapTokenResponse, err := gssapi.NewInitiatorWrapToken(wrapTokenReq.Payload, krbAuth.encKey) - if err != nil { - return nil, err - } - krbAuth.step = GSS_API_FINISH - return wrapTokenResponse.Marshal() - } - return nil, nil -} - -/* This does the handshake for authorization */ -func (krbAuth *GSSAPIKerberosAuth) Authorize(broker *Broker) error { - kerberosClient, err := krbAuth.NewKerberosClientFunc(krbAuth.Config) - if err != nil { - Logger.Printf("Kerberos client error: %s", err) - return err - } - - err = kerberosClient.Login() - if err != nil { - Logger.Printf("Kerberos client error: %s", err) - return err - } - // Construct SPN using serviceName and host - // SPN format: / - - host := strings.SplitN(broker.addr, ":", 2)[0] // Strip port part - spn := fmt.Sprintf("%s/%s", broker.conf.Net.SASL.GSSAPI.ServiceName, host) - - ticket, encKey, err := kerberosClient.GetServiceTicket(spn) - - if err != nil { - Logger.Printf("Error getting Kerberos service ticket : %s", err) - return err - } - krbAuth.ticket = ticket - krbAuth.encKey = encKey - krbAuth.step = GSS_API_INITIAL - var receivedBytes []byte = nil - defer kerberosClient.Destroy() - for { - packBytes, err := krbAuth.initSecContext(receivedBytes, kerberosClient) - if err != nil { - Logger.Printf("Error while performing GSSAPI Kerberos Authentication: %s\n", err) - return err - } - requestTime := time.Now() - bytesWritten, err := krbAuth.writePackage(broker, packBytes) - if err != nil { - Logger.Printf("Error while performing GSSAPI Kerberos Authentication: %s\n", err) - return err - } - broker.updateOutgoingCommunicationMetrics(bytesWritten) - if krbAuth.step == GSS_API_VERIFY { - bytesRead := 0 - receivedBytes, bytesRead, err = krbAuth.readPackage(broker) - requestLatency := time.Since(requestTime) - broker.updateIncomingCommunicationMetrics(bytesRead, requestLatency) - if err != nil { - Logger.Printf("Error while performing GSSAPI Kerberos Authentication: %s\n", err) - return err - } - } else if krbAuth.step == GSS_API_FINISH { - return nil - } - } -} diff --git a/vendor/github.com/Shopify/sarama/heartbeat_request.go b/vendor/github.com/Shopify/sarama/heartbeat_request.go deleted file mode 100644 index e9d9af19..00000000 --- a/vendor/github.com/Shopify/sarama/heartbeat_request.go +++ /dev/null @@ -1,51 +0,0 @@ -package sarama - -type HeartbeatRequest struct { - GroupId string - GenerationId int32 - MemberId string -} - -func (r *HeartbeatRequest) encode(pe packetEncoder) error { - if err := pe.putString(r.GroupId); err != nil { - return err - } - - pe.putInt32(r.GenerationId) - - if err := pe.putString(r.MemberId); err != nil { - return err - } - - return nil -} - -func (r *HeartbeatRequest) decode(pd packetDecoder, version int16) (err error) { - if r.GroupId, err = pd.getString(); err != nil { - return - } - if r.GenerationId, err = pd.getInt32(); err != nil { - return - } - if r.MemberId, err = pd.getString(); err != nil { - return - } - - return nil -} - -func (r *HeartbeatRequest) key() int16 { - return 12 -} - -func (r *HeartbeatRequest) version() int16 { - return 0 -} - -func (r *HeartbeatRequest) headerVersion() int16 { - return 1 -} - -func (r *HeartbeatRequest) requiredVersion() KafkaVersion { - return V0_9_0_0 -} diff --git a/vendor/github.com/Shopify/sarama/heartbeat_response.go b/vendor/github.com/Shopify/sarama/heartbeat_response.go deleted file mode 100644 index 577ab72e..00000000 --- a/vendor/github.com/Shopify/sarama/heartbeat_response.go +++ /dev/null @@ -1,36 +0,0 @@ -package sarama - -type HeartbeatResponse struct { - Err KError -} - -func (r *HeartbeatResponse) encode(pe packetEncoder) error { - pe.putInt16(int16(r.Err)) - return nil -} - -func (r *HeartbeatResponse) decode(pd packetDecoder, version int16) error { - kerr, err := pd.getInt16() - if err != nil { - return err - } - r.Err = KError(kerr) - - return nil -} - -func (r *HeartbeatResponse) key() int16 { - return 12 -} - -func (r *HeartbeatResponse) version() int16 { - return 0 -} - -func (r *HeartbeatResponse) headerVersion() int16 { - return 0 -} - -func (r *HeartbeatResponse) requiredVersion() KafkaVersion { - return V0_9_0_0 -} diff --git a/vendor/github.com/Shopify/sarama/init_producer_id_request.go b/vendor/github.com/Shopify/sarama/init_producer_id_request.go deleted file mode 100644 index 68944439..00000000 --- a/vendor/github.com/Shopify/sarama/init_producer_id_request.go +++ /dev/null @@ -1,47 +0,0 @@ -package sarama - -import "time" - -type InitProducerIDRequest struct { - TransactionalID *string - TransactionTimeout time.Duration -} - -func (i *InitProducerIDRequest) encode(pe packetEncoder) error { - if err := pe.putNullableString(i.TransactionalID); err != nil { - return err - } - pe.putInt32(int32(i.TransactionTimeout / time.Millisecond)) - - return nil -} - -func (i *InitProducerIDRequest) decode(pd packetDecoder, version int16) (err error) { - if i.TransactionalID, err = pd.getNullableString(); err != nil { - return err - } - - timeout, err := pd.getInt32() - if err != nil { - return err - } - i.TransactionTimeout = time.Duration(timeout) * time.Millisecond - - return nil -} - -func (i *InitProducerIDRequest) key() int16 { - return 22 -} - -func (i *InitProducerIDRequest) version() int16 { - return 0 -} - -func (i *InitProducerIDRequest) headerVersion() int16 { - return 1 -} - -func (i *InitProducerIDRequest) requiredVersion() KafkaVersion { - return V0_11_0_0 -} diff --git a/vendor/github.com/Shopify/sarama/init_producer_id_response.go b/vendor/github.com/Shopify/sarama/init_producer_id_response.go deleted file mode 100644 index 3e1242bf..00000000 --- a/vendor/github.com/Shopify/sarama/init_producer_id_response.go +++ /dev/null @@ -1,59 +0,0 @@ -package sarama - -import "time" - -type InitProducerIDResponse struct { - ThrottleTime time.Duration - Err KError - ProducerID int64 - ProducerEpoch int16 -} - -func (i *InitProducerIDResponse) encode(pe packetEncoder) error { - pe.putInt32(int32(i.ThrottleTime / time.Millisecond)) - pe.putInt16(int16(i.Err)) - pe.putInt64(i.ProducerID) - pe.putInt16(i.ProducerEpoch) - - return nil -} - -func (i *InitProducerIDResponse) decode(pd packetDecoder, version int16) (err error) { - throttleTime, err := pd.getInt32() - if err != nil { - return err - } - i.ThrottleTime = time.Duration(throttleTime) * time.Millisecond - - kerr, err := pd.getInt16() - if err != nil { - return err - } - i.Err = KError(kerr) - - if i.ProducerID, err = pd.getInt64(); err != nil { - return err - } - - if i.ProducerEpoch, err = pd.getInt16(); err != nil { - return err - } - - return nil -} - -func (i *InitProducerIDResponse) key() int16 { - return 22 -} - -func (i *InitProducerIDResponse) version() int16 { - return 0 -} - -func (i *InitProducerIDResponse) headerVersion() int16 { - return 0 -} - -func (i *InitProducerIDResponse) requiredVersion() KafkaVersion { - return V0_11_0_0 -} diff --git a/vendor/github.com/Shopify/sarama/interceptors.go b/vendor/github.com/Shopify/sarama/interceptors.go deleted file mode 100644 index d0d33e52..00000000 --- a/vendor/github.com/Shopify/sarama/interceptors.go +++ /dev/null @@ -1,43 +0,0 @@ -package sarama - -// ProducerInterceptor allows you to intercept (and possibly mutate) the records -// received by the producer before they are published to the Kafka cluster. -// https://cwiki.apache.org/confluence/display/KAFKA/KIP-42%3A+Add+Producer+and+Consumer+Interceptors#KIP42:AddProducerandConsumerInterceptors-Motivation -type ProducerInterceptor interface { - - // OnSend is called when the producer message is intercepted. Please avoid - // modifying the message until it's safe to do so, as this is _not_ a copy - // of the message. - OnSend(*ProducerMessage) -} - -// ConsumerInterceptor allows you to intercept (and possibly mutate) the records -// received by the consumer before they are sent to the messages channel. -// https://cwiki.apache.org/confluence/display/KAFKA/KIP-42%3A+Add+Producer+and+Consumer+Interceptors#KIP42:AddProducerandConsumerInterceptors-Motivation -type ConsumerInterceptor interface { - - // OnConsume is called when the consumed message is intercepted. Please - // avoid modifying the message until it's safe to do so, as this is _not_ a - // copy of the message. - OnConsume(*ConsumerMessage) -} - -func (msg *ProducerMessage) safelyApplyInterceptor(interceptor ProducerInterceptor) { - defer func() { - if r := recover(); r != nil { - Logger.Printf("Error when calling producer interceptor: %s, %w\n", interceptor, r) - } - }() - - interceptor.OnSend(msg) -} - -func (msg *ConsumerMessage) safelyApplyInterceptor(interceptor ConsumerInterceptor) { - defer func() { - if r := recover(); r != nil { - Logger.Printf("Error when calling consumer interceptor: %s, %w\n", interceptor, r) - } - }() - - interceptor.OnConsume(msg) -} diff --git a/vendor/github.com/Shopify/sarama/join_group_request.go b/vendor/github.com/Shopify/sarama/join_group_request.go deleted file mode 100644 index 3734e82e..00000000 --- a/vendor/github.com/Shopify/sarama/join_group_request.go +++ /dev/null @@ -1,167 +0,0 @@ -package sarama - -type GroupProtocol struct { - Name string - Metadata []byte -} - -func (p *GroupProtocol) decode(pd packetDecoder) (err error) { - p.Name, err = pd.getString() - if err != nil { - return err - } - p.Metadata, err = pd.getBytes() - return err -} - -func (p *GroupProtocol) encode(pe packetEncoder) (err error) { - if err := pe.putString(p.Name); err != nil { - return err - } - if err := pe.putBytes(p.Metadata); err != nil { - return err - } - return nil -} - -type JoinGroupRequest struct { - Version int16 - GroupId string - SessionTimeout int32 - RebalanceTimeout int32 - MemberId string - ProtocolType string - GroupProtocols map[string][]byte // deprecated; use OrderedGroupProtocols - OrderedGroupProtocols []*GroupProtocol -} - -func (r *JoinGroupRequest) encode(pe packetEncoder) error { - if err := pe.putString(r.GroupId); err != nil { - return err - } - pe.putInt32(r.SessionTimeout) - if r.Version >= 1 { - pe.putInt32(r.RebalanceTimeout) - } - if err := pe.putString(r.MemberId); err != nil { - return err - } - if err := pe.putString(r.ProtocolType); err != nil { - return err - } - - if len(r.GroupProtocols) > 0 { - if len(r.OrderedGroupProtocols) > 0 { - return PacketDecodingError{"cannot specify both GroupProtocols and OrderedGroupProtocols on JoinGroupRequest"} - } - - if err := pe.putArrayLength(len(r.GroupProtocols)); err != nil { - return err - } - for name, metadata := range r.GroupProtocols { - if err := pe.putString(name); err != nil { - return err - } - if err := pe.putBytes(metadata); err != nil { - return err - } - } - } else { - if err := pe.putArrayLength(len(r.OrderedGroupProtocols)); err != nil { - return err - } - for _, protocol := range r.OrderedGroupProtocols { - if err := protocol.encode(pe); err != nil { - return err - } - } - } - - return nil -} - -func (r *JoinGroupRequest) decode(pd packetDecoder, version int16) (err error) { - r.Version = version - - if r.GroupId, err = pd.getString(); err != nil { - return - } - - if r.SessionTimeout, err = pd.getInt32(); err != nil { - return - } - - if version >= 1 { - if r.RebalanceTimeout, err = pd.getInt32(); err != nil { - return err - } - } - - if r.MemberId, err = pd.getString(); err != nil { - return - } - - if r.ProtocolType, err = pd.getString(); err != nil { - return - } - - n, err := pd.getArrayLength() - if err != nil { - return err - } - if n == 0 { - return nil - } - - r.GroupProtocols = make(map[string][]byte) - for i := 0; i < n; i++ { - protocol := &GroupProtocol{} - if err := protocol.decode(pd); err != nil { - return err - } - r.GroupProtocols[protocol.Name] = protocol.Metadata - r.OrderedGroupProtocols = append(r.OrderedGroupProtocols, protocol) - } - - return nil -} - -func (r *JoinGroupRequest) key() int16 { - return 11 -} - -func (r *JoinGroupRequest) version() int16 { - return r.Version -} - -func (r *JoinGroupRequest) headerVersion() int16 { - return 1 -} - -func (r *JoinGroupRequest) requiredVersion() KafkaVersion { - switch r.Version { - case 2: - return V0_11_0_0 - case 1: - return V0_10_1_0 - default: - return V0_9_0_0 - } -} - -func (r *JoinGroupRequest) AddGroupProtocol(name string, metadata []byte) { - r.OrderedGroupProtocols = append(r.OrderedGroupProtocols, &GroupProtocol{ - Name: name, - Metadata: metadata, - }) -} - -func (r *JoinGroupRequest) AddGroupProtocolMetadata(name string, metadata *ConsumerGroupMemberMetadata) error { - bin, err := encode(metadata, nil) - if err != nil { - return err - } - - r.AddGroupProtocol(name, bin) - return nil -} diff --git a/vendor/github.com/Shopify/sarama/join_group_response.go b/vendor/github.com/Shopify/sarama/join_group_response.go deleted file mode 100644 index 54b0a45c..00000000 --- a/vendor/github.com/Shopify/sarama/join_group_response.go +++ /dev/null @@ -1,139 +0,0 @@ -package sarama - -type JoinGroupResponse struct { - Version int16 - ThrottleTime int32 - Err KError - GenerationId int32 - GroupProtocol string - LeaderId string - MemberId string - Members map[string][]byte -} - -func (r *JoinGroupResponse) GetMembers() (map[string]ConsumerGroupMemberMetadata, error) { - members := make(map[string]ConsumerGroupMemberMetadata, len(r.Members)) - for id, bin := range r.Members { - meta := new(ConsumerGroupMemberMetadata) - if err := decode(bin, meta); err != nil { - return nil, err - } - members[id] = *meta - } - return members, nil -} - -func (r *JoinGroupResponse) encode(pe packetEncoder) error { - if r.Version >= 2 { - pe.putInt32(r.ThrottleTime) - } - pe.putInt16(int16(r.Err)) - pe.putInt32(r.GenerationId) - - if err := pe.putString(r.GroupProtocol); err != nil { - return err - } - if err := pe.putString(r.LeaderId); err != nil { - return err - } - if err := pe.putString(r.MemberId); err != nil { - return err - } - - if err := pe.putArrayLength(len(r.Members)); err != nil { - return err - } - - for memberId, memberMetadata := range r.Members { - if err := pe.putString(memberId); err != nil { - return err - } - - if err := pe.putBytes(memberMetadata); err != nil { - return err - } - } - - return nil -} - -func (r *JoinGroupResponse) decode(pd packetDecoder, version int16) (err error) { - r.Version = version - - if version >= 2 { - if r.ThrottleTime, err = pd.getInt32(); err != nil { - return - } - } - - kerr, err := pd.getInt16() - if err != nil { - return err - } - - r.Err = KError(kerr) - - if r.GenerationId, err = pd.getInt32(); err != nil { - return - } - - if r.GroupProtocol, err = pd.getString(); err != nil { - return - } - - if r.LeaderId, err = pd.getString(); err != nil { - return - } - - if r.MemberId, err = pd.getString(); err != nil { - return - } - - n, err := pd.getArrayLength() - if err != nil { - return err - } - if n == 0 { - return nil - } - - r.Members = make(map[string][]byte) - for i := 0; i < n; i++ { - memberId, err := pd.getString() - if err != nil { - return err - } - - memberMetadata, err := pd.getBytes() - if err != nil { - return err - } - - r.Members[memberId] = memberMetadata - } - - return nil -} - -func (r *JoinGroupResponse) key() int16 { - return 11 -} - -func (r *JoinGroupResponse) version() int16 { - return r.Version -} - -func (r *JoinGroupResponse) headerVersion() int16 { - return 0 -} - -func (r *JoinGroupResponse) requiredVersion() KafkaVersion { - switch r.Version { - case 2: - return V0_11_0_0 - case 1: - return V0_10_1_0 - default: - return V0_9_0_0 - } -} diff --git a/vendor/github.com/Shopify/sarama/kerberos_client.go b/vendor/github.com/Shopify/sarama/kerberos_client.go deleted file mode 100644 index ebc11417..00000000 --- a/vendor/github.com/Shopify/sarama/kerberos_client.go +++ /dev/null @@ -1,46 +0,0 @@ -package sarama - -import ( - krb5client "gopkg.in/jcmturner/gokrb5.v7/client" - krb5config "gopkg.in/jcmturner/gokrb5.v7/config" - "gopkg.in/jcmturner/gokrb5.v7/keytab" - "gopkg.in/jcmturner/gokrb5.v7/types" -) - -type KerberosGoKrb5Client struct { - krb5client.Client -} - -func (c *KerberosGoKrb5Client) Domain() string { - return c.Credentials.Domain() -} - -func (c *KerberosGoKrb5Client) CName() types.PrincipalName { - return c.Credentials.CName() -} - -// NewKerberosClient creates kerberos client used to obtain TGT and TGS tokens. -// It uses pure go Kerberos 5 solution (RFC-4121 and RFC-4120). -// uses gokrb5 library underlying which is a pure go kerberos client with some GSS-API capabilities. -func NewKerberosClient(config *GSSAPIConfig) (KerberosClient, error) { - cfg, err := krb5config.Load(config.KerberosConfigPath) - if err != nil { - return nil, err - } - return createClient(config, cfg) -} - -func createClient(config *GSSAPIConfig, cfg *krb5config.Config) (KerberosClient, error) { - var client *krb5client.Client - if config.AuthType == KRB5_KEYTAB_AUTH { - kt, err := keytab.Load(config.KeyTabPath) - if err != nil { - return nil, err - } - client = krb5client.NewClientWithKeytab(config.Username, config.Realm, kt, cfg, krb5client.DisablePAFXFAST(config.DisablePAFXFAST)) - } else { - client = krb5client.NewClientWithPassword(config.Username, - config.Realm, config.Password, cfg, krb5client.DisablePAFXFAST(config.DisablePAFXFAST)) - } - return &KerberosGoKrb5Client{*client}, nil -} diff --git a/vendor/github.com/Shopify/sarama/leave_group_request.go b/vendor/github.com/Shopify/sarama/leave_group_request.go deleted file mode 100644 index d7789b68..00000000 --- a/vendor/github.com/Shopify/sarama/leave_group_request.go +++ /dev/null @@ -1,44 +0,0 @@ -package sarama - -type LeaveGroupRequest struct { - GroupId string - MemberId string -} - -func (r *LeaveGroupRequest) encode(pe packetEncoder) error { - if err := pe.putString(r.GroupId); err != nil { - return err - } - if err := pe.putString(r.MemberId); err != nil { - return err - } - - return nil -} - -func (r *LeaveGroupRequest) decode(pd packetDecoder, version int16) (err error) { - if r.GroupId, err = pd.getString(); err != nil { - return - } - if r.MemberId, err = pd.getString(); err != nil { - return - } - - return nil -} - -func (r *LeaveGroupRequest) key() int16 { - return 13 -} - -func (r *LeaveGroupRequest) version() int16 { - return 0 -} - -func (r *LeaveGroupRequest) headerVersion() int16 { - return 1 -} - -func (r *LeaveGroupRequest) requiredVersion() KafkaVersion { - return V0_9_0_0 -} diff --git a/vendor/github.com/Shopify/sarama/leave_group_response.go b/vendor/github.com/Shopify/sarama/leave_group_response.go deleted file mode 100644 index 25f8d5eb..00000000 --- a/vendor/github.com/Shopify/sarama/leave_group_response.go +++ /dev/null @@ -1,36 +0,0 @@ -package sarama - -type LeaveGroupResponse struct { - Err KError -} - -func (r *LeaveGroupResponse) encode(pe packetEncoder) error { - pe.putInt16(int16(r.Err)) - return nil -} - -func (r *LeaveGroupResponse) decode(pd packetDecoder, version int16) (err error) { - kerr, err := pd.getInt16() - if err != nil { - return err - } - r.Err = KError(kerr) - - return nil -} - -func (r *LeaveGroupResponse) key() int16 { - return 13 -} - -func (r *LeaveGroupResponse) version() int16 { - return 0 -} - -func (r *LeaveGroupResponse) headerVersion() int16 { - return 0 -} - -func (r *LeaveGroupResponse) requiredVersion() KafkaVersion { - return V0_9_0_0 -} diff --git a/vendor/github.com/Shopify/sarama/length_field.go b/vendor/github.com/Shopify/sarama/length_field.go deleted file mode 100644 index 7d864f6b..00000000 --- a/vendor/github.com/Shopify/sarama/length_field.go +++ /dev/null @@ -1,99 +0,0 @@ -package sarama - -import ( - "encoding/binary" - "sync" -) - -// LengthField implements the PushEncoder and PushDecoder interfaces for calculating 4-byte lengths. -type lengthField struct { - startOffset int - length int32 -} - -var lengthFieldPool = sync.Pool{} - -func acquireLengthField() *lengthField { - val := lengthFieldPool.Get() - if val != nil { - return val.(*lengthField) - } - return &lengthField{} -} - -func releaseLengthField(m *lengthField) { - lengthFieldPool.Put(m) -} - -func (l *lengthField) decode(pd packetDecoder) error { - var err error - l.length, err = pd.getInt32() - if err != nil { - return err - } - if l.length > int32(pd.remaining()) { - return ErrInsufficientData - } - return nil -} - -func (l *lengthField) saveOffset(in int) { - l.startOffset = in -} - -func (l *lengthField) reserveLength() int { - return 4 -} - -func (l *lengthField) run(curOffset int, buf []byte) error { - binary.BigEndian.PutUint32(buf[l.startOffset:], uint32(curOffset-l.startOffset-4)) - return nil -} - -func (l *lengthField) check(curOffset int, buf []byte) error { - if int32(curOffset-l.startOffset-4) != l.length { - return PacketDecodingError{"length field invalid"} - } - - return nil -} - -type varintLengthField struct { - startOffset int - length int64 -} - -func (l *varintLengthField) decode(pd packetDecoder) error { - var err error - l.length, err = pd.getVarint() - return err -} - -func (l *varintLengthField) saveOffset(in int) { - l.startOffset = in -} - -func (l *varintLengthField) adjustLength(currOffset int) int { - oldFieldSize := l.reserveLength() - l.length = int64(currOffset - l.startOffset - oldFieldSize) - - return l.reserveLength() - oldFieldSize -} - -func (l *varintLengthField) reserveLength() int { - var tmp [binary.MaxVarintLen64]byte - return binary.PutVarint(tmp[:], l.length) -} - -func (l *varintLengthField) run(curOffset int, buf []byte) error { - binary.PutVarint(buf[l.startOffset:], l.length) - return nil -} - -func (l *varintLengthField) check(curOffset int, buf []byte) error { - if int64(curOffset-l.startOffset-l.reserveLength()) != l.length { - return PacketDecodingError{"length field invalid"} - } - - return nil -} diff --git a/vendor/github.com/Shopify/sarama/list_groups_request.go b/vendor/github.com/Shopify/sarama/list_groups_request.go deleted file mode 100644 index ed44cc27..00000000 --- a/vendor/github.com/Shopify/sarama/list_groups_request.go +++ /dev/null @@ -1,28 +0,0 @@ -package sarama - -type ListGroupsRequest struct { -} - -func (r *ListGroupsRequest) encode(pe packetEncoder) error { - return nil -} - -func (r *ListGroupsRequest) decode(pd packetDecoder, version int16) (err error) { - return nil -} - -func (r *ListGroupsRequest) key() int16 { - return 16 -} - -func (r *ListGroupsRequest) version() int16 { - return 0 -} - -func (r *ListGroupsRequest) headerVersion() int16 { - return 1 -} - -func (r *ListGroupsRequest) requiredVersion() KafkaVersion { - return V0_9_0_0 -} diff --git a/vendor/github.com/Shopify/sarama/list_groups_response.go b/vendor/github.com/Shopify/sarama/list_groups_response.go deleted file mode 100644 index 777bae7e..00000000 --- a/vendor/github.com/Shopify/sarama/list_groups_response.go +++ /dev/null @@ -1,73 +0,0 @@ -package sarama - -type ListGroupsResponse struct { - Err KError - Groups map[string]string -} - -func (r *ListGroupsResponse) encode(pe packetEncoder) error { - pe.putInt16(int16(r.Err)) - - if err := pe.putArrayLength(len(r.Groups)); err != nil { - return err - } - for groupId, protocolType := range r.Groups { - if err := pe.putString(groupId); err != nil { - return err - } - if err := pe.putString(protocolType); err != nil { - return err - } - } - - return nil -} - -func (r *ListGroupsResponse) decode(pd packetDecoder, version int16) error { - kerr, err := pd.getInt16() - if err != nil { - return err - } - - r.Err = KError(kerr) - - n, err := pd.getArrayLength() - if err != nil { - return err - } - if n == 0 { - return nil - } - - r.Groups = make(map[string]string) - for i := 0; i < n; i++ { - groupId, err := pd.getString() - if err != nil { - return err - } - protocolType, err := pd.getString() - if err != nil { - return err - } - - r.Groups[groupId] = protocolType - } - - return nil -} - -func (r *ListGroupsResponse) key() int16 { - return 16 -} - -func (r *ListGroupsResponse) version() int16 { - return 0 -} - -func (r *ListGroupsResponse) headerVersion() int16 { - return 0 -} - -func (r *ListGroupsResponse) requiredVersion() KafkaVersion { - return V0_9_0_0 -} diff --git a/vendor/github.com/Shopify/sarama/list_partition_reassignments_request.go b/vendor/github.com/Shopify/sarama/list_partition_reassignments_request.go deleted file mode 100644 index c1ffa9ba..00000000 --- a/vendor/github.com/Shopify/sarama/list_partition_reassignments_request.go +++ /dev/null @@ -1,98 +0,0 @@ -package sarama - -type ListPartitionReassignmentsRequest struct { - TimeoutMs int32 - blocks map[string][]int32 - Version int16 -} - -func (r *ListPartitionReassignmentsRequest) encode(pe packetEncoder) error { - pe.putInt32(r.TimeoutMs) - - pe.putCompactArrayLength(len(r.blocks)) - - for topic, partitions := range r.blocks { - if err := pe.putCompactString(topic); err != nil { - return err - } - - if err := pe.putCompactInt32Array(partitions); err != nil { - return err - } - - pe.putEmptyTaggedFieldArray() - } - - pe.putEmptyTaggedFieldArray() - - return nil -} - -func (r *ListPartitionReassignmentsRequest) decode(pd packetDecoder, version int16) (err error) { - r.Version = version - - if r.TimeoutMs, err = pd.getInt32(); err != nil { - return err - } - - topicCount, err := pd.getCompactArrayLength() - if err != nil { - return err - } - if topicCount > 0 { - r.blocks = make(map[string][]int32) - for i := 0; i < topicCount; i++ { - topic, err := pd.getCompactString() - if err != nil { - return err - } - partitionCount, err := pd.getCompactArrayLength() - if err != nil { - return err - } - r.blocks[topic] = make([]int32, partitionCount) - for j := 0; j < partitionCount; j++ { - partition, err := pd.getInt32() - if err != nil { - return err - } - r.blocks[topic][j] = partition - } - if _, err := pd.getEmptyTaggedFieldArray(); err != nil { - return err - } - } - } - - if _, err := pd.getEmptyTaggedFieldArray(); err != nil { - return err - } - - return -} - -func (r *ListPartitionReassignmentsRequest) key() int16 { - return 46 -} - -func (r *ListPartitionReassignmentsRequest) version() int16 { - return r.Version -} - -func (r *ListPartitionReassignmentsRequest) headerVersion() int16 { - return 2 -} - -func (r *ListPartitionReassignmentsRequest) requiredVersion() KafkaVersion { - return V2_4_0_0 -} - -func (r *ListPartitionReassignmentsRequest) AddBlock(topic string, partitionIDs []int32) { - if r.blocks == nil { - r.blocks = make(map[string][]int32) - } - - if r.blocks[topic] == nil { - r.blocks[topic] = partitionIDs - } -} diff --git a/vendor/github.com/Shopify/sarama/list_partition_reassignments_response.go b/vendor/github.com/Shopify/sarama/list_partition_reassignments_response.go deleted file mode 100644 index 4baa6a08..00000000 --- a/vendor/github.com/Shopify/sarama/list_partition_reassignments_response.go +++ /dev/null @@ -1,169 +0,0 @@ -package sarama - -type PartitionReplicaReassignmentsStatus struct { - Replicas []int32 - AddingReplicas []int32 - RemovingReplicas []int32 -} - -func (b *PartitionReplicaReassignmentsStatus) encode(pe packetEncoder) error { - if err := pe.putCompactInt32Array(b.Replicas); err != nil { - return err - } - if err := pe.putCompactInt32Array(b.AddingReplicas); err != nil { - return err - } - if err := pe.putCompactInt32Array(b.RemovingReplicas); err != nil { - return err - } - - pe.putEmptyTaggedFieldArray() - - return nil -} - -func (b *PartitionReplicaReassignmentsStatus) decode(pd packetDecoder) (err error) { - if b.Replicas, err = pd.getCompactInt32Array(); err != nil { - return err - } - - if b.AddingReplicas, err = pd.getCompactInt32Array(); err != nil { - return err - } - - if b.RemovingReplicas, err = pd.getCompactInt32Array(); err != nil { - return err - } - - if _, err := pd.getEmptyTaggedFieldArray(); err != nil { - return err - } - - return err -} - -type ListPartitionReassignmentsResponse struct { - Version int16 - ThrottleTimeMs int32 - ErrorCode KError - ErrorMessage *string - TopicStatus map[string]map[int32]*PartitionReplicaReassignmentsStatus -} - -func (r *ListPartitionReassignmentsResponse) AddBlock(topic string, partition int32, replicas, addingReplicas, removingReplicas []int32) { - if r.TopicStatus == nil { - r.TopicStatus = make(map[string]map[int32]*PartitionReplicaReassignmentsStatus) - } - partitions := r.TopicStatus[topic] - if partitions == nil { - partitions = make(map[int32]*PartitionReplicaReassignmentsStatus) - r.TopicStatus[topic] = partitions - } - - partitions[partition] = &PartitionReplicaReassignmentsStatus{Replicas: replicas, AddingReplicas: addingReplicas, RemovingReplicas: removingReplicas} -} - -func (r *ListPartitionReassignmentsResponse) encode(pe packetEncoder) error { - pe.putInt32(r.ThrottleTimeMs) - pe.putInt16(int16(r.ErrorCode)) - if err := pe.putNullableCompactString(r.ErrorMessage); err != nil { - return err - } - - pe.putCompactArrayLength(len(r.TopicStatus)) - for topic, partitions := range r.TopicStatus { - if err := pe.putCompactString(topic); err != nil { - return err - } - pe.putCompactArrayLength(len(partitions)) - for partition, block := range partitions { - pe.putInt32(partition) - - if err := block.encode(pe); err != nil { - return err - } - } - pe.putEmptyTaggedFieldArray() - } - - pe.putEmptyTaggedFieldArray() - - return nil -} - -func (r *ListPartitionReassignmentsResponse) decode(pd packetDecoder, version int16) (err error) { - r.Version = version - - if r.ThrottleTimeMs, err = pd.getInt32(); err != nil { - return err - } - - kerr, err := pd.getInt16() - if err != nil { - return err - } - - r.ErrorCode = KError(kerr) - - if r.ErrorMessage, err = pd.getCompactNullableString(); err != nil { - return err - } - - numTopics, err := pd.getCompactArrayLength() - if err != nil { - return err - } - - r.TopicStatus = make(map[string]map[int32]*PartitionReplicaReassignmentsStatus, numTopics) - for i := 0; i < numTopics; i++ { - topic, err := pd.getCompactString() - if err != nil { - return err - } - - ongoingPartitionReassignments, err := pd.getCompactArrayLength() - if err != nil { - return err - } - - r.TopicStatus[topic] = make(map[int32]*PartitionReplicaReassignmentsStatus, ongoingPartitionReassignments) - - for j := 0; j < ongoingPartitionReassignments; j++ { - partition, err := pd.getInt32() - if err != nil { - return err - } - - block := &PartitionReplicaReassignmentsStatus{} - if err := block.decode(pd); err != nil { - return err - } - r.TopicStatus[topic][partition] = block - } - - if _, err := pd.getEmptyTaggedFieldArray(); err != nil { - return err - } - } - if _, err := pd.getEmptyTaggedFieldArray(); err != nil { - return err - } - - return nil -} - -func (r *ListPartitionReassignmentsResponse) key() int16 { - return 46 -} - -func (r *ListPartitionReassignmentsResponse) version() int16 { - return r.Version -} - -func (r *ListPartitionReassignmentsResponse) headerVersion() int16 { - return 1 -} - -func (r *ListPartitionReassignmentsResponse) requiredVersion() KafkaVersion { - return V2_4_0_0 -} diff --git a/vendor/github.com/Shopify/sarama/message.go b/vendor/github.com/Shopify/sarama/message.go deleted file mode 100644 index e48566b3..00000000 --- a/vendor/github.com/Shopify/sarama/message.go +++ /dev/null @@ -1,174 +0,0 @@ -package sarama - -import ( - "fmt" - "time" -) - -const ( - //CompressionNone no compression - CompressionNone CompressionCodec = iota - //CompressionGZIP compression using GZIP - CompressionGZIP - //CompressionSnappy compression using snappy - CompressionSnappy - //CompressionLZ4 compression using LZ4 - CompressionLZ4 - //CompressionZSTD compression using ZSTD - CompressionZSTD - - // The lowest 3 bits contain the compression codec used for the message - compressionCodecMask int8 = 0x07 - - // Bit 3 set for "LogAppend" timestamps - timestampTypeMask = 0x08 - - // CompressionLevelDefault is the constant to use in CompressionLevel - // to have the default compression level for any codec. The value is picked - // that we don't use any existing compression levels. - CompressionLevelDefault = -1000 -) - -// CompressionCodec represents the various compression codecs recognized by Kafka in messages. -type CompressionCodec int8 - -func (cc CompressionCodec) String() string { - return []string{ - "none", - "gzip", - "snappy", - "lz4", - "zstd", - }[int(cc)] -} - -//Message is a kafka message type -type Message struct { - Codec CompressionCodec // codec used to compress the message contents - CompressionLevel int // compression level - LogAppendTime bool // the used timestamp is LogAppendTime - Key []byte // the message key, may be nil - Value []byte // the message contents - Set *MessageSet // the message set a message might wrap - Version int8 // v1 requires Kafka 0.10 - Timestamp time.Time // the timestamp of the message (version 1+ only) - - compressedCache []byte - compressedSize int // used for computing the compression ratio metrics -} - -func (m *Message) encode(pe packetEncoder) error { - pe.push(newCRC32Field(crcIEEE)) - - pe.putInt8(m.Version) - - attributes := int8(m.Codec) & compressionCodecMask - if m.LogAppendTime { - attributes |= timestampTypeMask - } - pe.putInt8(attributes) - - if m.Version >= 1 { - if err := (Timestamp{&m.Timestamp}).encode(pe); err != nil { - return err - } - } - - err := pe.putBytes(m.Key) - if err != nil { - return err - } - - var payload []byte - - if m.compressedCache != nil { - payload = m.compressedCache - m.compressedCache = nil - } else if m.Value != nil { - payload, err = compress(m.Codec, m.CompressionLevel, m.Value) - if err != nil { - return err - } - m.compressedCache = payload - // Keep in mind the compressed payload size for metric gathering - m.compressedSize = len(payload) - } - - if err = pe.putBytes(payload); err != nil { - return err - } - - return pe.pop() -} - -func (m *Message) decode(pd packetDecoder) (err error) { - crc32Decoder := acquireCrc32Field(crcIEEE) - defer releaseCrc32Field(crc32Decoder) - - err = pd.push(crc32Decoder) - if err != nil { - return err - } - - m.Version, err = pd.getInt8() - if err != nil { - return err - } - - if m.Version > 1 { - return PacketDecodingError{fmt.Sprintf("unknown magic byte (%v)", m.Version)} - } - - attribute, err := pd.getInt8() - if err != nil { - return err - } - m.Codec = CompressionCodec(attribute & compressionCodecMask) - m.LogAppendTime = attribute×tampTypeMask == timestampTypeMask - - if m.Version == 1 { - if err := (Timestamp{&m.Timestamp}).decode(pd); err != nil { - return err - } - } - - m.Key, err = pd.getBytes() - if err != nil { - return err - } - - m.Value, err = pd.getBytes() - if err != nil { - return err - } - - // Required for deep equal assertion during tests but might be useful - // for future metrics about the compression ratio in fetch requests - m.compressedSize = len(m.Value) - - switch m.Codec { - case CompressionNone: - // nothing to do - default: - if m.Value == nil { - break - } - - m.Value, err = decompress(m.Codec, m.Value) - if err != nil { - return err - } - if err := m.decodeSet(); err != nil { - return err - } - } - - return pd.pop() -} - -// decodes a message set from a previously encoded bulk-message -func (m *Message) decodeSet() (err error) { - pd := realDecoder{raw: m.Value} - m.Set = &MessageSet{} - return m.Set.decode(&pd) -} diff --git a/vendor/github.com/Shopify/sarama/message_set.go b/vendor/github.com/Shopify/sarama/message_set.go deleted file mode 100644 index 6523ec2f..00000000 --- a/vendor/github.com/Shopify/sarama/message_set.go +++ /dev/null @@ -1,111 +0,0 @@ -package sarama - -type MessageBlock struct { - Offset int64 - Msg *Message -} - -// Messages convenience helper which returns either all the -// messages that are wrapped in this block -func (msb *MessageBlock) Messages() []*MessageBlock { - if msb.Msg.Set != nil { - return msb.Msg.Set.Messages - } - return []*MessageBlock{msb} -} - -func (msb *MessageBlock) encode(pe packetEncoder) error { - pe.putInt64(msb.Offset) - pe.push(&lengthField{}) - err := msb.Msg.encode(pe) - if err != nil { - return err - } - return pe.pop() -} - -func (msb *MessageBlock) decode(pd packetDecoder) (err error) { - if msb.Offset, err = pd.getInt64(); err != nil { - return err - } - - lengthDecoder := acquireLengthField() - defer releaseLengthField(lengthDecoder) - - if err = pd.push(lengthDecoder); err != nil { - return err - } - - msb.Msg = new(Message) - if err = msb.Msg.decode(pd); err != nil { - return err - } - - if err = pd.pop(); err != nil { - return err - } - - return nil -} - -type MessageSet struct { - PartialTrailingMessage bool // whether the set on the wire contained an incomplete trailing MessageBlock - OverflowMessage bool // whether the set on the wire contained an overflow message - Messages []*MessageBlock -} - -func (ms *MessageSet) encode(pe packetEncoder) error { - for i := range ms.Messages { - err := ms.Messages[i].encode(pe) - if err != nil { - return err - } - } - return nil -} - -func (ms *MessageSet) decode(pd packetDecoder) (err error) { - ms.Messages = nil - - for pd.remaining() > 0 { - magic, err := magicValue(pd) - if err != nil { - if err == ErrInsufficientData { - ms.PartialTrailingMessage = true - return nil - } - return err - } - - if magic > 1 { - return nil - } - - msb := new(MessageBlock) - err = msb.decode(pd) - switch err { - case nil: - ms.Messages = append(ms.Messages, msb) - case ErrInsufficientData: - // As an optimization the server is allowed to return a partial message at the - // end of the message set. Clients should handle this case. So we just ignore such things. - if msb.Offset == -1 { - // This is an overflow message caused by chunked down conversion - ms.OverflowMessage = true - } else { - ms.PartialTrailingMessage = true - } - return nil - default: - return err - } - } - - return nil -} - -func (ms *MessageSet) addMessage(msg *Message) { - block := new(MessageBlock) - block.Msg = msg - ms.Messages = append(ms.Messages, block) -} diff --git a/vendor/github.com/Shopify/sarama/metadata_request.go b/vendor/github.com/Shopify/sarama/metadata_request.go deleted file mode 100644 index e835f5a9..00000000 --- a/vendor/github.com/Shopify/sarama/metadata_request.go +++ /dev/null @@ -1,85 +0,0 @@ -package sarama - -type MetadataRequest struct { - Version int16 - Topics []string - AllowAutoTopicCreation bool -} - -func (r *MetadataRequest) encode(pe packetEncoder) error { - if r.Version < 0 || r.Version > 5 { - return PacketEncodingError{"invalid or unsupported MetadataRequest version field"} - } - if r.Version == 0 || len(r.Topics) > 0 { - err := pe.putArrayLength(len(r.Topics)) - if err != nil { - return err - } - - for i := range r.Topics { - err = pe.putString(r.Topics[i]) - if err != nil { - return err - } - } - } else { - pe.putInt32(-1) - } - if r.Version > 3 { - pe.putBool(r.AllowAutoTopicCreation) - } - return nil -} - -func (r *MetadataRequest) decode(pd packetDecoder, version int16) error { - r.Version = version - size, err := pd.getInt32() - if err != nil { - return err - } - if size > 0 { - r.Topics = make([]string, size) - for i := range r.Topics { - topic, err := pd.getString() - if err != nil { - return err - } - r.Topics[i] = topic - } - } - if r.Version > 3 { - autoCreation, err := pd.getBool() - if err != nil { - return err - } - r.AllowAutoTopicCreation = autoCreation - } - return nil -} - -func (r *MetadataRequest) key() int16 { - return 3 -} - -func (r *MetadataRequest) version() int16 { - return r.Version -} - -func (r *MetadataRequest) headerVersion() int16 { - return 1 -} - -func (r *MetadataRequest) requiredVersion() KafkaVersion { - switch r.Version { - case 1: - return V0_10_0_0 - case 2: - return V0_10_1_0 - case 3, 4: - return V0_11_0_0 - case 5: - return V1_0_0_0 - default: - return MinVersion - } -} diff --git a/vendor/github.com/Shopify/sarama/metadata_response.go b/vendor/github.com/Shopify/sarama/metadata_response.go deleted file mode 100644 index 0bb8702c..00000000 --- a/vendor/github.com/Shopify/sarama/metadata_response.go +++ /dev/null @@ -1,325 +0,0 @@ -package sarama - -type PartitionMetadata struct { - Err KError - ID int32 - Leader int32 - Replicas []int32 - Isr []int32 - OfflineReplicas []int32 -} - -func (pm *PartitionMetadata) decode(pd packetDecoder, version int16) (err error) { - tmp, err := pd.getInt16() - if err != nil { - return err - } - pm.Err = KError(tmp) - - pm.ID, err = pd.getInt32() - if err != nil { - return err - } - - pm.Leader, err = pd.getInt32() - if err != nil { - return err - } - - pm.Replicas, err = pd.getInt32Array() - if err != nil { - return err - } - - pm.Isr, err = pd.getInt32Array() - if err != nil { - return err - } - - if version >= 5 { - pm.OfflineReplicas, err = pd.getInt32Array() - if err != nil { - return err - } - } - - return nil -} - -func (pm *PartitionMetadata) encode(pe packetEncoder, version int16) (err error) { - pe.putInt16(int16(pm.Err)) - pe.putInt32(pm.ID) - pe.putInt32(pm.Leader) - - err = pe.putInt32Array(pm.Replicas) - if err != nil { - return err - } - - err = pe.putInt32Array(pm.Isr) - if err != nil { - return err - } - - if version >= 5 { - err = pe.putInt32Array(pm.OfflineReplicas) - if err != nil { - return err - } - } - - return nil -} - -type TopicMetadata struct { - Err KError - Name string - IsInternal bool // Only valid for Version >= 1 - Partitions []*PartitionMetadata -} - -func (tm *TopicMetadata) decode(pd packetDecoder, version int16) (err error) { - tmp, err := pd.getInt16() - if err != nil { - return err - } - tm.Err = KError(tmp) - - tm.Name, err = pd.getString() - if err != nil { - return err - } - - if version >= 1 { - tm.IsInternal, err = pd.getBool() - if err != nil { - return err - } - } - - n, err := pd.getArrayLength() - if err != nil { - return err - } - tm.Partitions = make([]*PartitionMetadata, n) - for i := 0; i < n; i++ { - tm.Partitions[i] = new(PartitionMetadata) - err = tm.Partitions[i].decode(pd, version) - if err != nil { - return err - } - } - - return nil -} - -func (tm *TopicMetadata) encode(pe packetEncoder, version int16) (err error) { - pe.putInt16(int16(tm.Err)) - - err = pe.putString(tm.Name) - if err != nil { - return err - } - - if version >= 1 { - pe.putBool(tm.IsInternal) - } - - err = pe.putArrayLength(len(tm.Partitions)) - if err != nil { - return err - } - - for _, pm := range tm.Partitions { - err = pm.encode(pe, version) - if err != nil { - return err - } - } - - return nil -} - -type MetadataResponse struct { - Version int16 - ThrottleTimeMs int32 - Brokers []*Broker - ClusterID *string - ControllerID int32 - Topics []*TopicMetadata -} - -func (r *MetadataResponse) decode(pd packetDecoder, version int16) (err error) { - r.Version = version - - if version >= 3 { - r.ThrottleTimeMs, err = pd.getInt32() - if err != nil { - return err - } - } - - n, err := pd.getArrayLength() - if err != nil { - return err - } - - r.Brokers = make([]*Broker, n) - for i := 0; i < n; i++ { - r.Brokers[i] = new(Broker) - err = r.Brokers[i].decode(pd, version) - if err != nil { - return err - } - } - - if version >= 2 { - r.ClusterID, err = pd.getNullableString() - if err != nil { - return err - } - } - - if version >= 1 { - r.ControllerID, err = pd.getInt32() - if err != nil { - return err - } - } else { - r.ControllerID = -1 - } - - n, err = pd.getArrayLength() - if err != nil { - return err - } - - r.Topics = make([]*TopicMetadata, n) - for i := 0; i < n; i++ { - r.Topics[i] = new(TopicMetadata) - err = r.Topics[i].decode(pd, version) - if err != nil { - return err - } - } - - return nil -} - -func (r *MetadataResponse) encode(pe packetEncoder) error { - if r.Version >= 3 { - pe.putInt32(r.ThrottleTimeMs) - } - - err := pe.putArrayLength(len(r.Brokers)) - if err != nil { - return err - } - for _, broker := range r.Brokers { - err = broker.encode(pe, r.Version) - if err != nil { - return err - } - } - - if r.Version >= 2 { - err := pe.putNullableString(r.ClusterID) - if err != nil { - return err - } - } - - if r.Version >= 1 { - pe.putInt32(r.ControllerID) - } - - err = pe.putArrayLength(len(r.Topics)) - if err != nil { - return err - } - for _, tm := range r.Topics { - err = tm.encode(pe, r.Version) - if err != nil { - return err - } - } - - return nil -} - -func (r *MetadataResponse) key() int16 { - return 3 -} - -func (r *MetadataResponse) version() int16 { - return r.Version -} - -func (r *MetadataResponse) headerVersion() int16 { - return 0 -} - -func (r *MetadataResponse) requiredVersion() KafkaVersion { - switch r.Version { - case 1: - return V0_10_0_0 - case 2: - return V0_10_1_0 - case 3, 4: - return V0_11_0_0 - case 5: - return V1_0_0_0 - default: - return MinVersion - } -} - -// testing API - -func (r *MetadataResponse) AddBroker(addr string, id int32) { - r.Brokers = append(r.Brokers, &Broker{id: id, addr: addr}) -} - -func (r *MetadataResponse) AddTopic(topic string, err KError) *TopicMetadata { - var tmatch *TopicMetadata - - for _, tm := range r.Topics { - if tm.Name == topic { - tmatch = tm - goto foundTopic - } - } - - tmatch = new(TopicMetadata) - tmatch.Name = topic - r.Topics = append(r.Topics, tmatch) - -foundTopic: - - tmatch.Err = err - return tmatch -} - -func (r *MetadataResponse) AddTopicPartition(topic string, partition, brokerID int32, replicas, isr []int32, offline []int32, err KError) { - tmatch := r.AddTopic(topic, ErrNoError) - var pmatch *PartitionMetadata - - for _, pm := range tmatch.Partitions { - if pm.ID == partition { - pmatch = pm - goto foundPartition - } - } - - pmatch = new(PartitionMetadata) - pmatch.ID = partition - tmatch.Partitions = append(tmatch.Partitions, pmatch) - -foundPartition: - - pmatch.Leader = brokerID - pmatch.Replicas = replicas - pmatch.Isr = isr - pmatch.OfflineReplicas = offline - pmatch.Err = err -} diff --git a/vendor/github.com/Shopify/sarama/metrics.go b/vendor/github.com/Shopify/sarama/metrics.go deleted file mode 100644 index 90e5a87f..00000000 --- a/vendor/github.com/Shopify/sarama/metrics.go +++ /dev/null @@ -1,43 +0,0 @@ -package sarama - -import ( - "fmt" - "strings" - - "github.com/rcrowley/go-metrics" -) - -// Use exponentially decaying reservoir for sampling histograms with the same defaults as the Java library: -// 1028 elements, which offers a 99.9% confidence level with a 5% margin of error assuming a normal distribution, -// and an alpha factor of 0.015, which heavily biases the reservoir to the past 5 minutes of measurements. -// See https://github.com/dropwizard/metrics/blob/v3.1.0/metrics-core/src/main/java/com/codahale/metrics/ExponentiallyDecayingReservoir.java#L38 -const ( - metricsReservoirSize = 1028 - metricsAlphaFactor = 0.015 -) - -func getOrRegisterHistogram(name string, r metrics.Registry) metrics.Histogram { - return r.GetOrRegister(name, func() metrics.Histogram { - return metrics.NewHistogram(metrics.NewExpDecaySample(metricsReservoirSize, metricsAlphaFactor)) - }).(metrics.Histogram) -} - -func getMetricNameForBroker(name string, broker *Broker) string { - // Use broker id like the Java client as it does not contain '.' or ':' characters that - // can be interpreted as special character by monitoring tool (e.g. Graphite) - return fmt.Sprintf(name+"-for-broker-%d", broker.ID()) -} - -func getMetricNameForTopic(name string, topic string) string { - // Convert dot to _ since reporters like Graphite typically use dot to represent hierarchy - // cf. KAFKA-1902 and KAFKA-2337 - return fmt.Sprintf(name+"-for-topic-%s", strings.Replace(topic, ".", "_", -1)) -} - -func getOrRegisterTopicMeter(name string, topic string, r metrics.Registry) metrics.Meter { - return metrics.GetOrRegisterMeter(getMetricNameForTopic(name, topic), r) -} - -func getOrRegisterTopicHistogram(name string, topic string, r metrics.Registry) metrics.Histogram { - return getOrRegisterHistogram(getMetricNameForTopic(name, topic), r) -} diff --git a/vendor/github.com/Shopify/sarama/mockbroker.go b/vendor/github.com/Shopify/sarama/mockbroker.go deleted file mode 100644 index ff5a68ae..00000000 --- a/vendor/github.com/Shopify/sarama/mockbroker.go +++ /dev/null @@ -1,415 +0,0 @@ -package sarama - -import ( - "bytes" - "encoding/binary" - "fmt" - "io" - "net" - "reflect" - "strconv" - "sync" - "time" - - "github.com/davecgh/go-spew/spew" -) - -const ( - expectationTimeout = 500 * time.Millisecond -) - -type GSSApiHandlerFunc func([]byte) []byte - -type requestHandlerFunc func(req *request) (res encoderWithHeader) - -// RequestNotifierFunc is invoked when a mock broker processes a request successfully -// and will provides the number of bytes read and written. -type RequestNotifierFunc func(bytesRead, bytesWritten int) - -// MockBroker is a mock Kafka broker that is used in unit tests. It is exposed -// to facilitate testing of higher level or specialized consumers and producers -// built on top of Sarama. Note that it does not 'mimic' the Kafka API protocol, -// but rather provides a facility to do that. It takes care of the TCP -// transport, request unmarshaling, response marshaling, and makes it the test -// writer responsibility to program correct according to the Kafka API protocol -// MockBroker behaviour. -// -// MockBroker is implemented as a TCP server listening on a kernel-selected -// localhost port that can accept many connections. It reads Kafka requests -// from that connection and returns responses programmed by the SetHandlerByMap -// function. If a MockBroker receives a request that it has no programmed -// response for, then it returns nothing and the request times out. -// -// A set of MockRequest builders to define mappings used by MockBroker is -// provided by Sarama. But users can develop MockRequests of their own and use -// them along with or instead of the standard ones. -// -// When running tests with MockBroker it is strongly recommended to specify -// a timeout to `go test` so that if the broker hangs waiting for a response, -// the test panics. -// -// It is not necessary to prefix message length or correlation ID to your -// response bytes, the server does that automatically as a convenience. -type MockBroker struct { - brokerID int32 - port int32 - closing chan none - stopper chan none - expectations chan encoderWithHeader - listener net.Listener - t TestReporter - latency time.Duration - handler requestHandlerFunc - notifier RequestNotifierFunc - history []RequestResponse - lock sync.Mutex - gssApiHandler GSSApiHandlerFunc -} - -// RequestResponse represents a Request/Response pair processed by MockBroker. -type RequestResponse struct { - Request protocolBody - Response encoder -} - -// SetLatency makes broker pause for the specified period every time before -// replying. -func (b *MockBroker) SetLatency(latency time.Duration) { - b.latency = latency -} - -// SetHandlerByMap defines mapping of Request types to MockResponses. When a -// request is received by the broker, it looks up the request type in the map -// and uses the found MockResponse instance to generate an appropriate reply. -// If the request type is not found in the map then nothing is sent. -func (b *MockBroker) SetHandlerByMap(handlerMap map[string]MockResponse) { - b.setHandler(func(req *request) (res encoderWithHeader) { - reqTypeName := reflect.TypeOf(req.body).Elem().Name() - mockResponse := handlerMap[reqTypeName] - if mockResponse == nil { - return nil - } - return mockResponse.For(req.body) - }) -} - -// SetNotifier set a function that will get invoked whenever a request has been -// processed successfully and will provide the number of bytes read and written -func (b *MockBroker) SetNotifier(notifier RequestNotifierFunc) { - b.lock.Lock() - b.notifier = notifier - b.lock.Unlock() -} - -// BrokerID returns broker ID assigned to the broker. -func (b *MockBroker) BrokerID() int32 { - return b.brokerID -} - -// History returns a slice of RequestResponse pairs in the order they were -// processed by the broker. Note that in case of multiple connections to the -// broker the order expected by a test can be different from the order recorded -// in the history, unless some synchronization is implemented in the test. -func (b *MockBroker) History() []RequestResponse { - b.lock.Lock() - history := make([]RequestResponse, len(b.history)) - copy(history, b.history) - b.lock.Unlock() - return history -} - -// Port returns the TCP port number the broker is listening for requests on. -func (b *MockBroker) Port() int32 { - return b.port -} - -// Addr returns the broker connection string in the form "
:". -func (b *MockBroker) Addr() string { - return b.listener.Addr().String() -} - -// Close terminates the broker blocking until it stops internal goroutines and -// releases all resources. -func (b *MockBroker) Close() { - close(b.expectations) - if len(b.expectations) > 0 { - buf := bytes.NewBufferString(fmt.Sprintf("mockbroker/%d: not all expectations were satisfied! Still waiting on:\n", b.BrokerID())) - for e := range b.expectations { - _, _ = buf.WriteString(spew.Sdump(e)) - } - b.t.Error(buf.String()) - } - close(b.closing) - <-b.stopper -} - -// setHandler sets the specified function as the request handler. Whenever -// a mock broker reads a request from the wire it passes the request to the -// function and sends back whatever the handler function returns. -func (b *MockBroker) setHandler(handler requestHandlerFunc) { - b.lock.Lock() - b.handler = handler - b.lock.Unlock() -} - -func (b *MockBroker) serverLoop() { - defer close(b.stopper) - var err error - var conn net.Conn - - go func() { - <-b.closing - err := b.listener.Close() - if err != nil { - b.t.Error(err) - } - }() - - wg := &sync.WaitGroup{} - i := 0 - for conn, err = b.listener.Accept(); err == nil; conn, err = b.listener.Accept() { - wg.Add(1) - go b.handleRequests(conn, i, wg) - i++ - } - wg.Wait() - Logger.Printf("*** mockbroker/%d: listener closed, err=%v", b.BrokerID(), err) -} - -func (b *MockBroker) SetGSSAPIHandler(handler GSSApiHandlerFunc) { - b.gssApiHandler = handler -} - -func (b *MockBroker) readToBytes(r io.Reader) ([]byte, error) { - var ( - bytesRead int - lengthBytes = make([]byte, 4) - ) - - if _, err := io.ReadFull(r, lengthBytes); err != nil { - return nil, err - } - - bytesRead += len(lengthBytes) - length := int32(binary.BigEndian.Uint32(lengthBytes)) - - if length <= 4 || length > MaxRequestSize { - return nil, PacketDecodingError{fmt.Sprintf("message of length %d too large or too small", length)} - } - - encodedReq := make([]byte, length) - if _, err := io.ReadFull(r, encodedReq); err != nil { - return nil, err - } - - bytesRead += len(encodedReq) - - fullBytes := append(lengthBytes, encodedReq...) - - return fullBytes, nil -} - -func (b *MockBroker) isGSSAPI(buffer []byte) bool { - return buffer[4] == 0x60 || bytes.Equal(buffer[4:6], []byte{0x05, 0x04}) -} - -func (b *MockBroker) handleRequests(conn io.ReadWriteCloser, idx int, wg *sync.WaitGroup) { - defer wg.Done() - defer func() { - _ = conn.Close() - }() - Logger.Printf("*** mockbroker/%d/%d: connection opened", b.BrokerID(), idx) - var err error - - abort := make(chan none) - defer close(abort) - go func() { - select { - case <-b.closing: - _ = conn.Close() - case <-abort: - } - }() - - var bytesWritten int - var bytesRead int - for { - buffer, err := b.readToBytes(conn) - if err != nil { - Logger.Printf("*** mockbroker/%d/%d: invalid request: err=%+v, %+v", b.brokerID, idx, err, spew.Sdump(buffer)) - b.serverError(err) - break - } - - bytesWritten = 0 - if !b.isGSSAPI(buffer) { - req, br, err := decodeRequest(bytes.NewReader(buffer)) - bytesRead = br - if err != nil { - Logger.Printf("*** mockbroker/%d/%d: invalid request: err=%+v, %+v", b.brokerID, idx, err, spew.Sdump(req)) - b.serverError(err) - break - } - - if b.latency > 0 { - time.Sleep(b.latency) - } - - b.lock.Lock() - res := b.handler(req) - b.history = append(b.history, RequestResponse{req.body, res}) - b.lock.Unlock() - - if res == nil { - Logger.Printf("*** mockbroker/%d/%d: ignored %v", b.brokerID, idx, spew.Sdump(req)) - continue - } - Logger.Printf("*** mockbroker/%d/%d: served %v -> %v", b.brokerID, idx, req, res) - - encodedRes, err := encode(res, nil) - if err != nil { - b.serverError(err) - break - } - if len(encodedRes) == 0 { - b.lock.Lock() - if b.notifier != nil { - b.notifier(bytesRead, 0) - } - b.lock.Unlock() - continue - } - - resHeader := b.encodeHeader(res.headerVersion(), req.correlationID, uint32(len(encodedRes))) - if _, err = conn.Write(resHeader); err != nil { - b.serverError(err) - break - } - if _, err = conn.Write(encodedRes); err != nil { - b.serverError(err) - break - } - bytesWritten = len(resHeader) + len(encodedRes) - } else { - // GSSAPI is not part of kafka protocol, but is supported for authentication proposes. - // Don't support history for this kind of request as is only used for test GSSAPI authentication mechanism - b.lock.Lock() - res := b.gssApiHandler(buffer) - b.lock.Unlock() - if res == nil { - Logger.Printf("*** mockbroker/%d/%d: ignored %v", b.brokerID, idx, spew.Sdump(buffer)) - continue - } - if _, err = conn.Write(res); err != nil { - b.serverError(err) - break - } - bytesWritten = len(res) - } - - b.lock.Lock() - if b.notifier != nil { - b.notifier(bytesRead, bytesWritten) - } - b.lock.Unlock() - } - Logger.Printf("*** mockbroker/%d/%d: connection closed, err=%v", b.BrokerID(), idx, err) -} - -func (b *MockBroker) encodeHeader(headerVersion int16, correlationId int32, payloadLength uint32) []byte { - headerLength := uint32(8) - - if headerVersion >= 1 { - headerLength = 9 - } - - resHeader := make([]byte, headerLength) - binary.BigEndian.PutUint32(resHeader, payloadLength+headerLength-4) - binary.BigEndian.PutUint32(resHeader[4:], uint32(correlationId)) - - if headerVersion >= 1 { - binary.PutUvarint(resHeader[8:], 0) - } - - return resHeader -} - -func (b *MockBroker) defaultRequestHandler(req *request) (res encoderWithHeader) { - select { - case res, ok := <-b.expectations: - if !ok { - return nil - } - return res - case <-time.After(expectationTimeout): - return nil - } -} - -func (b *MockBroker) serverError(err error) { - isConnectionClosedError := false - if _, ok := err.(*net.OpError); ok { - isConnectionClosedError = true - } else if err == io.EOF { - isConnectionClosedError = true - } else if err.Error() == "use of closed network connection" { - isConnectionClosedError = true - } - - if isConnectionClosedError { - return - } - - b.t.Errorf(err.Error()) -} - -// NewMockBroker launches a fake Kafka broker. It takes a TestReporter as provided by the -// test framework and a channel of responses to use. If an error occurs it is -// simply logged to the TestReporter and the broker exits. -func NewMockBroker(t TestReporter, brokerID int32) *MockBroker { - return NewMockBrokerAddr(t, brokerID, "localhost:0") -} - -// NewMockBrokerAddr behaves like newMockBroker but listens on the address you give -// it rather than just some ephemeral port. -func NewMockBrokerAddr(t TestReporter, brokerID int32, addr string) *MockBroker { - listener, err := net.Listen("tcp", addr) - if err != nil { - t.Fatal(err) - } - return NewMockBrokerListener(t, brokerID, listener) -} - -// NewMockBrokerListener behaves like newMockBrokerAddr but accepts connections on the listener specified. -func NewMockBrokerListener(t TestReporter, brokerID int32, listener net.Listener) *MockBroker { - var err error - - broker := &MockBroker{ - closing: make(chan none), - stopper: make(chan none), - t: t, - brokerID: brokerID, - expectations: make(chan encoderWithHeader, 512), - listener: listener, - } - broker.handler = broker.defaultRequestHandler - - Logger.Printf("*** mockbroker/%d listening on %s\n", brokerID, broker.listener.Addr().String()) - _, portStr, err := net.SplitHostPort(broker.listener.Addr().String()) - if err != nil { - t.Fatal(err) - } - tmp, err := strconv.ParseInt(portStr, 10, 32) - if err != nil { - t.Fatal(err) - } - broker.port = int32(tmp) - - go broker.serverLoop() - - return broker -} - -func (b *MockBroker) Returns(e encoderWithHeader) { - b.expectations <- e -} diff --git a/vendor/github.com/Shopify/sarama/mockkerberos.go b/vendor/github.com/Shopify/sarama/mockkerberos.go deleted file mode 100644 index d36649d8..00000000 --- a/vendor/github.com/Shopify/sarama/mockkerberos.go +++ /dev/null @@ -1,123 +0,0 @@ -package sarama - -import ( - "encoding/binary" - "encoding/hex" - - "gopkg.in/jcmturner/gokrb5.v7/credentials" - "gopkg.in/jcmturner/gokrb5.v7/gssapi" - "gopkg.in/jcmturner/gokrb5.v7/iana/keyusage" - "gopkg.in/jcmturner/gokrb5.v7/messages" - "gopkg.in/jcmturner/gokrb5.v7/types" -) - -type KafkaGSSAPIHandler struct { - client *MockKerberosClient - badResponse bool - badKeyChecksum bool -} - -func (h *KafkaGSSAPIHandler) MockKafkaGSSAPI(buffer []byte) []byte { - // Default payload used for verify - err := h.client.Login() // Mock client construct keys when login - if err != nil { - return nil - } - if h.badResponse { // Returns trash - return []byte{0x00, 0x00, 0x00, 0x01, 0xAD} - } - - var pack = gssapi.WrapToken{ - Flags: KRB5_USER_AUTH, - EC: 12, - RRC: 0, - SndSeqNum: 3398292281, - Payload: []byte{0x11, 0x00}, // 1100 - } - // Compute checksum - if h.badKeyChecksum { - pack.CheckSum = []byte{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF} - } else { - err = pack.SetCheckSum(h.client.ASRep.DecryptedEncPart.Key, keyusage.GSSAPI_ACCEPTOR_SEAL) - if err != nil { - return nil - } - } - - packBytes, err := pack.Marshal() - if err != nil { - return nil - } - lenBytes := len(packBytes) - response := make([]byte, lenBytes+4) - copy(response[4:], packBytes) - binary.BigEndian.PutUint32(response, uint32(lenBytes)) - return response -} - -type MockKerberosClient struct { - asRepBytes string - ASRep messages.ASRep - credentials *credentials.Credentials - mockError error - errorStage string -} - -func (c *MockKerberosClient) Login() error { - if c.errorStage == "login" && c.mockError != nil { - return c.mockError - } - c.asRepBytes = "6b8202e9308202e5a003020105a10302010ba22b30293027a103020113a220041e301c301aa003020112a1131b114" + - "558414d504c452e434f4d636c69656e74a30d1b0b4558414d504c452e434f4da4133011a003020101a10a30081b06636c69656e7" + - "4a5820156618201523082014ea003020105a10d1b0b4558414d504c452e434f4da220301ea003020102a11730151b066b7262746" + - "7741b0b4558414d504c452e434f4da382011430820110a003020112a103020101a28201020481ffdb9891175d106818e61008c51" + - "d0b3462bca92f3bf9d4cfa82de4c4d7aff9994ec87c573e3a3d54dcb2bb79618c76f2bf4a3d006f90d5bdbd049bc18f48be39203" + - "549ca02acaf63f292b12404f9b74c34b83687119d8f56552ccc0c50ebee2a53bb114c1b4619bb1d5d31f0f49b4d40a08a9b4c046" + - "2e1398d0b648be1c0e50c552ad16e1d8d8e74263dd0bf0ec591e4797dfd40a9a1be4ae830d03a306e053fd7586fef84ffc5e4a83" + - "7c3122bf3e6a40fe87e84019f6283634461b955712b44a5f7386c278bff94ec2c2dc0403247e29c2450e853471ceababf9b8911f" + - "997f2e3010b046d2c49eb438afb0f4c210821e80d4ffa4c9521eb895dcd68610b3feaa682012c30820128a003020112a282011f0" + - "482011bce73cbce3f1dd17661c412005f0f2257c756fe8e98ff97e6ec24b7bab66e5fd3a3827aeeae4757af0c6e892948122d8b2" + - "03c8df48df0ef5d142d0e416d688f11daa0fcd63d96bdd431d02b8e951c664eeff286a2be62383d274a04016d5f0e141da58cb86" + - "331de64063062f4f885e8e9ce5b181ca2fdc67897c5995e0ae1ae0c171a64493ff7bd91bc6d89cd4fce1e2b3ea0a10e34b0d5eda" + - "aa38ee727b50c5632ed1d2f2b457908e616178d0d80b72af209fb8ac9dbaa1768fa45931392b36b6d8c12400f8ded2efaa0654d0" + - "da1db966e8b5aab4706c800f95d559664646041fdb38b411c62fc0fbe0d25083a28562b0e1c8df16e62e9d5626b0addee489835f" + - "eedb0f26c05baa596b69b17f47920aa64b29dc77cfcc97ba47885" - apRepBytes, err := hex.DecodeString(c.asRepBytes) - if err != nil { - return err - } - err = c.ASRep.Unmarshal(apRepBytes) - if err != nil { - return err - } - c.credentials = credentials.New("client", "EXAMPLE.COM").WithPassword("qwerty") - _, err = c.ASRep.DecryptEncPart(c.credentials) - if err != nil { - return err - } - return nil -} - -func (c *MockKerberosClient) GetServiceTicket(spn string) (messages.Ticket, types.EncryptionKey, error) { - if c.errorStage == "service_ticket" && c.mockError != nil { - return messages.Ticket{}, types.EncryptionKey{}, c.mockError - } - return c.ASRep.Ticket, c.ASRep.DecryptedEncPart.Key, nil -} - -func (c *MockKerberosClient) Domain() string { - return "EXAMPLE.COM" -} -func (c *MockKerberosClient) CName() types.PrincipalName { - var p = types.PrincipalName{ - NameType: KRB5_USER_AUTH, - NameString: []string{ - "kafka", - "kafka", - }, - } - return p -} -func (c *MockKerberosClient) Destroy() { - // Do nothing. -} diff --git a/vendor/github.com/Shopify/sarama/mockresponses.go b/vendor/github.com/Shopify/sarama/mockresponses.go deleted file mode 100644 index be8af854..00000000 --- a/vendor/github.com/Shopify/sarama/mockresponses.go +++ /dev/null @@ -1,1121 +0,0 @@ -package sarama - -import ( - "fmt" - "strings" -) - -// TestReporter has methods matching go's testing.T to avoid importing -// `testing` in the main part of the library. -type TestReporter interface { - Error(...interface{}) - Errorf(string, ...interface{}) - Fatal(...interface{}) - Fatalf(string, ...interface{}) -} - -// MockResponse is a response builder interface it defines one method that -// allows generating a response based on a request body. MockResponses are used -// to program behavior of MockBroker in tests. -type MockResponse interface { - For(reqBody versionedDecoder) (res encoderWithHeader) -} - -// MockWrapper is a mock response builder that returns a particular concrete -// response regardless of the actual request passed to the `For` method. -type MockWrapper struct { - res encoderWithHeader -} - -func (mw *MockWrapper) For(reqBody versionedDecoder) (res encoderWithHeader) { - return mw.res -} - -func NewMockWrapper(res encoderWithHeader) *MockWrapper { - return &MockWrapper{res: res} -} - -// MockSequence is a mock response builder that is created from a sequence of -// concrete responses. Every time when a `MockBroker` calls its `For` method -// the next response from the sequence is returned. When the end of the -// sequence is reached the last element from the sequence is returned. -type MockSequence struct { - responses []MockResponse -} - -func NewMockSequence(responses ...interface{}) *MockSequence { - ms := &MockSequence{} - ms.responses = make([]MockResponse, len(responses)) - for i, res := range responses { - switch res := res.(type) { - case MockResponse: - ms.responses[i] = res - case encoderWithHeader: - ms.responses[i] = NewMockWrapper(res) - default: - panic(fmt.Sprintf("Unexpected response type: %T", res)) - } - } - return ms -} - -func (mc *MockSequence) For(reqBody versionedDecoder) (res encoderWithHeader) { - res = mc.responses[0].For(reqBody) - if len(mc.responses) > 1 { - mc.responses = mc.responses[1:] - } - return res -} - -type MockListGroupsResponse struct { - groups map[string]string - t TestReporter -} - -func NewMockListGroupsResponse(t TestReporter) *MockListGroupsResponse { - return &MockListGroupsResponse{ - groups: make(map[string]string), - t: t, - } -} - -func (m *MockListGroupsResponse) For(reqBody versionedDecoder) encoderWithHeader { - request := reqBody.(*ListGroupsRequest) - _ = request - response := &ListGroupsResponse{ - Groups: m.groups, - } - return response -} - -func (m *MockListGroupsResponse) AddGroup(groupID, protocolType string) *MockListGroupsResponse { - m.groups[groupID] = protocolType - return m -} - -type MockDescribeGroupsResponse struct { - groups map[string]*GroupDescription - t TestReporter -} - -func NewMockDescribeGroupsResponse(t TestReporter) *MockDescribeGroupsResponse { - return &MockDescribeGroupsResponse{ - t: t, - groups: make(map[string]*GroupDescription), - } -} - -func (m *MockDescribeGroupsResponse) AddGroupDescription(groupID string, description *GroupDescription) *MockDescribeGroupsResponse { - m.groups[groupID] = description - return m -} - -func (m *MockDescribeGroupsResponse) For(reqBody versionedDecoder) encoderWithHeader { - request := reqBody.(*DescribeGroupsRequest) - - response := &DescribeGroupsResponse{} - for _, requestedGroup := range request.Groups { - if group, ok := m.groups[requestedGroup]; ok { - response.Groups = append(response.Groups, group) - } else { - // Mimic real kafka - if a group doesn't exist, return - // an entry with state "Dead" - response.Groups = append(response.Groups, &GroupDescription{ - GroupId: requestedGroup, - State: "Dead", - }) - } - } - - return response -} - -// MockMetadataResponse is a `MetadataResponse` builder. -type MockMetadataResponse struct { - controllerID int32 - leaders map[string]map[int32]int32 - brokers map[string]int32 - t TestReporter -} - -func NewMockMetadataResponse(t TestReporter) *MockMetadataResponse { - return &MockMetadataResponse{ - leaders: make(map[string]map[int32]int32), - brokers: make(map[string]int32), - t: t, - } -} - -func (mmr *MockMetadataResponse) SetLeader(topic string, partition, brokerID int32) *MockMetadataResponse { - partitions := mmr.leaders[topic] - if partitions == nil { - partitions = make(map[int32]int32) - mmr.leaders[topic] = partitions - } - partitions[partition] = brokerID - return mmr -} - -func (mmr *MockMetadataResponse) SetBroker(addr string, brokerID int32) *MockMetadataResponse { - mmr.brokers[addr] = brokerID - return mmr -} - -func (mmr *MockMetadataResponse) SetController(brokerID int32) *MockMetadataResponse { - mmr.controllerID = brokerID - return mmr -} - -func (mmr *MockMetadataResponse) For(reqBody versionedDecoder) encoderWithHeader { - metadataRequest := reqBody.(*MetadataRequest) - metadataResponse := &MetadataResponse{ - Version: metadataRequest.version(), - ControllerID: mmr.controllerID, - } - for addr, brokerID := range mmr.brokers { - metadataResponse.AddBroker(addr, brokerID) - } - - // Generate set of replicas - var replicas []int32 - var offlineReplicas []int32 - for _, brokerID := range mmr.brokers { - replicas = append(replicas, brokerID) - } - - if len(metadataRequest.Topics) == 0 { - for topic, partitions := range mmr.leaders { - for partition, brokerID := range partitions { - metadataResponse.AddTopicPartition(topic, partition, brokerID, replicas, replicas, offlineReplicas, ErrNoError) - } - } - return metadataResponse - } - for _, topic := range metadataRequest.Topics { - for partition, brokerID := range mmr.leaders[topic] { - metadataResponse.AddTopicPartition(topic, partition, brokerID, replicas, replicas, offlineReplicas, ErrNoError) - } - } - return metadataResponse -} - -// MockOffsetResponse is an `OffsetResponse` builder. -type MockOffsetResponse struct { - offsets map[string]map[int32]map[int64]int64 - t TestReporter - version int16 -} - -func NewMockOffsetResponse(t TestReporter) *MockOffsetResponse { - return &MockOffsetResponse{ - offsets: make(map[string]map[int32]map[int64]int64), - t: t, - } -} - -func (mor *MockOffsetResponse) SetVersion(version int16) *MockOffsetResponse { - mor.version = version - return mor -} - -func (mor *MockOffsetResponse) SetOffset(topic string, partition int32, time, offset int64) *MockOffsetResponse { - partitions := mor.offsets[topic] - if partitions == nil { - partitions = make(map[int32]map[int64]int64) - mor.offsets[topic] = partitions - } - times := partitions[partition] - if times == nil { - times = make(map[int64]int64) - partitions[partition] = times - } - times[time] = offset - return mor -} - -func (mor *MockOffsetResponse) For(reqBody versionedDecoder) encoderWithHeader { - offsetRequest := reqBody.(*OffsetRequest) - offsetResponse := &OffsetResponse{Version: mor.version} - for topic, partitions := range offsetRequest.blocks { - for partition, block := range partitions { - offset := mor.getOffset(topic, partition, block.time) - offsetResponse.AddTopicPartition(topic, partition, offset) - } - } - return offsetResponse -} - -func (mor *MockOffsetResponse) getOffset(topic string, partition int32, time int64) int64 { - partitions := mor.offsets[topic] - if partitions == nil { - mor.t.Errorf("missing topic: %s", topic) - } - times := partitions[partition] - if times == nil { - mor.t.Errorf("missing partition: %d", partition) - } - offset, ok := times[time] - if !ok { - mor.t.Errorf("missing time: %d", time) - } - return offset -} - -// MockFetchResponse is a `FetchResponse` builder. -type MockFetchResponse struct { - messages map[string]map[int32]map[int64]Encoder - highWaterMarks map[string]map[int32]int64 - t TestReporter - batchSize int - version int16 -} - -func NewMockFetchResponse(t TestReporter, batchSize int) *MockFetchResponse { - return &MockFetchResponse{ - messages: make(map[string]map[int32]map[int64]Encoder), - highWaterMarks: make(map[string]map[int32]int64), - t: t, - batchSize: batchSize, - } -} - -func (mfr *MockFetchResponse) SetVersion(version int16) *MockFetchResponse { - mfr.version = version - return mfr -} - -func (mfr *MockFetchResponse) SetMessage(topic string, partition int32, offset int64, msg Encoder) *MockFetchResponse { - partitions := mfr.messages[topic] - if partitions == nil { - partitions = make(map[int32]map[int64]Encoder) - mfr.messages[topic] = partitions - } - messages := partitions[partition] - if messages == nil { - messages = make(map[int64]Encoder) - partitions[partition] = messages - } - messages[offset] = msg - return mfr -} - -func (mfr *MockFetchResponse) SetHighWaterMark(topic string, partition int32, offset int64) *MockFetchResponse { - partitions := mfr.highWaterMarks[topic] - if partitions == nil { - partitions = make(map[int32]int64) - mfr.highWaterMarks[topic] = partitions - } - partitions[partition] = offset - return mfr -} - -func (mfr *MockFetchResponse) For(reqBody versionedDecoder) encoderWithHeader { - fetchRequest := reqBody.(*FetchRequest) - res := &FetchResponse{ - Version: mfr.version, - } - for topic, partitions := range fetchRequest.blocks { - for partition, block := range partitions { - initialOffset := block.fetchOffset - offset := initialOffset - maxOffset := initialOffset + int64(mfr.getMessageCount(topic, partition)) - for i := 0; i < mfr.batchSize && offset < maxOffset; { - msg := mfr.getMessage(topic, partition, offset) - if msg != nil { - res.AddMessage(topic, partition, nil, msg, offset) - i++ - } - offset++ - } - fb := res.GetBlock(topic, partition) - if fb == nil { - res.AddError(topic, partition, ErrNoError) - fb = res.GetBlock(topic, partition) - } - fb.HighWaterMarkOffset = mfr.getHighWaterMark(topic, partition) - } - } - return res -} - -func (mfr *MockFetchResponse) getMessage(topic string, partition int32, offset int64) Encoder { - partitions := mfr.messages[topic] - if partitions == nil { - return nil - } - messages := partitions[partition] - if messages == nil { - return nil - } - return messages[offset] -} - -func (mfr *MockFetchResponse) getMessageCount(topic string, partition int32) int { - partitions := mfr.messages[topic] - if partitions == nil { - return 0 - } - messages := partitions[partition] - if messages == nil { - return 0 - } - return len(messages) -} - -func (mfr *MockFetchResponse) getHighWaterMark(topic string, partition int32) int64 { - partitions := mfr.highWaterMarks[topic] - if partitions == nil { - return 0 - } - return partitions[partition] -} - -// MockConsumerMetadataResponse is a `ConsumerMetadataResponse` builder. -type MockConsumerMetadataResponse struct { - coordinators map[string]interface{} - t TestReporter -} - -func NewMockConsumerMetadataResponse(t TestReporter) *MockConsumerMetadataResponse { - return &MockConsumerMetadataResponse{ - coordinators: make(map[string]interface{}), - t: t, - } -} - -func (mr *MockConsumerMetadataResponse) SetCoordinator(group string, broker *MockBroker) *MockConsumerMetadataResponse { - mr.coordinators[group] = broker - return mr -} - -func (mr *MockConsumerMetadataResponse) SetError(group string, kerror KError) *MockConsumerMetadataResponse { - mr.coordinators[group] = kerror - return mr -} - -func (mr *MockConsumerMetadataResponse) For(reqBody versionedDecoder) encoderWithHeader { - req := reqBody.(*ConsumerMetadataRequest) - group := req.ConsumerGroup - res := &ConsumerMetadataResponse{} - v := mr.coordinators[group] - switch v := v.(type) { - case *MockBroker: - res.Coordinator = &Broker{id: v.BrokerID(), addr: v.Addr()} - case KError: - res.Err = v - } - return res -} - -// MockFindCoordinatorResponse is a `FindCoordinatorResponse` builder. -type MockFindCoordinatorResponse struct { - groupCoordinators map[string]interface{} - transCoordinators map[string]interface{} - t TestReporter -} - -func NewMockFindCoordinatorResponse(t TestReporter) *MockFindCoordinatorResponse { - return &MockFindCoordinatorResponse{ - groupCoordinators: make(map[string]interface{}), - transCoordinators: make(map[string]interface{}), - t: t, - } -} - -func (mr *MockFindCoordinatorResponse) SetCoordinator(coordinatorType CoordinatorType, group string, broker *MockBroker) *MockFindCoordinatorResponse { - switch coordinatorType { - case CoordinatorGroup: - mr.groupCoordinators[group] = broker - case CoordinatorTransaction: - mr.transCoordinators[group] = broker - } - return mr -} - -func (mr *MockFindCoordinatorResponse) SetError(coordinatorType CoordinatorType, group string, kerror KError) *MockFindCoordinatorResponse { - switch coordinatorType { - case CoordinatorGroup: - mr.groupCoordinators[group] = kerror - case CoordinatorTransaction: - mr.transCoordinators[group] = kerror - } - return mr -} - -func (mr *MockFindCoordinatorResponse) For(reqBody versionedDecoder) encoderWithHeader { - req := reqBody.(*FindCoordinatorRequest) - res := &FindCoordinatorResponse{} - var v interface{} - switch req.CoordinatorType { - case CoordinatorGroup: - v = mr.groupCoordinators[req.CoordinatorKey] - case CoordinatorTransaction: - v = mr.transCoordinators[req.CoordinatorKey] - } - switch v := v.(type) { - case *MockBroker: - res.Coordinator = &Broker{id: v.BrokerID(), addr: v.Addr()} - case KError: - res.Err = v - } - return res -} - -// MockOffsetCommitResponse is a `OffsetCommitResponse` builder. -type MockOffsetCommitResponse struct { - errors map[string]map[string]map[int32]KError - t TestReporter -} - -func NewMockOffsetCommitResponse(t TestReporter) *MockOffsetCommitResponse { - return &MockOffsetCommitResponse{t: t} -} - -func (mr *MockOffsetCommitResponse) SetError(group, topic string, partition int32, kerror KError) *MockOffsetCommitResponse { - if mr.errors == nil { - mr.errors = make(map[string]map[string]map[int32]KError) - } - topics := mr.errors[group] - if topics == nil { - topics = make(map[string]map[int32]KError) - mr.errors[group] = topics - } - partitions := topics[topic] - if partitions == nil { - partitions = make(map[int32]KError) - topics[topic] = partitions - } - partitions[partition] = kerror - return mr -} - -func (mr *MockOffsetCommitResponse) For(reqBody versionedDecoder) encoderWithHeader { - req := reqBody.(*OffsetCommitRequest) - group := req.ConsumerGroup - res := &OffsetCommitResponse{} - for topic, partitions := range req.blocks { - for partition := range partitions { - res.AddError(topic, partition, mr.getError(group, topic, partition)) - } - } - return res -} - -func (mr *MockOffsetCommitResponse) getError(group, topic string, partition int32) KError { - topics := mr.errors[group] - if topics == nil { - return ErrNoError - } - partitions := topics[topic] - if partitions == nil { - return ErrNoError - } - kerror, ok := partitions[partition] - if !ok { - return ErrNoError - } - return kerror -} - -// MockProduceResponse is a `ProduceResponse` builder. -type MockProduceResponse struct { - version int16 - errors map[string]map[int32]KError - t TestReporter -} - -func NewMockProduceResponse(t TestReporter) *MockProduceResponse { - return &MockProduceResponse{t: t} -} - -func (mr *MockProduceResponse) SetVersion(version int16) *MockProduceResponse { - mr.version = version - return mr -} - -func (mr *MockProduceResponse) SetError(topic string, partition int32, kerror KError) *MockProduceResponse { - if mr.errors == nil { - mr.errors = make(map[string]map[int32]KError) - } - partitions := mr.errors[topic] - if partitions == nil { - partitions = make(map[int32]KError) - mr.errors[topic] = partitions - } - partitions[partition] = kerror - return mr -} - -func (mr *MockProduceResponse) For(reqBody versionedDecoder) encoderWithHeader { - req := reqBody.(*ProduceRequest) - res := &ProduceResponse{ - Version: mr.version, - } - for topic, partitions := range req.records { - for partition := range partitions { - res.AddTopicPartition(topic, partition, mr.getError(topic, partition)) - } - } - return res -} - -func (mr *MockProduceResponse) getError(topic string, partition int32) KError { - partitions := mr.errors[topic] - if partitions == nil { - return ErrNoError - } - kerror, ok := partitions[partition] - if !ok { - return ErrNoError - } - return kerror -} - -// MockOffsetFetchResponse is a `OffsetFetchResponse` builder. -type MockOffsetFetchResponse struct { - offsets map[string]map[string]map[int32]*OffsetFetchResponseBlock - error KError - t TestReporter -} - -func NewMockOffsetFetchResponse(t TestReporter) *MockOffsetFetchResponse { - return &MockOffsetFetchResponse{t: t} -} - -func (mr *MockOffsetFetchResponse) SetOffset(group, topic string, partition int32, offset int64, metadata string, kerror KError) *MockOffsetFetchResponse { - if mr.offsets == nil { - mr.offsets = make(map[string]map[string]map[int32]*OffsetFetchResponseBlock) - } - topics := mr.offsets[group] - if topics == nil { - topics = make(map[string]map[int32]*OffsetFetchResponseBlock) - mr.offsets[group] = topics - } - partitions := topics[topic] - if partitions == nil { - partitions = make(map[int32]*OffsetFetchResponseBlock) - topics[topic] = partitions - } - partitions[partition] = &OffsetFetchResponseBlock{offset, 0, metadata, kerror} - return mr -} - -func (mr *MockOffsetFetchResponse) SetError(kerror KError) *MockOffsetFetchResponse { - mr.error = kerror - return mr -} - -func (mr *MockOffsetFetchResponse) For(reqBody versionedDecoder) encoderWithHeader { - req := reqBody.(*OffsetFetchRequest) - group := req.ConsumerGroup - res := &OffsetFetchResponse{Version: req.Version} - - for topic, partitions := range mr.offsets[group] { - for partition, block := range partitions { - res.AddBlock(topic, partition, block) - } - } - - if res.Version >= 2 { - res.Err = mr.error - } - return res -} - -type MockCreateTopicsResponse struct { - t TestReporter -} - -func NewMockCreateTopicsResponse(t TestReporter) *MockCreateTopicsResponse { - return &MockCreateTopicsResponse{t: t} -} - -func (mr *MockCreateTopicsResponse) For(reqBody versionedDecoder) encoderWithHeader { - req := reqBody.(*CreateTopicsRequest) - res := &CreateTopicsResponse{ - Version: req.Version, - } - res.TopicErrors = make(map[string]*TopicError) - - for topic := range req.TopicDetails { - if res.Version >= 1 && strings.HasPrefix(topic, "_") { - msg := "insufficient permissions to create topic with reserved prefix" - res.TopicErrors[topic] = &TopicError{ - Err: ErrTopicAuthorizationFailed, - ErrMsg: &msg, - } - continue - } - res.TopicErrors[topic] = &TopicError{Err: ErrNoError} - } - return res -} - -type MockDeleteTopicsResponse struct { - t TestReporter -} - -func NewMockDeleteTopicsResponse(t TestReporter) *MockDeleteTopicsResponse { - return &MockDeleteTopicsResponse{t: t} -} - -func (mr *MockDeleteTopicsResponse) For(reqBody versionedDecoder) encoderWithHeader { - req := reqBody.(*DeleteTopicsRequest) - res := &DeleteTopicsResponse{} - res.TopicErrorCodes = make(map[string]KError) - - for _, topic := range req.Topics { - res.TopicErrorCodes[topic] = ErrNoError - } - res.Version = req.Version - return res -} - -type MockCreatePartitionsResponse struct { - t TestReporter -} - -func NewMockCreatePartitionsResponse(t TestReporter) *MockCreatePartitionsResponse { - return &MockCreatePartitionsResponse{t: t} -} - -func (mr *MockCreatePartitionsResponse) For(reqBody versionedDecoder) encoderWithHeader { - req := reqBody.(*CreatePartitionsRequest) - res := &CreatePartitionsResponse{} - res.TopicPartitionErrors = make(map[string]*TopicPartitionError) - - for topic := range req.TopicPartitions { - if strings.HasPrefix(topic, "_") { - msg := "insufficient permissions to create partition on topic with reserved prefix" - res.TopicPartitionErrors[topic] = &TopicPartitionError{ - Err: ErrTopicAuthorizationFailed, - ErrMsg: &msg, - } - continue - } - res.TopicPartitionErrors[topic] = &TopicPartitionError{Err: ErrNoError} - } - return res -} - -type MockAlterPartitionReassignmentsResponse struct { - t TestReporter -} - -func NewMockAlterPartitionReassignmentsResponse(t TestReporter) *MockAlterPartitionReassignmentsResponse { - return &MockAlterPartitionReassignmentsResponse{t: t} -} - -func (mr *MockAlterPartitionReassignmentsResponse) For(reqBody versionedDecoder) encoderWithHeader { - req := reqBody.(*AlterPartitionReassignmentsRequest) - _ = req - res := &AlterPartitionReassignmentsResponse{} - return res -} - -type MockListPartitionReassignmentsResponse struct { - t TestReporter -} - -func NewMockListPartitionReassignmentsResponse(t TestReporter) *MockListPartitionReassignmentsResponse { - return &MockListPartitionReassignmentsResponse{t: t} -} - -func (mr *MockListPartitionReassignmentsResponse) For(reqBody versionedDecoder) encoderWithHeader { - req := reqBody.(*ListPartitionReassignmentsRequest) - _ = req - res := &ListPartitionReassignmentsResponse{} - - for topic, partitions := range req.blocks { - for _, partition := range partitions { - res.AddBlock(topic, partition, []int32{0}, []int32{1}, []int32{2}) - } - } - - return res -} - -type MockDeleteRecordsResponse struct { - t TestReporter -} - -func NewMockDeleteRecordsResponse(t TestReporter) *MockDeleteRecordsResponse { - return &MockDeleteRecordsResponse{t: t} -} - -func (mr *MockDeleteRecordsResponse) For(reqBody versionedDecoder) encoderWithHeader { - req := reqBody.(*DeleteRecordsRequest) - res := &DeleteRecordsResponse{} - res.Topics = make(map[string]*DeleteRecordsResponseTopic) - - for topic, deleteRecordRequestTopic := range req.Topics { - partitions := make(map[int32]*DeleteRecordsResponsePartition) - for partition := range deleteRecordRequestTopic.PartitionOffsets { - partitions[partition] = &DeleteRecordsResponsePartition{Err: ErrNoError} - } - res.Topics[topic] = &DeleteRecordsResponseTopic{Partitions: partitions} - } - return res -} - -type MockDescribeConfigsResponse struct { - t TestReporter -} - -func NewMockDescribeConfigsResponse(t TestReporter) *MockDescribeConfigsResponse { - return &MockDescribeConfigsResponse{t: t} -} - -func (mr *MockDescribeConfigsResponse) For(reqBody versionedDecoder) encoderWithHeader { - req := reqBody.(*DescribeConfigsRequest) - res := &DescribeConfigsResponse{ - Version: req.Version, - } - - includeSynonyms := req.Version > 0 - includeSource := req.Version > 0 - - for _, r := range req.Resources { - var configEntries []*ConfigEntry - switch r.Type { - case BrokerResource: - configEntries = append(configEntries, - &ConfigEntry{ - Name: "min.insync.replicas", - Value: "2", - ReadOnly: false, - Default: false, - }, - ) - res.Resources = append(res.Resources, &ResourceResponse{ - Name: r.Name, - Configs: configEntries, - }) - case BrokerLoggerResource: - configEntries = append(configEntries, - &ConfigEntry{ - Name: "kafka.controller.KafkaController", - Value: "DEBUG", - ReadOnly: false, - Default: false, - }, - ) - res.Resources = append(res.Resources, &ResourceResponse{ - Name: r.Name, - Configs: configEntries, - }) - case TopicResource: - maxMessageBytes := &ConfigEntry{Name: "max.message.bytes", - Value: "1000000", - ReadOnly: false, - Default: !includeSource, - Sensitive: false, - } - if includeSource { - maxMessageBytes.Source = SourceDefault - } - if includeSynonyms { - maxMessageBytes.Synonyms = []*ConfigSynonym{ - { - ConfigName: "max.message.bytes", - ConfigValue: "500000", - }, - } - } - retentionMs := &ConfigEntry{Name: "retention.ms", - Value: "5000", - ReadOnly: false, - Default: false, - Sensitive: false, - } - if includeSynonyms { - retentionMs.Synonyms = []*ConfigSynonym{ - { - ConfigName: "log.retention.ms", - ConfigValue: "2500", - }, - } - } - password := &ConfigEntry{Name: "password", - Value: "12345", - ReadOnly: false, - Default: false, - Sensitive: true, - } - configEntries = append( - configEntries, maxMessageBytes, retentionMs, password) - res.Resources = append(res.Resources, &ResourceResponse{ - Name: r.Name, - Configs: configEntries, - }) - } - } - return res -} - -type MockDescribeConfigsResponseWithErrorCode struct { - t TestReporter -} - -func NewMockDescribeConfigsResponseWithErrorCode(t TestReporter) *MockDescribeConfigsResponseWithErrorCode { - return &MockDescribeConfigsResponseWithErrorCode{t: t} -} - -func (mr *MockDescribeConfigsResponseWithErrorCode) For(reqBody versionedDecoder) encoderWithHeader { - req := reqBody.(*DescribeConfigsRequest) - res := &DescribeConfigsResponse{ - Version: req.Version, - } - - for _, r := range req.Resources { - res.Resources = append(res.Resources, &ResourceResponse{ - Name: r.Name, - Type: r.Type, - ErrorCode: 83, - ErrorMsg: "", - }) - } - return res -} - -type MockAlterConfigsResponse struct { - t TestReporter -} - -func NewMockAlterConfigsResponse(t TestReporter) *MockAlterConfigsResponse { - return &MockAlterConfigsResponse{t: t} -} - -func (mr *MockAlterConfigsResponse) For(reqBody versionedDecoder) encoderWithHeader { - req := reqBody.(*AlterConfigsRequest) - res := &AlterConfigsResponse{} - - for _, r := range req.Resources { - res.Resources = append(res.Resources, &AlterConfigsResourceResponse{Name: r.Name, - Type: r.Type, - ErrorMsg: "", - }) - } - return res -} - -type MockAlterConfigsResponseWithErrorCode struct { - t TestReporter -} - -func NewMockAlterConfigsResponseWithErrorCode(t TestReporter) *MockAlterConfigsResponseWithErrorCode { - return &MockAlterConfigsResponseWithErrorCode{t: t} -} - -func (mr *MockAlterConfigsResponseWithErrorCode) For(reqBody versionedDecoder) encoderWithHeader { - req := reqBody.(*AlterConfigsRequest) - res := &AlterConfigsResponse{} - - for _, r := range req.Resources { - res.Resources = append(res.Resources, &AlterConfigsResourceResponse{ - Name: r.Name, - Type: r.Type, - ErrorCode: 83, - ErrorMsg: "", - }) - } - return res -} - -type MockCreateAclsResponse struct { - t TestReporter -} - -func NewMockCreateAclsResponse(t TestReporter) *MockCreateAclsResponse { - return &MockCreateAclsResponse{t: t} -} - -func (mr *MockCreateAclsResponse) For(reqBody versionedDecoder) encoderWithHeader { - req := reqBody.(*CreateAclsRequest) - res := &CreateAclsResponse{} - - for range req.AclCreations { - res.AclCreationResponses = append(res.AclCreationResponses, &AclCreationResponse{Err: ErrNoError}) - } - return res -} - -type MockListAclsResponse struct { - t TestReporter -} - -func NewMockListAclsResponse(t TestReporter) *MockListAclsResponse { - return &MockListAclsResponse{t: t} -} - -func (mr *MockListAclsResponse) For(reqBody versionedDecoder) encoderWithHeader { - req := reqBody.(*DescribeAclsRequest) - res := &DescribeAclsResponse{} - res.Err = ErrNoError - acl := &ResourceAcls{} - if req.ResourceName != nil { - acl.Resource.ResourceName = *req.ResourceName - } - acl.Resource.ResourcePatternType = req.ResourcePatternTypeFilter - acl.Resource.ResourceType = req.ResourceType - - host := "*" - if req.Host != nil { - host = *req.Host - } - - principal := "User:test" - if req.Principal != nil { - principal = *req.Principal - } - - permissionType := req.PermissionType - if permissionType == AclPermissionAny { - permissionType = AclPermissionAllow - } - - acl.Acls = append(acl.Acls, &Acl{Operation: req.Operation, PermissionType: permissionType, Host: host, Principal: principal}) - res.ResourceAcls = append(res.ResourceAcls, acl) - res.Version = int16(req.Version) - return res -} - -type MockSaslAuthenticateResponse struct { - t TestReporter - kerror KError - saslAuthBytes []byte -} - -func NewMockSaslAuthenticateResponse(t TestReporter) *MockSaslAuthenticateResponse { - return &MockSaslAuthenticateResponse{t: t} -} - -func (msar *MockSaslAuthenticateResponse) For(reqBody versionedDecoder) encoderWithHeader { - res := &SaslAuthenticateResponse{} - res.Err = msar.kerror - res.SaslAuthBytes = msar.saslAuthBytes - return res -} - -func (msar *MockSaslAuthenticateResponse) SetError(kerror KError) *MockSaslAuthenticateResponse { - msar.kerror = kerror - return msar -} - -func (msar *MockSaslAuthenticateResponse) SetAuthBytes(saslAuthBytes []byte) *MockSaslAuthenticateResponse { - msar.saslAuthBytes = saslAuthBytes - return msar -} - -type MockDeleteAclsResponse struct { - t TestReporter -} - -type MockSaslHandshakeResponse struct { - enabledMechanisms []string - kerror KError - t TestReporter -} - -func NewMockSaslHandshakeResponse(t TestReporter) *MockSaslHandshakeResponse { - return &MockSaslHandshakeResponse{t: t} -} - -func (mshr *MockSaslHandshakeResponse) For(reqBody versionedDecoder) encoderWithHeader { - res := &SaslHandshakeResponse{} - res.Err = mshr.kerror - res.EnabledMechanisms = mshr.enabledMechanisms - return res -} - -func (mshr *MockSaslHandshakeResponse) SetError(kerror KError) *MockSaslHandshakeResponse { - mshr.kerror = kerror - return mshr -} - -func (mshr *MockSaslHandshakeResponse) SetEnabledMechanisms(enabledMechanisms []string) *MockSaslHandshakeResponse { - mshr.enabledMechanisms = enabledMechanisms - return mshr -} - -func NewMockDeleteAclsResponse(t TestReporter) *MockDeleteAclsResponse { - return &MockDeleteAclsResponse{t: t} -} - -func (mr *MockDeleteAclsResponse) For(reqBody versionedDecoder) encoderWithHeader { - req := reqBody.(*DeleteAclsRequest) - res := &DeleteAclsResponse{} - - for range req.Filters { - response := &FilterResponse{Err: ErrNoError} - response.MatchingAcls = append(response.MatchingAcls, &MatchingAcl{Err: ErrNoError}) - res.FilterResponses = append(res.FilterResponses, response) - } - res.Version = int16(req.Version) - return res -} - -type MockDeleteGroupsResponse struct { - deletedGroups []string -} - -func NewMockDeleteGroupsRequest(t TestReporter) *MockDeleteGroupsResponse { - return &MockDeleteGroupsResponse{} -} - -func (m *MockDeleteGroupsResponse) SetDeletedGroups(groups []string) *MockDeleteGroupsResponse { - m.deletedGroups = groups - return m -} - -func (m *MockDeleteGroupsResponse) For(reqBody versionedDecoder) encoderWithHeader { - resp := &DeleteGroupsResponse{ - GroupErrorCodes: map[string]KError{}, - } - for _, group := range m.deletedGroups { - resp.GroupErrorCodes[group] = ErrNoError - } - return resp -} - -type MockDescribeLogDirsResponse struct { - t TestReporter - logDirs []DescribeLogDirsResponseDirMetadata -} - -func NewMockDescribeLogDirsResponse(t TestReporter) *MockDescribeLogDirsResponse { - return &MockDescribeLogDirsResponse{t: t} -} - -func (m *MockDescribeLogDirsResponse) SetLogDirs(logDirPath string, topicPartitions map[string]int) *MockDescribeLogDirsResponse { - var topics []DescribeLogDirsResponseTopic - for topic := range topicPartitions { - var partitions []DescribeLogDirsResponsePartition - for i := 0; i < topicPartitions[topic]; i++ { - partitions = append(partitions, DescribeLogDirsResponsePartition{ - PartitionID: int32(i), - IsTemporary: false, - OffsetLag: int64(0), - Size: int64(1234), - }) - } - topics = append(topics, DescribeLogDirsResponseTopic{ - Topic: topic, - Partitions: partitions, - }) - } - logDir := DescribeLogDirsResponseDirMetadata{ - ErrorCode: ErrNoError, - Path: logDirPath, - Topics: topics, - } - m.logDirs = []DescribeLogDirsResponseDirMetadata{logDir} - return m -} - -func (m *MockDescribeLogDirsResponse) For(reqBody versionedDecoder) encoderWithHeader { - resp := &DescribeLogDirsResponse{ - LogDirs: m.logDirs, - } - return resp -} diff --git a/vendor/github.com/Shopify/sarama/offset_commit_request.go b/vendor/github.com/Shopify/sarama/offset_commit_request.go deleted file mode 100644 index 9931cade..00000000 --- a/vendor/github.com/Shopify/sarama/offset_commit_request.go +++ /dev/null @@ -1,214 +0,0 @@ -package sarama - -import "errors" - -// ReceiveTime is a special value for the timestamp field of Offset Commit Requests which -// tells the broker to set the timestamp to the time at which the request was received. -// The timestamp is only used if message version 1 is used, which requires kafka 0.8.2. -const ReceiveTime int64 = -1 - -// GroupGenerationUndefined is a special value for the group generation field of -// Offset Commit Requests that should be used when a consumer group does not rely -// on Kafka for partition management. -const GroupGenerationUndefined = -1 - -type offsetCommitRequestBlock struct { - offset int64 - timestamp int64 - metadata string -} - -func (b *offsetCommitRequestBlock) encode(pe packetEncoder, version int16) error { - pe.putInt64(b.offset) - if version == 1 { - pe.putInt64(b.timestamp) - } else if b.timestamp != 0 { - Logger.Println("Non-zero timestamp specified for OffsetCommitRequest not v1, it will be ignored") - } - - return pe.putString(b.metadata) -} - -func (b *offsetCommitRequestBlock) decode(pd packetDecoder, version int16) (err error) { - if b.offset, err = pd.getInt64(); err != nil { - return err - } - if version == 1 { - if b.timestamp, err = pd.getInt64(); err != nil { - return err - } - } - b.metadata, err = pd.getString() - return err -} - -type OffsetCommitRequest struct { - ConsumerGroup string - ConsumerGroupGeneration int32 // v1 or later - ConsumerID string // v1 or later - RetentionTime int64 // v2 or later - - // Version can be: - // - 0 (kafka 0.8.1 and later) - // - 1 (kafka 0.8.2 and later) - // - 2 (kafka 0.9.0 and later) - // - 3 (kafka 0.11.0 and later) - // - 4 (kafka 2.0.0 and later) - Version int16 - blocks map[string]map[int32]*offsetCommitRequestBlock -} - -func (r *OffsetCommitRequest) encode(pe packetEncoder) error { - if r.Version < 0 || r.Version > 4 { - return PacketEncodingError{"invalid or unsupported OffsetCommitRequest version field"} - } - - if err := pe.putString(r.ConsumerGroup); err != nil { - return err - } - - if r.Version >= 1 { - pe.putInt32(r.ConsumerGroupGeneration) - if err := pe.putString(r.ConsumerID); err != nil { - return err - } - } else { - if r.ConsumerGroupGeneration != 0 { - Logger.Println("Non-zero ConsumerGroupGeneration specified for OffsetCommitRequest v0, it will be ignored") - } - if r.ConsumerID != "" { - Logger.Println("Non-empty ConsumerID specified for OffsetCommitRequest v0, it will be ignored") - } - } - - if r.Version >= 2 { - pe.putInt64(r.RetentionTime) - } else if r.RetentionTime != 0 { - Logger.Println("Non-zero RetentionTime specified for OffsetCommitRequest version <2, it will be ignored") - } - - if err := pe.putArrayLength(len(r.blocks)); err != nil { - return err - } - for topic, partitions := range r.blocks { - if err := pe.putString(topic); err != nil { - return err - } - if err := pe.putArrayLength(len(partitions)); err != nil { - return err - } - for partition, block := range partitions { - pe.putInt32(partition) - if err := block.encode(pe, r.Version); err != nil { - return err - } - } - } - return nil -} - -func (r *OffsetCommitRequest) decode(pd packetDecoder, version int16) (err error) { - r.Version = version - - if r.ConsumerGroup, err = pd.getString(); err != nil { - return err - } - - if r.Version >= 1 { - if r.ConsumerGroupGeneration, err = pd.getInt32(); err != nil { - return err - } - if r.ConsumerID, err = pd.getString(); err != nil { - return err - } - } - - if r.Version >= 2 { - if r.RetentionTime, err = pd.getInt64(); err != nil { - return err - } - } - - topicCount, err := pd.getArrayLength() - if err != nil { - return err - } - if topicCount == 0 { - return nil - } - r.blocks = make(map[string]map[int32]*offsetCommitRequestBlock) - for i := 0; i < topicCount; i++ { - topic, err := pd.getString() - if err != nil { - return err - } - partitionCount, err := pd.getArrayLength() - if err != nil { - return err - } - r.blocks[topic] = make(map[int32]*offsetCommitRequestBlock) - for j := 0; j < partitionCount; j++ { - partition, err := pd.getInt32() - if err != nil { - return err - } - block := &offsetCommitRequestBlock{} - if err := block.decode(pd, r.Version); err != nil { - return err - } - r.blocks[topic][partition] = block - } - } - return nil -} - -func (r *OffsetCommitRequest) key() int16 { - return 8 -} - -func (r *OffsetCommitRequest) version() int16 { - return r.Version -} - -func (r *OffsetCommitRequest) headerVersion() int16 { - return 1 -} - -func (r *OffsetCommitRequest) requiredVersion() KafkaVersion { - switch r.Version { - case 1: - return V0_8_2_0 - case 2: - return V0_9_0_0 - case 3: - return V0_11_0_0 - case 4: - return V2_0_0_0 - default: - return MinVersion - } -} - -func (r *OffsetCommitRequest) AddBlock(topic string, partitionID int32, offset int64, timestamp int64, metadata string) { - if r.blocks == nil { - r.blocks = make(map[string]map[int32]*offsetCommitRequestBlock) - } - - if r.blocks[topic] == nil { - r.blocks[topic] = make(map[int32]*offsetCommitRequestBlock) - } - - r.blocks[topic][partitionID] = &offsetCommitRequestBlock{offset, timestamp, metadata} -} - -func (r *OffsetCommitRequest) Offset(topic string, partitionID int32) (int64, string, error) { - partitions := r.blocks[topic] - if partitions == nil { - return 0, "", errors.New("no such offset") - } - block := partitions[partitionID] - if block == nil { - return 0, "", errors.New("no such offset") - } - return block.offset, block.metadata, nil -} diff --git a/vendor/github.com/Shopify/sarama/offset_commit_response.go b/vendor/github.com/Shopify/sarama/offset_commit_response.go deleted file mode 100644 index 342260ef..00000000 --- a/vendor/github.com/Shopify/sarama/offset_commit_response.go +++ /dev/null @@ -1,114 +0,0 @@ -package sarama - -type OffsetCommitResponse struct { - Version int16 - ThrottleTimeMs int32 - Errors map[string]map[int32]KError -} - -func (r *OffsetCommitResponse) AddError(topic string, partition int32, kerror KError) { - if r.Errors == nil { - r.Errors = make(map[string]map[int32]KError) - } - partitions := r.Errors[topic] - if partitions == nil { - partitions = make(map[int32]KError) - r.Errors[topic] = partitions - } - partitions[partition] = kerror -} - -func (r *OffsetCommitResponse) encode(pe packetEncoder) error { - if r.Version >= 3 { - pe.putInt32(r.ThrottleTimeMs) - } - if err := pe.putArrayLength(len(r.Errors)); err != nil { - return err - } - for topic, partitions := range r.Errors { - if err := pe.putString(topic); err != nil { - return err - } - if err := pe.putArrayLength(len(partitions)); err != nil { - return err - } - for partition, kerror := range partitions { - pe.putInt32(partition) - pe.putInt16(int16(kerror)) - } - } - return nil -} - -func (r *OffsetCommitResponse) decode(pd packetDecoder, version int16) (err error) { - r.Version = version - - if version >= 3 { - r.ThrottleTimeMs, err = pd.getInt32() - if err != nil { - return err - } - } - - numTopics, err := pd.getArrayLength() - if err != nil || numTopics == 0 { - return err - } - - r.Errors = make(map[string]map[int32]KError, numTopics) - for i := 0; i < numTopics; i++ { - name, err := pd.getString() - if err != nil { - return err - } - - numErrors, err := pd.getArrayLength() - if err != nil { - return err - } - - r.Errors[name] = make(map[int32]KError, numErrors) - - for j := 0; j < numErrors; j++ { - id, err := pd.getInt32() - if err != nil { - return err - } - - tmp, err := pd.getInt16() - if err != nil { - return err - } - r.Errors[name][id] = KError(tmp) - } - } - - return nil -} - -func (r *OffsetCommitResponse) key() int16 { - return 8 -} - -func (r *OffsetCommitResponse) version() int16 { - return r.Version -} - -func (r *OffsetCommitResponse) headerVersion() int16 { - return 0 -} - -func (r *OffsetCommitResponse) requiredVersion() KafkaVersion { - switch r.Version { - case 1: - return V0_8_2_0 - case 2: - return V0_9_0_0 - case 3: - return V0_11_0_0 - case 4: - return V2_0_0_0 - default: - return MinVersion - } -} diff --git a/vendor/github.com/Shopify/sarama/offset_fetch_request.go b/vendor/github.com/Shopify/sarama/offset_fetch_request.go deleted file mode 100644 index 51e9faa3..00000000 --- a/vendor/github.com/Shopify/sarama/offset_fetch_request.go +++ /dev/null @@ -1,104 +0,0 @@ -package sarama - -type OffsetFetchRequest struct { - Version int16 - ConsumerGroup string - partitions map[string][]int32 -} - -func (r *OffsetFetchRequest) encode(pe packetEncoder) (err error) { - if r.Version < 0 || r.Version > 5 { - return PacketEncodingError{"invalid or unsupported OffsetFetchRequest version field"} - } - - if err = pe.putString(r.ConsumerGroup); err != nil { - return err - } - - if r.Version >= 2 && r.partitions == nil { - pe.putInt32(-1) - } else { - if err = pe.putArrayLength(len(r.partitions)); err != nil { - return err - } - for topic, partitions := range r.partitions { - if err = pe.putString(topic); err != nil { - return err - } - if err = pe.putInt32Array(partitions); err != nil { - return err - } - } - } - return nil -} - -func (r *OffsetFetchRequest) decode(pd packetDecoder, version int16) (err error) { - r.Version = version - if r.ConsumerGroup, err = pd.getString(); err != nil { - return err - } - partitionCount, err := pd.getArrayLength() - if err != nil { - return err - } - if (partitionCount == 0 && version < 2) || partitionCount < 0 { - return nil - } - r.partitions = make(map[string][]int32) - for i := 0; i < partitionCount; i++ { - topic, err := pd.getString() - if err != nil { - return err - } - partitions, err := pd.getInt32Array() - if err != nil { - return err - } - r.partitions[topic] = partitions - } - return nil -} - -func (r *OffsetFetchRequest) key() int16 { - return 9 -} - -func (r *OffsetFetchRequest) version() int16 { - return r.Version -} - -func (r *OffsetFetchRequest) headerVersion() int16 { - return 1 -} - -func (r *OffsetFetchRequest) requiredVersion() KafkaVersion { - switch r.Version { - case 1: - return V0_8_2_0 - case 2: - return V0_10_2_0 - case 3: - return V0_11_0_0 - case 4: - return V2_0_0_0 - case 5: - return V2_1_0_0 - default: - return MinVersion - } -} - -func (r *OffsetFetchRequest) ZeroPartitions() { - if r.partitions == nil && r.Version >= 2 { - r.partitions = make(map[string][]int32) - } -} - -func (r *OffsetFetchRequest) AddPartition(topic string, partitionID int32) { - if r.partitions == nil { - r.partitions = make(map[string][]int32) - } - - r.partitions[topic] = append(r.partitions[topic], partitionID) -} diff --git a/vendor/github.com/Shopify/sarama/offset_fetch_response.go b/vendor/github.com/Shopify/sarama/offset_fetch_response.go deleted file mode 100644 index 9c64e070..00000000 --- a/vendor/github.com/Shopify/sarama/offset_fetch_response.go +++ /dev/null @@ -1,201 +0,0 @@ -package sarama - -type OffsetFetchResponseBlock struct { - Offset int64 - LeaderEpoch int32 - Metadata string - Err KError -} - -func (b *OffsetFetchResponseBlock) decode(pd packetDecoder, version int16) (err error) { - b.Offset, err = pd.getInt64() - if err != nil { - return err - } - - if version >= 5 { - b.LeaderEpoch, err = pd.getInt32() - if err != nil { - return err - } - } - - b.Metadata, err = pd.getString() - if err != nil { - return err - } - - tmp, err := pd.getInt16() - if err != nil { - return err - } - b.Err = KError(tmp) - - return nil -} - -func (b *OffsetFetchResponseBlock) encode(pe packetEncoder, version int16) (err error) { - pe.putInt64(b.Offset) - - if version >= 5 { - pe.putInt32(b.LeaderEpoch) - } - - err = pe.putString(b.Metadata) - if err != nil { - return err - } - - pe.putInt16(int16(b.Err)) - - return nil -} - -type OffsetFetchResponse struct { - Version int16 - ThrottleTimeMs int32 - Blocks map[string]map[int32]*OffsetFetchResponseBlock - Err KError -} - -func (r *OffsetFetchResponse) encode(pe packetEncoder) error { - if r.Version >= 3 { - pe.putInt32(r.ThrottleTimeMs) - } - - if err := pe.putArrayLength(len(r.Blocks)); err != nil { - return err - } - for topic, partitions := range r.Blocks { - if err := pe.putString(topic); err != nil { - return err - } - if err := pe.putArrayLength(len(partitions)); err != nil { - return err - } - for partition, block := range partitions { - pe.putInt32(partition) - if err := block.encode(pe, r.Version); err != nil { - return err - } - } - } - if r.Version >= 2 { - pe.putInt16(int16(r.Err)) - } - return nil -} - -func (r *OffsetFetchResponse) decode(pd packetDecoder, version int16) (err error) { - r.Version = version - - if version >= 3 { - r.ThrottleTimeMs, err = pd.getInt32() - if err != nil { - return err - } - } - - numTopics, err := pd.getArrayLength() - if err != nil { - return err - } - - if numTopics > 0 { - r.Blocks = make(map[string]map[int32]*OffsetFetchResponseBlock, numTopics) - for i := 0; i < numTopics; i++ { - name, err := pd.getString() - if err != nil { - return err - } - - numBlocks, err := pd.getArrayLength() - if err != nil { - return err - } - - if numBlocks == 0 { - r.Blocks[name] = nil - continue - } - r.Blocks[name] = make(map[int32]*OffsetFetchResponseBlock, numBlocks) - - for j := 0; j < numBlocks; j++ { - id, err := pd.getInt32() - if err != nil { - return err - } - - block := new(OffsetFetchResponseBlock) - err = block.decode(pd, version) - if err != nil { - return err - } - r.Blocks[name][id] = block - } - } - } - - if version >= 2 { - kerr, err := pd.getInt16() - if err != nil { - return err - } - r.Err = KError(kerr) - } - - return nil -} - -func (r *OffsetFetchResponse) key() int16 { - return 9 -} - -func (r *OffsetFetchResponse) version() int16 { - return r.Version -} - -func (r *OffsetFetchResponse) headerVersion() int16 { - return 0 -} - -func (r *OffsetFetchResponse) requiredVersion() KafkaVersion { - switch r.Version { - case 1: - return V0_8_2_0 - case 2: - return V0_10_2_0 - case 3: - return V0_11_0_0 - case 4: - return V2_0_0_0 - case 5: - return V2_1_0_0 - default: - return MinVersion - } -} - -func (r *OffsetFetchResponse) GetBlock(topic string, partition int32) *OffsetFetchResponseBlock { - if r.Blocks == nil { - return nil - } - - if r.Blocks[topic] == nil { - return nil - } - - return r.Blocks[topic][partition] -} - -func (r *OffsetFetchResponse) AddBlock(topic string, partition int32, block *OffsetFetchResponseBlock) { - if r.Blocks == nil { - r.Blocks = make(map[string]map[int32]*OffsetFetchResponseBlock) - } - partitions := r.Blocks[topic] - if partitions == nil { - partitions = make(map[int32]*OffsetFetchResponseBlock) - r.Blocks[topic] = partitions - } - partitions[partition] = block -} diff --git a/vendor/github.com/Shopify/sarama/offset_manager.go b/vendor/github.com/Shopify/sarama/offset_manager.go deleted file mode 100644 index b4fea822..00000000 --- a/vendor/github.com/Shopify/sarama/offset_manager.go +++ /dev/null @@ -1,591 +0,0 @@ -package sarama - -import ( - "sync" - "time" -) - -// Offset Manager - -// OffsetManager uses Kafka to store and fetch consumed partition offsets. -type OffsetManager interface { - // ManagePartition creates a PartitionOffsetManager on the given topic/partition. - // It will return an error if this OffsetManager is already managing the given - // topic/partition. - ManagePartition(topic string, partition int32) (PartitionOffsetManager, error) - - // Close stops the OffsetManager from managing offsets. It is required to call - // this function before an OffsetManager object passes out of scope, as it - // will otherwise leak memory. You must call this after all the - // PartitionOffsetManagers are closed. - Close() error - - // Commit commits the offsets. This method can be used if AutoCommit.Enable is - // set to false. - Commit() -} - -type offsetManager struct { - client Client - conf *Config - group string - ticker *time.Ticker - - memberID string - generation int32 - - broker *Broker - brokerLock sync.RWMutex - - poms map[string]map[int32]*partitionOffsetManager - pomsLock sync.RWMutex - - closeOnce sync.Once - closing chan none - closed chan none -} - -// NewOffsetManagerFromClient creates a new OffsetManager from the given client. -// It is still necessary to call Close() on the underlying client when finished with the partition manager. -func NewOffsetManagerFromClient(group string, client Client) (OffsetManager, error) { - return newOffsetManagerFromClient(group, "", GroupGenerationUndefined, client) -} - -func newOffsetManagerFromClient(group, memberID string, generation int32, client Client) (*offsetManager, error) { - // Check that we are not dealing with a closed Client before processing any other arguments - if client.Closed() { - return nil, ErrClosedClient - } - - conf := client.Config() - om := &offsetManager{ - client: client, - conf: conf, - group: group, - poms: make(map[string]map[int32]*partitionOffsetManager), - - memberID: memberID, - generation: generation, - - closing: make(chan none), - closed: make(chan none), - } - if conf.Consumer.Offsets.AutoCommit.Enable { - om.ticker = time.NewTicker(conf.Consumer.Offsets.AutoCommit.Interval) - go withRecover(om.mainLoop) - } - - return om, nil -} - -func (om *offsetManager) ManagePartition(topic string, partition int32) (PartitionOffsetManager, error) { - pom, err := om.newPartitionOffsetManager(topic, partition) - if err != nil { - return nil, err - } - - om.pomsLock.Lock() - defer om.pomsLock.Unlock() - - topicManagers := om.poms[topic] - if topicManagers == nil { - topicManagers = make(map[int32]*partitionOffsetManager) - om.poms[topic] = topicManagers - } - - if topicManagers[partition] != nil { - return nil, ConfigurationError("That topic/partition is already being managed") - } - - topicManagers[partition] = pom - return pom, nil -} - -func (om *offsetManager) Close() error { - om.closeOnce.Do(func() { - // exit the mainLoop - close(om.closing) - if om.conf.Consumer.Offsets.AutoCommit.Enable { - <-om.closed - } - - // mark all POMs as closed - om.asyncClosePOMs() - - // flush one last time - for attempt := 0; attempt <= om.conf.Consumer.Offsets.Retry.Max; attempt++ { - om.flushToBroker() - if om.releasePOMs(false) == 0 { - break - } - } - - om.releasePOMs(true) - om.brokerLock.Lock() - om.broker = nil - om.brokerLock.Unlock() - }) - return nil -} - -func (om *offsetManager) computeBackoff(retries int) time.Duration { - if om.conf.Metadata.Retry.BackoffFunc != nil { - return om.conf.Metadata.Retry.BackoffFunc(retries, om.conf.Metadata.Retry.Max) - } else { - return om.conf.Metadata.Retry.Backoff - } -} - -func (om *offsetManager) fetchInitialOffset(topic string, partition int32, retries int) (int64, string, error) { - broker, err := om.coordinator() - if err != nil { - if retries <= 0 { - return 0, "", err - } - return om.fetchInitialOffset(topic, partition, retries-1) - } - - req := new(OffsetFetchRequest) - req.Version = 1 - req.ConsumerGroup = om.group - req.AddPartition(topic, partition) - - resp, err := broker.FetchOffset(req) - if err != nil { - if retries <= 0 { - return 0, "", err - } - om.releaseCoordinator(broker) - return om.fetchInitialOffset(topic, partition, retries-1) - } - - block := resp.GetBlock(topic, partition) - if block == nil { - return 0, "", ErrIncompleteResponse - } - - switch block.Err { - case ErrNoError: - return block.Offset, block.Metadata, nil - case ErrNotCoordinatorForConsumer: - if retries <= 0 { - return 0, "", block.Err - } - om.releaseCoordinator(broker) - return om.fetchInitialOffset(topic, partition, retries-1) - case ErrOffsetsLoadInProgress: - if retries <= 0 { - return 0, "", block.Err - } - backoff := om.computeBackoff(retries) - select { - case <-om.closing: - return 0, "", block.Err - case <-time.After(backoff): - } - return om.fetchInitialOffset(topic, partition, retries-1) - default: - return 0, "", block.Err - } -} - -func (om *offsetManager) coordinator() (*Broker, error) { - om.brokerLock.RLock() - broker := om.broker - om.brokerLock.RUnlock() - - if broker != nil { - return broker, nil - } - - om.brokerLock.Lock() - defer om.brokerLock.Unlock() - - if broker := om.broker; broker != nil { - return broker, nil - } - - if err := om.client.RefreshCoordinator(om.group); err != nil { - return nil, err - } - - broker, err := om.client.Coordinator(om.group) - if err != nil { - return nil, err - } - - om.broker = broker - return broker, nil -} - -func (om *offsetManager) releaseCoordinator(b *Broker) { - om.brokerLock.Lock() - if om.broker == b { - om.broker = nil - } - om.brokerLock.Unlock() -} - -func (om *offsetManager) mainLoop() { - defer om.ticker.Stop() - defer close(om.closed) - - for { - select { - case <-om.ticker.C: - om.Commit() - case <-om.closing: - return - } - } -} - -func (om *offsetManager) Commit() { - om.flushToBroker() - om.releasePOMs(false) -} - -func (om *offsetManager) flushToBroker() { - req := om.constructRequest() - if req == nil { - return - } - - broker, err := om.coordinator() - if err != nil { - om.handleError(err) - return - } - - resp, err := broker.CommitOffset(req) - if err != nil { - om.handleError(err) - om.releaseCoordinator(broker) - _ = broker.Close() - return - } - - om.handleResponse(broker, req, resp) -} - -func (om *offsetManager) constructRequest() *OffsetCommitRequest { - var r *OffsetCommitRequest - var perPartitionTimestamp int64 - if om.conf.Consumer.Offsets.Retention == 0 { - perPartitionTimestamp = ReceiveTime - r = &OffsetCommitRequest{ - Version: 1, - ConsumerGroup: om.group, - ConsumerID: om.memberID, - ConsumerGroupGeneration: om.generation, - } - } else { - r = &OffsetCommitRequest{ - Version: 2, - RetentionTime: int64(om.conf.Consumer.Offsets.Retention / time.Millisecond), - ConsumerGroup: om.group, - ConsumerID: om.memberID, - ConsumerGroupGeneration: om.generation, - } - } - - om.pomsLock.RLock() - defer om.pomsLock.RUnlock() - - for _, topicManagers := range om.poms { - for _, pom := range topicManagers { - pom.lock.Lock() - if pom.dirty { - r.AddBlock(pom.topic, pom.partition, pom.offset, perPartitionTimestamp, pom.metadata) - } - pom.lock.Unlock() - } - } - - if len(r.blocks) > 0 { - return r - } - - return nil -} - -func (om *offsetManager) handleResponse(broker *Broker, req *OffsetCommitRequest, resp *OffsetCommitResponse) { - om.pomsLock.RLock() - defer om.pomsLock.RUnlock() - - for _, topicManagers := range om.poms { - for _, pom := range topicManagers { - if req.blocks[pom.topic] == nil || req.blocks[pom.topic][pom.partition] == nil { - continue - } - - var err KError - var ok bool - - if resp.Errors[pom.topic] == nil { - pom.handleError(ErrIncompleteResponse) - continue - } - if err, ok = resp.Errors[pom.topic][pom.partition]; !ok { - pom.handleError(ErrIncompleteResponse) - continue - } - - switch err { - case ErrNoError: - block := req.blocks[pom.topic][pom.partition] - pom.updateCommitted(block.offset, block.metadata) - case ErrNotLeaderForPartition, ErrLeaderNotAvailable, - ErrConsumerCoordinatorNotAvailable, ErrNotCoordinatorForConsumer: - // not a critical error, we just need to redispatch - om.releaseCoordinator(broker) - case ErrOffsetMetadataTooLarge, ErrInvalidCommitOffsetSize: - // nothing we can do about this, just tell the user and carry on - pom.handleError(err) - case ErrOffsetsLoadInProgress: - // nothing wrong but we didn't commit, we'll get it next time round - case ErrUnknownTopicOrPartition: - // let the user know *and* try redispatching - if topic-auto-create is - // enabled, redispatching should trigger a metadata req and create the - // topic; if not then re-dispatching won't help, but we've let the user - // know and it shouldn't hurt either (see https://github.com/Shopify/sarama/issues/706) - fallthrough - default: - // dunno, tell the user and try redispatching - pom.handleError(err) - om.releaseCoordinator(broker) - } - } - } -} - -func (om *offsetManager) handleError(err error) { - om.pomsLock.RLock() - defer om.pomsLock.RUnlock() - - for _, topicManagers := range om.poms { - for _, pom := range topicManagers { - pom.handleError(err) - } - } -} - -func (om *offsetManager) asyncClosePOMs() { - om.pomsLock.RLock() - defer om.pomsLock.RUnlock() - - for _, topicManagers := range om.poms { - for _, pom := range topicManagers { - pom.AsyncClose() - } - } -} - -// Releases/removes closed POMs once they are clean (or when forced) -func (om *offsetManager) releasePOMs(force bool) (remaining int) { - om.pomsLock.Lock() - defer om.pomsLock.Unlock() - - for topic, topicManagers := range om.poms { - for partition, pom := range topicManagers { - pom.lock.Lock() - releaseDue := pom.done && (force || !pom.dirty) - pom.lock.Unlock() - - if releaseDue { - pom.release() - - delete(om.poms[topic], partition) - if len(om.poms[topic]) == 0 { - delete(om.poms, topic) - } - } - } - remaining += len(om.poms[topic]) - } - return -} - -func (om *offsetManager) findPOM(topic string, partition int32) *partitionOffsetManager { - om.pomsLock.RLock() - defer om.pomsLock.RUnlock() - - if partitions, ok := om.poms[topic]; ok { - if pom, ok := partitions[partition]; ok { - return pom - } - } - return nil -} - -// Partition Offset Manager - -// PartitionOffsetManager uses Kafka to store and fetch consumed partition offsets. You MUST call Close() -// on a partition offset manager to avoid leaks, it will not be garbage-collected automatically when it passes -// out of scope. -type PartitionOffsetManager interface { - // NextOffset returns the next offset that should be consumed for the managed - // partition, accompanied by metadata which can be used to reconstruct the state - // of the partition consumer when it resumes. NextOffset() will return - // `config.Consumer.Offsets.Initial` and an empty metadata string if no offset - // was committed for this partition yet. - NextOffset() (int64, string) - - // MarkOffset marks the provided offset, alongside a metadata string - // that represents the state of the partition consumer at that point in time. The - // metadata string can be used by another consumer to restore that state, so it - // can resume consumption. - // - // To follow upstream conventions, you are expected to mark the offset of the - // next message to read, not the last message read. Thus, when calling `MarkOffset` - // you should typically add one to the offset of the last consumed message. - // - // Note: calling MarkOffset does not necessarily commit the offset to the backend - // store immediately for efficiency reasons, and it may never be committed if - // your application crashes. This means that you may end up processing the same - // message twice, and your processing should ideally be idempotent. - MarkOffset(offset int64, metadata string) - - // ResetOffset resets to the provided offset, alongside a metadata string that - // represents the state of the partition consumer at that point in time. Reset - // acts as a counterpart to MarkOffset, the difference being that it allows to - // reset an offset to an earlier or smaller value, where MarkOffset only - // allows incrementing the offset. cf MarkOffset for more details. - ResetOffset(offset int64, metadata string) - - // Errors returns a read channel of errors that occur during offset management, if - // enabled. By default, errors are logged and not returned over this channel. If - // you want to implement any custom error handling, set your config's - // Consumer.Return.Errors setting to true, and read from this channel. - Errors() <-chan *ConsumerError - - // AsyncClose initiates a shutdown of the PartitionOffsetManager. This method will - // return immediately, after which you should wait until the 'errors' channel has - // been drained and closed. It is required to call this function, or Close before - // a consumer object passes out of scope, as it will otherwise leak memory. You - // must call this before calling Close on the underlying client. - AsyncClose() - - // Close stops the PartitionOffsetManager from managing offsets. It is required to - // call this function (or AsyncClose) before a PartitionOffsetManager object - // passes out of scope, as it will otherwise leak memory. You must call this - // before calling Close on the underlying client. - Close() error -} - -type partitionOffsetManager struct { - parent *offsetManager - topic string - partition int32 - - lock sync.Mutex - offset int64 - metadata string - dirty bool - done bool - - releaseOnce sync.Once - errors chan *ConsumerError -} - -func (om *offsetManager) newPartitionOffsetManager(topic string, partition int32) (*partitionOffsetManager, error) { - offset, metadata, err := om.fetchInitialOffset(topic, partition, om.conf.Metadata.Retry.Max) - if err != nil { - return nil, err - } - - return &partitionOffsetManager{ - parent: om, - topic: topic, - partition: partition, - errors: make(chan *ConsumerError, om.conf.ChannelBufferSize), - offset: offset, - metadata: metadata, - }, nil -} - -func (pom *partitionOffsetManager) Errors() <-chan *ConsumerError { - return pom.errors -} - -func (pom *partitionOffsetManager) MarkOffset(offset int64, metadata string) { - pom.lock.Lock() - defer pom.lock.Unlock() - - if offset > pom.offset { - pom.offset = offset - pom.metadata = metadata - pom.dirty = true - } -} - -func (pom *partitionOffsetManager) ResetOffset(offset int64, metadata string) { - pom.lock.Lock() - defer pom.lock.Unlock() - - if offset <= pom.offset { - pom.offset = offset - pom.metadata = metadata - pom.dirty = true - } -} - -func (pom *partitionOffsetManager) updateCommitted(offset int64, metadata string) { - pom.lock.Lock() - defer pom.lock.Unlock() - - if pom.offset == offset && pom.metadata == metadata { - pom.dirty = false - } -} - -func (pom *partitionOffsetManager) NextOffset() (int64, string) { - pom.lock.Lock() - defer pom.lock.Unlock() - - if pom.offset >= 0 { - return pom.offset, pom.metadata - } - - return pom.parent.conf.Consumer.Offsets.Initial, "" -} - -func (pom *partitionOffsetManager) AsyncClose() { - pom.lock.Lock() - pom.done = true - pom.lock.Unlock() -} - -func (pom *partitionOffsetManager) Close() error { - pom.AsyncClose() - - var errors ConsumerErrors - for err := range pom.errors { - errors = append(errors, err) - } - - if len(errors) > 0 { - return errors - } - return nil -} - -func (pom *partitionOffsetManager) handleError(err error) { - cErr := &ConsumerError{ - Topic: pom.topic, - Partition: pom.partition, - Err: err, - } - - if pom.parent.conf.Consumer.Return.Errors { - pom.errors <- cErr - } else { - Logger.Println(cErr) - } -} - -func (pom *partitionOffsetManager) release() { - pom.releaseOnce.Do(func() { - close(pom.errors) - }) -} diff --git a/vendor/github.com/Shopify/sarama/offset_request.go b/vendor/github.com/Shopify/sarama/offset_request.go deleted file mode 100644 index c0b3305f..00000000 --- a/vendor/github.com/Shopify/sarama/offset_request.go +++ /dev/null @@ -1,160 +0,0 @@ -package sarama - -type offsetRequestBlock struct { - time int64 - maxOffsets int32 // Only used in version 0 -} - -func (b *offsetRequestBlock) encode(pe packetEncoder, version int16) error { - pe.putInt64(b.time) - if version == 0 { - pe.putInt32(b.maxOffsets) - } - - return nil -} - -func (b *offsetRequestBlock) decode(pd packetDecoder, version int16) (err error) { - if b.time, err = pd.getInt64(); err != nil { - return err - } - if version == 0 { - if b.maxOffsets, err = pd.getInt32(); err != nil { - return err - } - } - return nil -} - -type OffsetRequest struct { - Version int16 - replicaID int32 - isReplicaIDSet bool - blocks map[string]map[int32]*offsetRequestBlock -} - -func (r *OffsetRequest) encode(pe packetEncoder) error { - if r.isReplicaIDSet { - pe.putInt32(r.replicaID) - } else { - // default replica ID is always -1 for clients - pe.putInt32(-1) - } - - err := pe.putArrayLength(len(r.blocks)) - if err != nil { - return err - } - for topic, partitions := range r.blocks { - err = pe.putString(topic) - if err != nil { - return err - } - err = pe.putArrayLength(len(partitions)) - if err != nil { - return err - } - for partition, block := range partitions { - pe.putInt32(partition) - if err = block.encode(pe, r.Version); err != nil { - return err - } - } - } - return nil -} - -func (r *OffsetRequest) decode(pd packetDecoder, version int16) error { - r.Version = version - - replicaID, err := pd.getInt32() - if err != nil { - return err - } - if replicaID >= 0 { - r.SetReplicaID(replicaID) - } - - blockCount, err := pd.getArrayLength() - if err != nil { - return err - } - if blockCount == 0 { - return nil - } - r.blocks = make(map[string]map[int32]*offsetRequestBlock) - for i := 0; i < blockCount; i++ { - topic, err := pd.getString() - if err != nil { - return err - } - partitionCount, err := pd.getArrayLength() - if err != nil { - return err - } - r.blocks[topic] = make(map[int32]*offsetRequestBlock) - for j := 0; j < partitionCount; j++ { - partition, err := pd.getInt32() - if err != nil { - return err - } - block := &offsetRequestBlock{} - if err := block.decode(pd, version); err != nil { - return err - } - r.blocks[topic][partition] = block - } - } - return nil -} - -func (r *OffsetRequest) key() int16 { - return 2 -} - -func (r *OffsetRequest) version() int16 { - return r.Version -} - -func (r *OffsetRequest) headerVersion() int16 { - return 1 -} - -func (r *OffsetRequest) requiredVersion() KafkaVersion { - switch r.Version { - case 1: - return V0_10_1_0 - default: - return MinVersion - } -} - -func (r *OffsetRequest) SetReplicaID(id int32) { - r.replicaID = id - r.isReplicaIDSet = true -} - -func (r *OffsetRequest) ReplicaID() int32 { - if r.isReplicaIDSet { - return r.replicaID - } - return -1 -} - -func (r *OffsetRequest) AddBlock(topic string, partitionID int32, time int64, maxOffsets int32) { - if r.blocks == nil { - r.blocks = make(map[string]map[int32]*offsetRequestBlock) - } - - if r.blocks[topic] == nil { - r.blocks[topic] = make(map[int32]*offsetRequestBlock) - } - - tmp := new(offsetRequestBlock) - tmp.time = time - if r.Version == 0 { - tmp.maxOffsets = maxOffsets - } - - r.blocks[topic][partitionID] = tmp -} diff --git a/vendor/github.com/Shopify/sarama/offset_response.go b/vendor/github.com/Shopify/sarama/offset_response.go deleted file mode 100644 index ead3ebbc..00000000 --- a/vendor/github.com/Shopify/sarama/offset_response.go +++ /dev/null @@ -1,178 +0,0 @@ -package sarama - -type OffsetResponseBlock struct { - Err KError - Offsets []int64 // Version 0 - Offset int64 // Version 1 - Timestamp int64 // Version 1 -} - -func (b *OffsetResponseBlock) decode(pd packetDecoder, version int16) (err error) { - tmp, err := pd.getInt16() - if err != nil { - return err - } - b.Err = KError(tmp) - - if version == 0 { - b.Offsets, err = pd.getInt64Array() - - return err - } - - b.Timestamp, err = pd.getInt64() - if err != nil { - return err - } - - b.Offset, err = pd.getInt64() - if err != nil { - return err - } - - // For backwards compatibility put the offset in the offsets array too - b.Offsets = []int64{b.Offset} - - return nil -} - -func (b *OffsetResponseBlock) encode(pe packetEncoder, version int16) (err error) { - pe.putInt16(int16(b.Err)) - - if version == 0 { - return pe.putInt64Array(b.Offsets) - } - - pe.putInt64(b.Timestamp) - pe.putInt64(b.Offset) - - return nil -} - -type OffsetResponse struct { - Version int16 - Blocks map[string]map[int32]*OffsetResponseBlock -} - -func (r *OffsetResponse) decode(pd packetDecoder, version int16) (err error) { - numTopics, err := pd.getArrayLength() - if err != nil { - return err - } - - r.Blocks = make(map[string]map[int32]*OffsetResponseBlock, numTopics) - for i := 0; i < numTopics; i++ { - name, err := pd.getString() - if err != nil { - return err - } - - numBlocks, err := pd.getArrayLength() - if err != nil { - return err - } - - r.Blocks[name] = make(map[int32]*OffsetResponseBlock, numBlocks) - - for j := 0; j < numBlocks; j++ { - id, err := pd.getInt32() - if err != nil { - return err - } - - block := new(OffsetResponseBlock) - err = block.decode(pd, version) - if err != nil { - return err - } - r.Blocks[name][id] = block - } - } - - return nil -} - -func (r *OffsetResponse) GetBlock(topic string, partition int32) *OffsetResponseBlock { - if r.Blocks == nil { - return nil - } - - if r.Blocks[topic] == nil { - return nil - } - - return r.Blocks[topic][partition] -} - -/* -// [0 0 0 1 ntopics -0 8 109 121 95 116 111 112 105 99 topic -0 0 0 1 npartitions -0 0 0 0 id -0 0 - -0 0 0 1 0 0 0 0 -0 1 1 1 0 0 0 1 -0 8 109 121 95 116 111 112 -105 99 0 0 0 1 0 0 -0 0 0 0 0 0 0 1 -0 0 0 0 0 1 1 1] - -*/ -func (r *OffsetResponse) encode(pe packetEncoder) (err error) { - if err = pe.putArrayLength(len(r.Blocks)); err != nil { - return err - } - - for topic, partitions := range r.Blocks { - if err = pe.putString(topic); err != nil { - return err - } - if err = pe.putArrayLength(len(partitions)); err != nil { - return err - } - for partition, block := range partitions { - pe.putInt32(partition) - if err = block.encode(pe, r.version()); err != nil { - return err - } - } - } - - return nil -} - -func (r *OffsetResponse) key() int16 { - return 2 -} - -func (r *OffsetResponse) version() int16 { - return r.Version -} - -func (r *OffsetResponse) headerVersion() int16 { - return 0 -} - -func (r *OffsetResponse) requiredVersion() KafkaVersion { - switch r.Version { - case 1: - return V0_10_1_0 - default: - return MinVersion - } -} - -// testing API - -func (r *OffsetResponse) AddTopicPartition(topic string, partition int32, offset int64) { - if r.Blocks == nil { - r.Blocks = make(map[string]map[int32]*OffsetResponseBlock) - } - byTopic, ok := r.Blocks[topic] - if !ok { - byTopic = make(map[int32]*OffsetResponseBlock) - r.Blocks[topic] = byTopic - } - byTopic[partition] = &OffsetResponseBlock{Offsets: []int64{offset}, Offset: offset} -} diff --git a/vendor/github.com/Shopify/sarama/packet_decoder.go b/vendor/github.com/Shopify/sarama/packet_decoder.go deleted file mode 100644 index ed00ba35..00000000 --- a/vendor/github.com/Shopify/sarama/packet_decoder.go +++ /dev/null @@ -1,67 +0,0 @@ -package sarama - -// PacketDecoder is the interface providing helpers for reading with Kafka's encoding rules. -// Types implementing Decoder only need to worry about calling methods like GetString, -// not about how a string is represented in Kafka. -type packetDecoder interface { - // Primitives - getInt8() (int8, error) - getInt16() (int16, error) - getInt32() (int32, error) - getInt64() (int64, error) - getVarint() (int64, error) - getUVarint() (uint64, error) - getArrayLength() (int, error) - getCompactArrayLength() (int, error) - getBool() (bool, error) - getEmptyTaggedFieldArray() (int, error) - - // Collections - getBytes() ([]byte, error) - getVarintBytes() ([]byte, error) - getRawBytes(length int) ([]byte, error) - getString() (string, error) - getNullableString() (*string, error) - getCompactString() (string, error) - getCompactNullableString() (*string, error) - getCompactInt32Array() ([]int32, error) - getInt32Array() ([]int32, error) - getInt64Array() ([]int64, error) - getStringArray() ([]string, error) - - // Subsets - remaining() int - getSubset(length int) (packetDecoder, error) - peek(offset, length int) (packetDecoder, error) // similar to getSubset, but it doesn't advance the offset - peekInt8(offset int) (int8, error) // similar to peek, but just one byte - - // Stacks, see PushDecoder - push(in pushDecoder) error - pop() error -} - -// PushDecoder is the interface for decoding fields like CRCs and lengths where the validity -// of the field depends on what is after it in the packet. Start them with PacketDecoder.Push() where -// the actual value is located in the packet, then PacketDecoder.Pop() them when all the bytes they -// depend upon have been decoded. -type pushDecoder interface { - // Saves the offset into the input buffer as the location to actually read the calculated value when able. - saveOffset(in int) - - // Returns the length of data to reserve for the input of this encoder (eg 4 bytes for a CRC32). - reserveLength() int - - // Indicates that all required data is now available to calculate and check the field. - // SaveOffset is guaranteed to have been called first. The implementation should read ReserveLength() bytes - // of data from the saved offset, and verify it based on the data between the saved offset and curOffset. - check(curOffset int, buf []byte) error -} - -// dynamicPushDecoder extends the interface of pushDecoder for uses cases where the length of the -// fields itself is unknown until its value was decoded (for instance varint encoded length -// fields). -// During push, dynamicPushDecoder.decode() method will be called instead of reserveLength() -type dynamicPushDecoder interface { - pushDecoder - decoder -} diff --git a/vendor/github.com/Shopify/sarama/packet_encoder.go b/vendor/github.com/Shopify/sarama/packet_encoder.go deleted file mode 100644 index 50c735c0..00000000 --- a/vendor/github.com/Shopify/sarama/packet_encoder.go +++ /dev/null @@ -1,72 +0,0 @@ -package sarama - -import "github.com/rcrowley/go-metrics" - -// PacketEncoder is the interface providing helpers for writing with Kafka's encoding rules. -// Types implementing Encoder only need to worry about calling methods like PutString, -// not about how a string is represented in Kafka. -type packetEncoder interface { - // Primitives - putInt8(in int8) - putInt16(in int16) - putInt32(in int32) - putInt64(in int64) - putVarint(in int64) - putUVarint(in uint64) - putCompactArrayLength(in int) - putArrayLength(in int) error - putBool(in bool) - - // Collections - putBytes(in []byte) error - putVarintBytes(in []byte) error - putRawBytes(in []byte) error - putCompactString(in string) error - putNullableCompactString(in *string) error - putString(in string) error - putNullableString(in *string) error - putStringArray(in []string) error - putCompactInt32Array(in []int32) error - putNullableCompactInt32Array(in []int32) error - putInt32Array(in []int32) error - putInt64Array(in []int64) error - putEmptyTaggedFieldArray() - - // Provide the current offset to record the batch size metric - offset() int - - // Stacks, see PushEncoder - push(in pushEncoder) - pop() error - - // To record metrics when provided - metricRegistry() metrics.Registry -} - -// PushEncoder is the interface for encoding fields like CRCs and lengths where the value -// of the field depends on what is encoded after it in the packet. Start them with PacketEncoder.Push() where -// the actual value is located in the packet, then PacketEncoder.Pop() them when all the bytes they -// depend upon have been written. -type pushEncoder interface { - // Saves the offset into the input buffer as the location to actually write the calculated value when able. - saveOffset(in int) - - // Returns the length of data to reserve for the output of this encoder (eg 4 bytes for a CRC32). - reserveLength() int - - // Indicates that all required data is now available to calculate and write the field. - // SaveOffset is guaranteed to have been called first. The implementation should write ReserveLength() bytes - // of data to the saved offset, based on the data between the saved offset and curOffset. - run(curOffset int, buf []byte) error -} - -// dynamicPushEncoder extends the interface of pushEncoder for uses cases where the length of the -// fields itself is unknown until its value was computed (for instance varint encoded length -// fields). -type dynamicPushEncoder interface { - pushEncoder - - // Called during pop() to adjust the length of the field. - // It should return the difference in bytes between the last computed length and current length. - adjustLength(currOffset int) int -} diff --git a/vendor/github.com/Shopify/sarama/partitioner.go b/vendor/github.com/Shopify/sarama/partitioner.go deleted file mode 100644 index 6a708e72..00000000 --- a/vendor/github.com/Shopify/sarama/partitioner.go +++ /dev/null @@ -1,217 +0,0 @@ -package sarama - -import ( - "hash" - "hash/fnv" - "math/rand" - "time" -) - -// Partitioner is anything that, given a Kafka message and a number of partitions indexed [0...numPartitions-1], -// decides to which partition to send the message. RandomPartitioner, RoundRobinPartitioner and HashPartitioner are provided -// as simple default implementations. -type Partitioner interface { - // Partition takes a message and partition count and chooses a partition - Partition(message *ProducerMessage, numPartitions int32) (int32, error) - - // RequiresConsistency indicates to the user of the partitioner whether the - // mapping of key->partition is consistent or not. Specifically, if a - // partitioner requires consistency then it must be allowed to choose from all - // partitions (even ones known to be unavailable), and its choice must be - // respected by the caller. The obvious example is the HashPartitioner. - RequiresConsistency() bool -} - -// DynamicConsistencyPartitioner can optionally be implemented by Partitioners -// in order to allow more flexibility than is originally allowed by the -// RequiresConsistency method in the Partitioner interface. This allows -// partitioners to require consistency sometimes, but not all times. It's useful -// for, e.g., the HashPartitioner, which does not require consistency if the -// message key is nil. -type DynamicConsistencyPartitioner interface { - Partitioner - - // MessageRequiresConsistency is similar to Partitioner.RequiresConsistency, - // but takes in the message being partitioned so that the partitioner can - // make a per-message determination. - MessageRequiresConsistency(message *ProducerMessage) bool -} - -// PartitionerConstructor is the type for a function capable of constructing new Partitioners. -type PartitionerConstructor func(topic string) Partitioner - -type manualPartitioner struct{} - -// HashPartitionOption lets you modify default values of the partitioner -type HashPartitionerOption func(*hashPartitioner) - -// WithAbsFirst means that the partitioner handles absolute values -// in the same way as the reference Java implementation -func WithAbsFirst() HashPartitionerOption { - return func(hp *hashPartitioner) { - hp.referenceAbs = true - } -} - -// WithCustomHashFunction lets you specify what hash function to use for the partitioning -func WithCustomHashFunction(hasher func() hash.Hash32) HashPartitionerOption { - return func(hp *hashPartitioner) { - hp.hasher = hasher() - } -} - -// WithCustomFallbackPartitioner lets you specify what HashPartitioner should be used in case a Distribution Key is empty -func WithCustomFallbackPartitioner(randomHP *hashPartitioner) HashPartitionerOption { - return func(hp *hashPartitioner) { - hp.random = hp - } -} - -// NewManualPartitioner returns a Partitioner which uses the partition manually set in the provided -// ProducerMessage's Partition field as the partition to produce to. -func NewManualPartitioner(topic string) Partitioner { - return new(manualPartitioner) -} - -func (p *manualPartitioner) Partition(message *ProducerMessage, numPartitions int32) (int32, error) { - return message.Partition, nil -} - -func (p *manualPartitioner) RequiresConsistency() bool { - return true -} - -type randomPartitioner struct { - generator *rand.Rand -} - -// NewRandomPartitioner returns a Partitioner which chooses a random partition each time. -func NewRandomPartitioner(topic string) Partitioner { - p := new(randomPartitioner) - p.generator = rand.New(rand.NewSource(time.Now().UTC().UnixNano())) - return p -} - -func (p *randomPartitioner) Partition(message *ProducerMessage, numPartitions int32) (int32, error) { - return int32(p.generator.Intn(int(numPartitions))), nil -} - -func (p *randomPartitioner) RequiresConsistency() bool { - return false -} - -type roundRobinPartitioner struct { - partition int32 -} - -// NewRoundRobinPartitioner returns a Partitioner which walks through the available partitions one at a time. -func NewRoundRobinPartitioner(topic string) Partitioner { - return &roundRobinPartitioner{} -} - -func (p *roundRobinPartitioner) Partition(message *ProducerMessage, numPartitions int32) (int32, error) { - if p.partition >= numPartitions { - p.partition = 0 - } - ret := p.partition - p.partition++ - return ret, nil -} - -func (p *roundRobinPartitioner) RequiresConsistency() bool { - return false -} - -type hashPartitioner struct { - random Partitioner - hasher hash.Hash32 - referenceAbs bool -} - -// NewCustomHashPartitioner is a wrapper around NewHashPartitioner, allowing the use of custom hasher. -// The argument is a function providing the instance, implementing the hash.Hash32 interface. This is to ensure that -// each partition dispatcher gets its own hasher, to avoid concurrency issues by sharing an instance. -func NewCustomHashPartitioner(hasher func() hash.Hash32) PartitionerConstructor { - return func(topic string) Partitioner { - p := new(hashPartitioner) - p.random = NewRandomPartitioner(topic) - p.hasher = hasher() - p.referenceAbs = false - return p - } -} - -// NewCustomPartitioner creates a default Partitioner but lets you specify the behavior of each component via options -func NewCustomPartitioner(options ...HashPartitionerOption) PartitionerConstructor { - return func(topic string) Partitioner { - p := new(hashPartitioner) - p.random = NewRandomPartitioner(topic) - p.hasher = fnv.New32a() - p.referenceAbs = false - for _, option := range options { - option(p) - } - return p - } -} - -// NewHashPartitioner returns a Partitioner which behaves as follows. If the message's key is nil then a -// random partition is chosen. Otherwise the FNV-1a hash of the encoded bytes of the message key is used, -// modulus the number of partitions. This ensures that messages with the same key always end up on the -// same partition. -func NewHashPartitioner(topic string) Partitioner { - p := new(hashPartitioner) - p.random = NewRandomPartitioner(topic) - p.hasher = fnv.New32a() - p.referenceAbs = false - return p -} - -// NewReferenceHashPartitioner is like NewHashPartitioner except that it handles absolute values -// in the same way as the reference Java implementation. NewHashPartitioner was supposed to do -// that but it had a mistake and now there are people depending on both behaviours. This will -// all go away on the next major version bump. -func NewReferenceHashPartitioner(topic string) Partitioner { - p := new(hashPartitioner) - p.random = NewRandomPartitioner(topic) - p.hasher = fnv.New32a() - p.referenceAbs = true - return p -} - -func (p *hashPartitioner) Partition(message *ProducerMessage, numPartitions int32) (int32, error) { - if message.Key == nil { - return p.random.Partition(message, numPartitions) - } - bytes, err := message.Key.Encode() - if err != nil { - return -1, err - } - p.hasher.Reset() - _, err = p.hasher.Write(bytes) - if err != nil { - return -1, err - } - var partition int32 - // Turns out we were doing our absolute value in a subtly different way from the upstream - // implementation, but now we need to maintain backwards compat for people who started using - // the old version; if referenceAbs is set we are compatible with the reference java client - // but not past Sarama versions - if p.referenceAbs { - partition = (int32(p.hasher.Sum32()) & 0x7fffffff) % numPartitions - } else { - partition = int32(p.hasher.Sum32()) % numPartitions - if partition < 0 { - partition = -partition - } - } - return partition, nil -} - -func (p *hashPartitioner) RequiresConsistency() bool { - return true -} - -func (p *hashPartitioner) MessageRequiresConsistency(message *ProducerMessage) bool { - return message.Key != nil -} diff --git a/vendor/github.com/Shopify/sarama/prep_encoder.go b/vendor/github.com/Shopify/sarama/prep_encoder.go deleted file mode 100644 index 827542c5..00000000 --- a/vendor/github.com/Shopify/sarama/prep_encoder.go +++ /dev/null @@ -1,202 +0,0 @@ -package sarama - -import ( - "encoding/binary" - "errors" - "fmt" - "math" - - "github.com/rcrowley/go-metrics" -) - -type prepEncoder struct { - stack []pushEncoder - length int -} - -// primitives - -func (pe *prepEncoder) putInt8(in int8) { - pe.length++ -} - -func (pe *prepEncoder) putInt16(in int16) { - pe.length += 2 -} - -func (pe *prepEncoder) putInt32(in int32) { - pe.length += 4 -} - -func (pe *prepEncoder) putInt64(in int64) { - pe.length += 8 -} - -func (pe *prepEncoder) putVarint(in int64) { - var buf [binary.MaxVarintLen64]byte - pe.length += binary.PutVarint(buf[:], in) -} - -func (pe *prepEncoder) putUVarint(in uint64) { - var buf [binary.MaxVarintLen64]byte - pe.length += binary.PutUvarint(buf[:], in) -} - -func (pe *prepEncoder) putArrayLength(in int) error { - if in > math.MaxInt32 { - return PacketEncodingError{fmt.Sprintf("array too long (%d)", in)} - } - pe.length += 4 - return nil -} - -func (pe *prepEncoder) putCompactArrayLength(in int) { - pe.putUVarint(uint64(in + 1)) -} - -func (pe *prepEncoder) putBool(in bool) { - pe.length++ -} - -// arrays - -func (pe *prepEncoder) putBytes(in []byte) error { - pe.length += 4 - if in == nil { - return nil - } - return pe.putRawBytes(in) -} - -func (pe *prepEncoder) putVarintBytes(in []byte) error { - if in == nil { - pe.putVarint(-1) - return nil - } - pe.putVarint(int64(len(in))) - return pe.putRawBytes(in) -} - -func (pe *prepEncoder) putCompactString(in string) error { - pe.putCompactArrayLength(len(in)) - return pe.putRawBytes([]byte(in)) -} - -func (pe *prepEncoder) putNullableCompactString(in *string) error { - if in == nil { - pe.putUVarint(0) - return nil - } else { - return pe.putCompactString(*in) - } -} - -func (pe *prepEncoder) putRawBytes(in []byte) error { - if len(in) > math.MaxInt32 { - return PacketEncodingError{fmt.Sprintf("byteslice too long (%d)", len(in))} - } - pe.length += len(in) - return nil -} - -func (pe *prepEncoder) putNullableString(in *string) error { - if in == nil { - pe.length += 2 - return nil - } - return pe.putString(*in) -} - -func (pe *prepEncoder) putString(in string) error { - pe.length += 2 - if len(in) > math.MaxInt16 { - return PacketEncodingError{fmt.Sprintf("string too long (%d)", len(in))} - } - pe.length += len(in) - return nil -} - -func (pe *prepEncoder) putStringArray(in []string) error { - err := pe.putArrayLength(len(in)) - if err != nil { - return err - } - - for _, str := range in { - if err := pe.putString(str); err != nil { - return err - } - } - - return nil -} - -func (pe *prepEncoder) putCompactInt32Array(in []int32) error { - if in == nil { - return errors.New("expected int32 array to be non null") - } - - pe.putUVarint(uint64(len(in)) + 1) - pe.length += 4 * len(in) - return nil -} - -func (pe *prepEncoder) putNullableCompactInt32Array(in []int32) error { - if in == nil { - pe.putUVarint(0) - return nil - } - - pe.putUVarint(uint64(len(in)) + 1) - pe.length += 4 * len(in) - return nil -} - -func (pe *prepEncoder) putInt32Array(in []int32) error { - err := pe.putArrayLength(len(in)) - if err != nil { - return err - } - pe.length += 4 * len(in) - return nil -} - -func (pe *prepEncoder) putInt64Array(in []int64) error { - err := pe.putArrayLength(len(in)) - if err != nil { - return err - } - pe.length += 8 * len(in) - return nil -} - -func (pe *prepEncoder) putEmptyTaggedFieldArray() { - pe.putUVarint(0) -} - -func (pe *prepEncoder) offset() int { - return pe.length -} - -// stackable - -func (pe *prepEncoder) push(in pushEncoder) { - in.saveOffset(pe.length) - pe.length += in.reserveLength() - pe.stack = append(pe.stack, in) -} - -func (pe *prepEncoder) pop() error { - in := pe.stack[len(pe.stack)-1] - pe.stack = pe.stack[:len(pe.stack)-1] - if dpe, ok := in.(dynamicPushEncoder); ok { - pe.length += dpe.adjustLength(pe.length) - } - - return nil -} - -// we do not record metrics during the prep encoder pass -func (pe *prepEncoder) metricRegistry() metrics.Registry { - return nil -} diff --git a/vendor/github.com/Shopify/sarama/produce_request.go b/vendor/github.com/Shopify/sarama/produce_request.go deleted file mode 100644 index 0034651e..00000000 --- a/vendor/github.com/Shopify/sarama/produce_request.go +++ /dev/null @@ -1,258 +0,0 @@ -package sarama - -import "github.com/rcrowley/go-metrics" - -// RequiredAcks is used in Produce Requests to tell the broker how many replica acknowledgements -// it must see before responding. Any of the constants defined here are valid. On broker versions -// prior to 0.8.2.0 any other positive int16 is also valid (the broker will wait for that many -// acknowledgements) but in 0.8.2.0 and later this will raise an exception (it has been replaced -// by setting the `min.isr` value in the brokers configuration). -type RequiredAcks int16 - -const ( - // NoResponse doesn't send any response, the TCP ACK is all you get. - NoResponse RequiredAcks = 0 - // WaitForLocal waits for only the local commit to succeed before responding. - WaitForLocal RequiredAcks = 1 - // WaitForAll waits for all in-sync replicas to commit before responding. - // The minimum number of in-sync replicas is configured on the broker via - // the `min.insync.replicas` configuration key. - WaitForAll RequiredAcks = -1 -) - -type ProduceRequest struct { - TransactionalID *string - RequiredAcks RequiredAcks - Timeout int32 - Version int16 // v1 requires Kafka 0.9, v2 requires Kafka 0.10, v3 requires Kafka 0.11 - records map[string]map[int32]Records -} - -func updateMsgSetMetrics(msgSet *MessageSet, compressionRatioMetric metrics.Histogram, - topicCompressionRatioMetric metrics.Histogram) int64 { - var topicRecordCount int64 - for _, messageBlock := range msgSet.Messages { - // Is this a fake "message" wrapping real messages? - if messageBlock.Msg.Set != nil { - topicRecordCount += int64(len(messageBlock.Msg.Set.Messages)) - } else { - // A single uncompressed message - topicRecordCount++ - } - // Better be safe than sorry when computing the compression ratio - if messageBlock.Msg.compressedSize != 0 { - compressionRatio := float64(len(messageBlock.Msg.Value)) / - float64(messageBlock.Msg.compressedSize) - // Histogram do not support decimal values, let's multiple it by 100 for better precision - intCompressionRatio := int64(100 * compressionRatio) - compressionRatioMetric.Update(intCompressionRatio) - topicCompressionRatioMetric.Update(intCompressionRatio) - } - } - return topicRecordCount -} - -func updateBatchMetrics(recordBatch *RecordBatch, compressionRatioMetric metrics.Histogram, - topicCompressionRatioMetric metrics.Histogram) int64 { - if recordBatch.compressedRecords != nil { - compressionRatio := int64(float64(recordBatch.recordsLen) / float64(len(recordBatch.compressedRecords)) * 100) - compressionRatioMetric.Update(compressionRatio) - topicCompressionRatioMetric.Update(compressionRatio) - } - - return int64(len(recordBatch.Records)) -} - -func (r *ProduceRequest) encode(pe packetEncoder) error { - if r.Version >= 3 { - if err := pe.putNullableString(r.TransactionalID); err != nil { - return err - } - } - pe.putInt16(int16(r.RequiredAcks)) - pe.putInt32(r.Timeout) - metricRegistry := pe.metricRegistry() - var batchSizeMetric metrics.Histogram - var compressionRatioMetric metrics.Histogram - if metricRegistry != nil { - batchSizeMetric = getOrRegisterHistogram("batch-size", metricRegistry) - compressionRatioMetric = getOrRegisterHistogram("compression-ratio", metricRegistry) - } - totalRecordCount := int64(0) - - err := pe.putArrayLength(len(r.records)) - if err != nil { - return err - } - - for topic, partitions := range r.records { - err = pe.putString(topic) - if err != nil { - return err - } - err = pe.putArrayLength(len(partitions)) - if err != nil { - return err - } - topicRecordCount := int64(0) - var topicCompressionRatioMetric metrics.Histogram - if metricRegistry != nil { - topicCompressionRatioMetric = getOrRegisterTopicHistogram("compression-ratio", topic, metricRegistry) - } - for id, records := range partitions { - startOffset := pe.offset() - pe.putInt32(id) - pe.push(&lengthField{}) - err = records.encode(pe) - if err != nil { - return err - } - err = pe.pop() - if err != nil { - return err - } - if metricRegistry != nil { - if r.Version >= 3 { - topicRecordCount += updateBatchMetrics(records.RecordBatch, compressionRatioMetric, topicCompressionRatioMetric) - } else { - topicRecordCount += updateMsgSetMetrics(records.MsgSet, compressionRatioMetric, topicCompressionRatioMetric) - } - batchSize := int64(pe.offset() - startOffset) - batchSizeMetric.Update(batchSize) - getOrRegisterTopicHistogram("batch-size", topic, metricRegistry).Update(batchSize) - } - } - if topicRecordCount > 0 { - getOrRegisterTopicMeter("record-send-rate", topic, metricRegistry).Mark(topicRecordCount) - getOrRegisterTopicHistogram("records-per-request", topic, metricRegistry).Update(topicRecordCount) - totalRecordCount += topicRecordCount - } - } - if totalRecordCount > 0 { - metrics.GetOrRegisterMeter("record-send-rate", metricRegistry).Mark(totalRecordCount) - getOrRegisterHistogram("records-per-request", metricRegistry).Update(totalRecordCount) - } - - return nil -} - -func (r *ProduceRequest) decode(pd packetDecoder, version int16) error { - r.Version = version - - if version >= 3 { - id, err := pd.getNullableString() - if err != nil { - return err - } - r.TransactionalID = id - } - requiredAcks, err := pd.getInt16() - if err != nil { - return err - } - r.RequiredAcks = RequiredAcks(requiredAcks) - if r.Timeout, err = pd.getInt32(); err != nil { - return err - } - topicCount, err := pd.getArrayLength() - if err != nil { - return err - } - if topicCount == 0 { - return nil - } - - r.records = make(map[string]map[int32]Records) - for i := 0; i < topicCount; i++ { - topic, err := pd.getString() - if err != nil { - return err - } - partitionCount, err := pd.getArrayLength() - if err != nil { - return err - } - r.records[topic] = make(map[int32]Records) - - for j := 0; j < partitionCount; j++ { - partition, err := pd.getInt32() - if err != nil { - return err - } - size, err := pd.getInt32() - if err != nil { - return err - } - recordsDecoder, err := pd.getSubset(int(size)) - if err != nil { - return err - } - var records Records - if err := records.decode(recordsDecoder); err != nil { - return err - } - r.records[topic][partition] = records - } - } - - return nil -} - -func (r *ProduceRequest) key() int16 { - return 0 -} - -func (r *ProduceRequest) version() int16 { - return r.Version -} - -func (r *ProduceRequest) headerVersion() int16 { - return 1 -} - -func (r *ProduceRequest) requiredVersion() KafkaVersion { - switch r.Version { - case 1: - return V0_9_0_0 - case 2: - return V0_10_0_0 - case 3: - return V0_11_0_0 - case 7: - return V2_1_0_0 - default: - return MinVersion - } -} - -func (r *ProduceRequest) ensureRecords(topic string, partition int32) { - if r.records == nil { - r.records = make(map[string]map[int32]Records) - } - - if r.records[topic] == nil { - r.records[topic] = make(map[int32]Records) - } -} - -func (r *ProduceRequest) AddMessage(topic string, partition int32, msg *Message) { - r.ensureRecords(topic, partition) - set := r.records[topic][partition].MsgSet - - if set == nil { - set = new(MessageSet) - r.records[topic][partition] = newLegacyRecords(set) - } - - set.addMessage(msg) -} - -func (r *ProduceRequest) AddSet(topic string, partition int32, set *MessageSet) { - r.ensureRecords(topic, partition) - r.records[topic][partition] = newLegacyRecords(set) -} - -func (r *ProduceRequest) AddBatch(topic string, partition int32, batch *RecordBatch) { - r.ensureRecords(topic, partition) - r.records[topic][partition] = newDefaultRecords(batch) -} diff --git a/vendor/github.com/Shopify/sarama/produce_response.go b/vendor/github.com/Shopify/sarama/produce_response.go deleted file mode 100644 index edf97879..00000000 --- a/vendor/github.com/Shopify/sarama/produce_response.go +++ /dev/null @@ -1,212 +0,0 @@ -package sarama - -import ( - "fmt" - "time" -) - -// Protocol, http://kafka.apache.org/protocol.html -// v1 -// v2 = v3 = v4 -// v5 = v6 = v7 -// Produce Response (Version: 7) => [responses] throttle_time_ms -// responses => topic [partition_responses] -// topic => STRING -// partition_responses => partition error_code base_offset log_append_time log_start_offset -// partition => INT32 -// error_code => INT16 -// base_offset => INT64 -// log_append_time => INT64 -// log_start_offset => INT64 -// throttle_time_ms => INT32 - -// partition_responses in protocol -type ProduceResponseBlock struct { - Err KError // v0, error_code - Offset int64 // v0, base_offset - Timestamp time.Time // v2, log_append_time, and the broker is configured with `LogAppendTime` - StartOffset int64 // v5, log_start_offset -} - -func (b *ProduceResponseBlock) decode(pd packetDecoder, version int16) (err error) { - tmp, err := pd.getInt16() - if err != nil { - return err - } - b.Err = KError(tmp) - - b.Offset, err = pd.getInt64() - if err != nil { - return err - } - - if version >= 2 { - if millis, err := pd.getInt64(); err != nil { - return err - } else if millis != -1 { - b.Timestamp = time.Unix(millis/1000, (millis%1000)*int64(time.Millisecond)) - } - } - - if version >= 5 { - b.StartOffset, err = pd.getInt64() - if err != nil { - return err - } - } - - return nil -} - -func (b *ProduceResponseBlock) encode(pe packetEncoder, version int16) (err error) { - pe.putInt16(int16(b.Err)) - pe.putInt64(b.Offset) - - if version >= 2 { - timestamp := int64(-1) - if !b.Timestamp.Before(time.Unix(0, 0)) { - timestamp = b.Timestamp.UnixNano() / int64(time.Millisecond) - } else if !b.Timestamp.IsZero() { - return PacketEncodingError{fmt.Sprintf("invalid timestamp (%v)", b.Timestamp)} - } - pe.putInt64(timestamp) - } - - if version >= 5 { - pe.putInt64(b.StartOffset) - } - - return nil -} - -type ProduceResponse struct { - Blocks map[string]map[int32]*ProduceResponseBlock // v0, responses - Version int16 - ThrottleTime time.Duration // v1, throttle_time_ms -} - -func (r *ProduceResponse) decode(pd packetDecoder, version int16) (err error) { - r.Version = version - - numTopics, err := pd.getArrayLength() - if err != nil { - return err - } - - r.Blocks = make(map[string]map[int32]*ProduceResponseBlock, numTopics) - for i := 0; i < numTopics; i++ { - name, err := pd.getString() - if err != nil { - return err - } - - numBlocks, err := pd.getArrayLength() - if err != nil { - return err - } - - r.Blocks[name] = make(map[int32]*ProduceResponseBlock, numBlocks) - - for j := 0; j < numBlocks; j++ { - id, err := pd.getInt32() - if err != nil { - return err - } - - block := new(ProduceResponseBlock) - err = block.decode(pd, version) - if err != nil { - return err - } - r.Blocks[name][id] = block - } - } - - if r.Version >= 1 { - millis, err := pd.getInt32() - if err != nil { - return err - } - - r.ThrottleTime = time.Duration(millis) * time.Millisecond - } - - return nil -} - -func (r *ProduceResponse) encode(pe packetEncoder) error { - err := pe.putArrayLength(len(r.Blocks)) - if err != nil { - return err - } - for topic, partitions := range r.Blocks { - err = pe.putString(topic) - if err != nil { - return err - } - err = pe.putArrayLength(len(partitions)) - if err != nil { - return err - } - for id, prb := range partitions { - pe.putInt32(id) - err = prb.encode(pe, r.Version) - if err != nil { - return err - } - } - } - - if r.Version >= 1 { - pe.putInt32(int32(r.ThrottleTime / time.Millisecond)) - } - return nil -} - -func (r *ProduceResponse) key() int16 { - return 0 -} - -func (r *ProduceResponse) version() int16 { - return r.Version -} - -func (r *ProduceResponse) headerVersion() int16 { - return 0 -} - -func (r *ProduceResponse) requiredVersion() KafkaVersion { - return MinVersion -} - -func (r *ProduceResponse) GetBlock(topic string, partition int32) *ProduceResponseBlock { - if r.Blocks == nil { - return nil - } - - if r.Blocks[topic] == nil { - return nil - } - - return r.Blocks[topic][partition] -} - -// Testing API - -func (r *ProduceResponse) AddTopicPartition(topic string, partition int32, err KError) { - if r.Blocks == nil { - r.Blocks = make(map[string]map[int32]*ProduceResponseBlock) - } - byTopic, ok := r.Blocks[topic] - if !ok { - byTopic = make(map[int32]*ProduceResponseBlock) - r.Blocks[topic] = byTopic - } - block := &ProduceResponseBlock{ - Err: err, - } - if r.Version >= 2 { - block.Timestamp = time.Now() - } - byTopic[partition] = block -} diff --git a/vendor/github.com/Shopify/sarama/produce_set.go b/vendor/github.com/Shopify/sarama/produce_set.go deleted file mode 100644 index 9c70f818..00000000 --- a/vendor/github.com/Shopify/sarama/produce_set.go +++ /dev/null @@ -1,273 +0,0 @@ -package sarama - -import ( - "encoding/binary" - "errors" - "time" -) - -type partitionSet struct { - msgs []*ProducerMessage - recordsToSend Records - bufferBytes int -} - -type produceSet struct { - parent *asyncProducer - msgs map[string]map[int32]*partitionSet - producerID int64 - producerEpoch int16 - - bufferBytes int - bufferCount int -} - -func newProduceSet(parent *asyncProducer) *produceSet { - pid, epoch := parent.txnmgr.getProducerID() - return &produceSet{ - msgs: make(map[string]map[int32]*partitionSet), - parent: parent, - producerID: pid, - producerEpoch: epoch, - } -} - -func (ps *produceSet) add(msg *ProducerMessage) error { - var err error - var key, val []byte - - if msg.Key != nil { - if key, err = msg.Key.Encode(); err != nil { - return err - } - } - - if msg.Value != nil { - if val, err = msg.Value.Encode(); err != nil { - return err - } - } - - timestamp := msg.Timestamp - if timestamp.IsZero() { - timestamp = time.Now() - } - timestamp = timestamp.Truncate(time.Millisecond) - - partitions := ps.msgs[msg.Topic] - if partitions == nil { - partitions = make(map[int32]*partitionSet) - ps.msgs[msg.Topic] = partitions - } - - var size int - - set := partitions[msg.Partition] - if set == nil { - if ps.parent.conf.Version.IsAtLeast(V0_11_0_0) { - batch := &RecordBatch{ - FirstTimestamp: timestamp, - Version: 2, - Codec: ps.parent.conf.Producer.Compression, - CompressionLevel: ps.parent.conf.Producer.CompressionLevel, - ProducerID: ps.producerID, - ProducerEpoch: ps.producerEpoch, - } - if ps.parent.conf.Producer.Idempotent { - batch.FirstSequence = msg.sequenceNumber - } - set = &partitionSet{recordsToSend: newDefaultRecords(batch)} - size = recordBatchOverhead - } else { - set = &partitionSet{recordsToSend: newLegacyRecords(new(MessageSet))} - } - partitions[msg.Partition] = set - } - - if ps.parent.conf.Version.IsAtLeast(V0_11_0_0) { - if ps.parent.conf.Producer.Idempotent && msg.sequenceNumber < set.recordsToSend.RecordBatch.FirstSequence { - return errors.New("assertion failed: message out of sequence added to a batch") - } - } - - // Past this point we can't return an error, because we've already added the message to the set. - set.msgs = append(set.msgs, msg) - - if ps.parent.conf.Version.IsAtLeast(V0_11_0_0) { - // We are being conservative here to avoid having to prep encode the record - size += maximumRecordOverhead - rec := &Record{ - Key: key, - Value: val, - TimestampDelta: timestamp.Sub(set.recordsToSend.RecordBatch.FirstTimestamp), - } - size += len(key) + len(val) - if len(msg.Headers) > 0 { - rec.Headers = make([]*RecordHeader, len(msg.Headers)) - for i := range msg.Headers { - rec.Headers[i] = &msg.Headers[i] - size += len(rec.Headers[i].Key) + len(rec.Headers[i].Value) + 2*binary.MaxVarintLen32 - } - } - set.recordsToSend.RecordBatch.addRecord(rec) - } else { - msgToSend := &Message{Codec: CompressionNone, Key: key, Value: val} - if ps.parent.conf.Version.IsAtLeast(V0_10_0_0) { - msgToSend.Timestamp = timestamp - msgToSend.Version = 1 - } - set.recordsToSend.MsgSet.addMessage(msgToSend) - size = producerMessageOverhead + len(key) + len(val) - } - - set.bufferBytes += size - ps.bufferBytes += size - ps.bufferCount++ - - return nil -} - -func (ps *produceSet) buildRequest() *ProduceRequest { - req := &ProduceRequest{ - RequiredAcks: ps.parent.conf.Producer.RequiredAcks, - Timeout: int32(ps.parent.conf.Producer.Timeout / time.Millisecond), - } - if ps.parent.conf.Version.IsAtLeast(V0_10_0_0) { - req.Version = 2 - } - if ps.parent.conf.Version.IsAtLeast(V0_11_0_0) { - req.Version = 3 - } - - if ps.parent.conf.Producer.Compression == CompressionZSTD && ps.parent.conf.Version.IsAtLeast(V2_1_0_0) { - req.Version = 7 - } - - for topic, partitionSets := range ps.msgs { - for partition, set := range partitionSets { - if req.Version >= 3 { - // If the API version we're hitting is 3 or greater, we need to calculate - // offsets for each record in the batch relative to FirstOffset. - // Additionally, we must set LastOffsetDelta to the value of the last offset - // in the batch. Since the OffsetDelta of the first record is 0, we know that the - // final record of any batch will have an offset of (# of records in batch) - 1. - // (See https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-Messagesets - // under the RecordBatch section for details.) - rb := set.recordsToSend.RecordBatch - if len(rb.Records) > 0 { - rb.LastOffsetDelta = int32(len(rb.Records) - 1) - for i, record := range rb.Records { - record.OffsetDelta = int64(i) - } - } - req.AddBatch(topic, partition, rb) - continue - } - if ps.parent.conf.Producer.Compression == CompressionNone { - req.AddSet(topic, partition, set.recordsToSend.MsgSet) - } else { - // When compression is enabled, the entire set for each partition is compressed - // and sent as the payload of a single fake "message" with the appropriate codec - // set and no key. When the server sees a message with a compression codec, it - // decompresses the payload and treats the result as its message set. - - if ps.parent.conf.Version.IsAtLeast(V0_10_0_0) { - // If our version is 0.10 or later, assign relative offsets - // to the inner messages. This lets the broker avoid - // recompressing the message set. - // (See https://cwiki.apache.org/confluence/display/KAFKA/KIP-31+-+Move+to+relative+offsets+in+compressed+message+sets - // for details on relative offsets.) - for i, msg := range set.recordsToSend.MsgSet.Messages { - msg.Offset = int64(i) - } - } - payload, err := encode(set.recordsToSend.MsgSet, ps.parent.conf.MetricRegistry) - if err != nil { - Logger.Println(err) // if this happens, it's basically our fault. - panic(err) - } - compMsg := &Message{ - Codec: ps.parent.conf.Producer.Compression, - CompressionLevel: ps.parent.conf.Producer.CompressionLevel, - Key: nil, - Value: payload, - Set: set.recordsToSend.MsgSet, // Provide the underlying message set for accurate metrics - } - if ps.parent.conf.Version.IsAtLeast(V0_10_0_0) { - compMsg.Version = 1 - compMsg.Timestamp = set.recordsToSend.MsgSet.Messages[0].Msg.Timestamp - } - req.AddMessage(topic, partition, compMsg) - } - } - } - - return req -} - -func (ps *produceSet) eachPartition(cb func(topic string, partition int32, pSet *partitionSet)) { - for topic, partitionSet := range ps.msgs { - for partition, set := range partitionSet { - cb(topic, partition, set) - } - } -} - -func (ps *produceSet) dropPartition(topic string, partition int32) []*ProducerMessage { - if ps.msgs[topic] == nil { - return nil - } - set := ps.msgs[topic][partition] - if set == nil { - return nil - } - ps.bufferBytes -= set.bufferBytes - ps.bufferCount -= len(set.msgs) - delete(ps.msgs[topic], partition) - return set.msgs -} - -func (ps *produceSet) wouldOverflow(msg *ProducerMessage) bool { - version := 1 - if ps.parent.conf.Version.IsAtLeast(V0_11_0_0) { - version = 2 - } - - switch { - // Would we overflow our maximum possible size-on-the-wire? 10KiB is arbitrary overhead for safety. - case ps.bufferBytes+msg.byteSize(version) >= int(MaxRequestSize-(10*1024)): - return true - // Would we overflow the size-limit of a message-batch for this partition? - case ps.msgs[msg.Topic] != nil && ps.msgs[msg.Topic][msg.Partition] != nil && - ps.msgs[msg.Topic][msg.Partition].bufferBytes+msg.byteSize(version) >= ps.parent.conf.Producer.MaxMessageBytes: - return true - // Would we overflow simply in number of messages? - case ps.parent.conf.Producer.Flush.MaxMessages > 0 && ps.bufferCount >= ps.parent.conf.Producer.Flush.MaxMessages: - return true - default: - return false - } -} - -func (ps *produceSet) readyToFlush() bool { - switch { - // If we don't have any messages, nothing else matters - case ps.empty(): - return false - // If all three config values are 0, we always flush as-fast-as-possible - case ps.parent.conf.Producer.Flush.Frequency == 0 && ps.parent.conf.Producer.Flush.Bytes == 0 && ps.parent.conf.Producer.Flush.Messages == 0: - return true - // If we've passed the message trigger-point - case ps.parent.conf.Producer.Flush.Messages > 0 && ps.bufferCount >= ps.parent.conf.Producer.Flush.Messages: - return true - // If we've passed the byte trigger-point - case ps.parent.conf.Producer.Flush.Bytes > 0 && ps.bufferBytes >= ps.parent.conf.Producer.Flush.Bytes: - return true - default: - return false - } -} - -func (ps *produceSet) empty() bool { - return ps.bufferCount == 0 -} diff --git a/vendor/github.com/Shopify/sarama/real_decoder.go b/vendor/github.com/Shopify/sarama/real_decoder.go deleted file mode 100644 index 8ac576db..00000000 --- a/vendor/github.com/Shopify/sarama/real_decoder.go +++ /dev/null @@ -1,426 +0,0 @@ -package sarama - -import ( - "encoding/binary" - "math" -) - -var errInvalidArrayLength = PacketDecodingError{"invalid array length"} -var errInvalidByteSliceLength = PacketDecodingError{"invalid byteslice length"} -var errInvalidStringLength = PacketDecodingError{"invalid string length"} -var errVarintOverflow = PacketDecodingError{"varint overflow"} -var errUVarintOverflow = PacketDecodingError{"uvarint overflow"} -var errInvalidBool = PacketDecodingError{"invalid bool"} -var errUnsupportedTaggedFields = PacketDecodingError{"non-empty tagged fields are not supported yet"} - -type realDecoder struct { - raw []byte - off int - stack []pushDecoder -} - -// primitives - -func (rd *realDecoder) getInt8() (int8, error) { - if rd.remaining() < 1 { - rd.off = len(rd.raw) - return -1, ErrInsufficientData - } - tmp := int8(rd.raw[rd.off]) - rd.off++ - return tmp, nil -} - -func (rd *realDecoder) getInt16() (int16, error) { - if rd.remaining() < 2 { - rd.off = len(rd.raw) - return -1, ErrInsufficientData - } - tmp := int16(binary.BigEndian.Uint16(rd.raw[rd.off:])) - rd.off += 2 - return tmp, nil -} - -func (rd *realDecoder) getInt32() (int32, error) { - if rd.remaining() < 4 { - rd.off = len(rd.raw) - return -1, ErrInsufficientData - } - tmp := int32(binary.BigEndian.Uint32(rd.raw[rd.off:])) - rd.off += 4 - return tmp, nil -} - -func (rd *realDecoder) getInt64() (int64, error) { - if rd.remaining() < 8 { - rd.off = len(rd.raw) - return -1, ErrInsufficientData - } - tmp := int64(binary.BigEndian.Uint64(rd.raw[rd.off:])) - rd.off += 8 - return tmp, nil -} - -func (rd *realDecoder) getVarint() (int64, error) { - tmp, n := binary.Varint(rd.raw[rd.off:]) - if n == 0 { - rd.off = len(rd.raw) - return -1, ErrInsufficientData - } - if n < 0 { - rd.off -= n - return -1, errVarintOverflow - } - rd.off += n - return tmp, nil -} - -func (rd *realDecoder) getUVarint() (uint64, error) { - tmp, n := binary.Uvarint(rd.raw[rd.off:]) - if n == 0 { - rd.off = len(rd.raw) - return 0, ErrInsufficientData - } - - if n < 0 { - rd.off -= n - return 0, errUVarintOverflow - } - - rd.off += n - return tmp, nil -} - -func (rd *realDecoder) getArrayLength() (int, error) { - if rd.remaining() < 4 { - rd.off = len(rd.raw) - return -1, ErrInsufficientData - } - tmp := int(int32(binary.BigEndian.Uint32(rd.raw[rd.off:]))) - rd.off += 4 - if tmp > rd.remaining() { - rd.off = len(rd.raw) - return -1, ErrInsufficientData - } else if tmp > 2*math.MaxUint16 { - return -1, errInvalidArrayLength - } - return tmp, nil -} - -func (rd *realDecoder) getCompactArrayLength() (int, error) { - n, err := rd.getUVarint() - if err != nil { - return 0, err - } - - if n == 0 { - return 0, nil - } - - return int(n) - 1, nil -} - -func (rd *realDecoder) getBool() (bool, error) { - b, err := rd.getInt8() - if err != nil || b == 0 { - return false, err - } - if b != 1 { - return false, errInvalidBool - } - return true, nil -} - -func (rd *realDecoder) getEmptyTaggedFieldArray() (int, error) { - tagCount, err := rd.getUVarint() - if err != nil { - return 0, err - } - - if tagCount != 0 { - return 0, errUnsupportedTaggedFields - } - - return 0, nil -} - -// collections - -func (rd *realDecoder) getBytes() ([]byte, error) { - tmp, err := rd.getInt32() - if err != nil { - return nil, err - } - if tmp == -1 { - return nil, nil - } - - return rd.getRawBytes(int(tmp)) -} - -func (rd *realDecoder) getVarintBytes() ([]byte, error) { - tmp, err := rd.getVarint() - if err != nil { - return nil, err - } - if tmp == -1 { - return nil, nil - } - - return rd.getRawBytes(int(tmp)) -} - -func (rd *realDecoder) getStringLength() (int, error) { - length, err := rd.getInt16() - if err != nil { - return 0, err - } - - n := int(length) - - switch { - case n < -1: - return 0, errInvalidStringLength - case n > rd.remaining(): - rd.off = len(rd.raw) - return 0, ErrInsufficientData - } - - return n, nil -} - -func (rd *realDecoder) getString() (string, error) { - n, err := rd.getStringLength() - if err != nil || n == -1 { - return "", err - } - - tmpStr := string(rd.raw[rd.off : rd.off+n]) - rd.off += n - return tmpStr, nil -} - -func (rd *realDecoder) getNullableString() (*string, error) { - n, err := rd.getStringLength() - if err != nil || n == -1 { - return nil, err - } - - tmpStr := string(rd.raw[rd.off : rd.off+n]) - rd.off += n - return &tmpStr, err -} - -func (rd *realDecoder) getCompactString() (string, error) { - n, err := rd.getUVarint() - if err != nil { - return "", err - } - - var length = int(n - 1) - - tmpStr := string(rd.raw[rd.off : rd.off+length]) - rd.off += length - return tmpStr, nil -} - -func (rd *realDecoder) getCompactNullableString() (*string, error) { - n, err := rd.getUVarint() - - if err != nil { - return nil, err - } - - var length = int(n - 1) - - if length < 0 { - return nil, err - } - - tmpStr := string(rd.raw[rd.off : rd.off+length]) - rd.off += length - return &tmpStr, err -} - -func (rd *realDecoder) getCompactInt32Array() ([]int32, error) { - n, err := rd.getUVarint() - if err != nil { - return nil, err - } - - if n == 0 { - return nil, nil - } - - arrayLength := int(n) - 1 - - ret := make([]int32, arrayLength) - - for i := range ret { - ret[i] = int32(binary.BigEndian.Uint32(rd.raw[rd.off:])) - rd.off += 4 - } - return ret, nil -} - -func (rd *realDecoder) getInt32Array() ([]int32, error) { - if rd.remaining() < 4 { - rd.off = len(rd.raw) - return nil, ErrInsufficientData - } - n := int(binary.BigEndian.Uint32(rd.raw[rd.off:])) - rd.off += 4 - - if rd.remaining() < 4*n { - rd.off = len(rd.raw) - return nil, ErrInsufficientData - } - - if n == 0 { - return nil, nil - } - - if n < 0 { - return nil, errInvalidArrayLength - } - - ret := make([]int32, n) - for i := range ret { - ret[i] = int32(binary.BigEndian.Uint32(rd.raw[rd.off:])) - rd.off += 4 - } - return ret, nil -} - -func (rd *realDecoder) getInt64Array() ([]int64, error) { - if rd.remaining() < 4 { - rd.off = len(rd.raw) - return nil, ErrInsufficientData - } - n := int(binary.BigEndian.Uint32(rd.raw[rd.off:])) - rd.off += 4 - - if rd.remaining() < 8*n { - rd.off = len(rd.raw) - return nil, ErrInsufficientData - } - - if n == 0 { - return nil, nil - } - - if n < 0 { - return nil, errInvalidArrayLength - } - - ret := make([]int64, n) - for i := range ret { - ret[i] = int64(binary.BigEndian.Uint64(rd.raw[rd.off:])) - rd.off += 8 - } - return ret, nil -} - -func (rd *realDecoder) getStringArray() ([]string, error) { - if rd.remaining() < 4 { - rd.off = len(rd.raw) - return nil, ErrInsufficientData - } - n := int(binary.BigEndian.Uint32(rd.raw[rd.off:])) - rd.off += 4 - - if n == 0 { - return nil, nil - } - - if n < 0 { - return nil, errInvalidArrayLength - } - - ret := make([]string, n) - for i := range ret { - str, err := rd.getString() - if err != nil { - return nil, err - } - - ret[i] = str - } - return ret, nil -} - -// subsets - -func (rd *realDecoder) remaining() int { - return len(rd.raw) - rd.off -} - -func (rd *realDecoder) getSubset(length int) (packetDecoder, error) { - buf, err := rd.getRawBytes(length) - if err != nil { - return nil, err - } - return &realDecoder{raw: buf}, nil -} - -func (rd *realDecoder) getRawBytes(length int) ([]byte, error) { - if length < 0 { - return nil, errInvalidByteSliceLength - } else if length > rd.remaining() { - rd.off = len(rd.raw) - return nil, ErrInsufficientData - } - - start := rd.off - rd.off += length - return rd.raw[start:rd.off], nil -} - -func (rd *realDecoder) peek(offset, length int) (packetDecoder, error) { - if rd.remaining() < offset+length { - return nil, ErrInsufficientData - } - off := rd.off + offset - return &realDecoder{raw: rd.raw[off : off+length]}, nil -} - -func (rd *realDecoder) peekInt8(offset int) (int8, error) { - const byteLen = 1 - if rd.remaining() < offset+byteLen { - return -1, ErrInsufficientData - } - return int8(rd.raw[rd.off+offset]), nil -} - -// stacks - -func (rd *realDecoder) push(in pushDecoder) error { - in.saveOffset(rd.off) - - var reserve int - if dpd, ok := in.(dynamicPushDecoder); ok { - if err := dpd.decode(rd); err != nil { - return err - } - } else { - reserve = in.reserveLength() - if rd.remaining() < reserve { - rd.off = len(rd.raw) - return ErrInsufficientData - } - } - - rd.stack = append(rd.stack, in) - - rd.off += reserve - - return nil -} - -func (rd *realDecoder) pop() error { - // this is go's ugly pop pattern (the inverse of append) - in := rd.stack[len(rd.stack)-1] - rd.stack = rd.stack[:len(rd.stack)-1] - - return in.check(rd.off, rd.raw) -} diff --git a/vendor/github.com/Shopify/sarama/real_encoder.go b/vendor/github.com/Shopify/sarama/real_encoder.go deleted file mode 100644 index ba073f7d..00000000 --- a/vendor/github.com/Shopify/sarama/real_encoder.go +++ /dev/null @@ -1,208 +0,0 @@ -package sarama - -import ( - "encoding/binary" - "errors" - - "github.com/rcrowley/go-metrics" -) - -type realEncoder struct { - raw []byte - off int - stack []pushEncoder - registry metrics.Registry -} - -// primitives - -func (re *realEncoder) putInt8(in int8) { - re.raw[re.off] = byte(in) - re.off++ -} - -func (re *realEncoder) putInt16(in int16) { - binary.BigEndian.PutUint16(re.raw[re.off:], uint16(in)) - re.off += 2 -} - -func (re *realEncoder) putInt32(in int32) { - binary.BigEndian.PutUint32(re.raw[re.off:], uint32(in)) - re.off += 4 -} - -func (re *realEncoder) putInt64(in int64) { - binary.BigEndian.PutUint64(re.raw[re.off:], uint64(in)) - re.off += 8 -} - -func (re *realEncoder) putVarint(in int64) { - re.off += binary.PutVarint(re.raw[re.off:], in) -} - -func (re *realEncoder) putUVarint(in uint64) { - re.off += binary.PutUvarint(re.raw[re.off:], in) -} - -func (re *realEncoder) putArrayLength(in int) error { - re.putInt32(int32(in)) - return nil -} - -func (re *realEncoder) putCompactArrayLength(in int) { - // 0 represents a null array, so +1 has to be added - re.putUVarint(uint64(in + 1)) -} - -func (re *realEncoder) putBool(in bool) { - if in { - re.putInt8(1) - return - } - re.putInt8(0) -} - -// collection - -func (re *realEncoder) putRawBytes(in []byte) error { - copy(re.raw[re.off:], in) - re.off += len(in) - return nil -} - -func (re *realEncoder) putBytes(in []byte) error { - if in == nil { - re.putInt32(-1) - return nil - } - re.putInt32(int32(len(in))) - return re.putRawBytes(in) -} - -func (re *realEncoder) putVarintBytes(in []byte) error { - if in == nil { - re.putVarint(-1) - return nil - } - re.putVarint(int64(len(in))) - return re.putRawBytes(in) -} - -func (re *realEncoder) putCompactString(in string) error { - re.putCompactArrayLength(len(in)) - return re.putRawBytes([]byte(in)) -} - -func (re *realEncoder) putNullableCompactString(in *string) error { - if in == nil { - re.putInt8(0) - return nil - } - return re.putCompactString(*in) -} - -func (re *realEncoder) putString(in string) error { - re.putInt16(int16(len(in))) - copy(re.raw[re.off:], in) - re.off += len(in) - return nil -} - -func (re *realEncoder) putNullableString(in *string) error { - if in == nil { - re.putInt16(-1) - return nil - } - return re.putString(*in) -} - -func (re *realEncoder) putStringArray(in []string) error { - err := re.putArrayLength(len(in)) - if err != nil { - return err - } - - for _, val := range in { - if err := re.putString(val); err != nil { - return err - } - } - - return nil -} - -func (re *realEncoder) putCompactInt32Array(in []int32) error { - if in == nil { - return errors.New("expected int32 array to be non null") - } - // 0 represents a null array, so +1 has to be added - re.putUVarint(uint64(len(in)) + 1) - for _, val := range in { - re.putInt32(val) - } - return nil -} - -func (re *realEncoder) putNullableCompactInt32Array(in []int32) error { - if in == nil { - re.putUVarint(0) - return nil - } - // 0 represents a null array, so +1 has to be added - re.putUVarint(uint64(len(in)) + 1) - for _, val := range in { - re.putInt32(val) - } - return nil -} - -func (re *realEncoder) putInt32Array(in []int32) error { - err := re.putArrayLength(len(in)) - if err != nil { - return err - } - for _, val := range in { - re.putInt32(val) - } - return nil -} - -func (re *realEncoder) putInt64Array(in []int64) error { - err := re.putArrayLength(len(in)) - if err != nil { - return err - } - for _, val := range in { - re.putInt64(val) - } - return nil -} - -func (re *realEncoder) putEmptyTaggedFieldArray() { - re.putUVarint(0) -} - -func (re *realEncoder) offset() int { - return re.off -} - -// stacks - -func (re *realEncoder) push(in pushEncoder) { - in.saveOffset(re.off) - re.off += in.reserveLength() - re.stack = append(re.stack, in) -} - -func (re *realEncoder) pop() error { - // this is go's ugly pop pattern (the inverse of append) - in := re.stack[len(re.stack)-1] - re.stack = re.stack[:len(re.stack)-1] - - return in.run(re.off, re.raw) -} - -// we do record metrics during the real encoder pass -func (re *realEncoder) metricRegistry() metrics.Registry { - return re.registry -} diff --git a/vendor/github.com/Shopify/sarama/record.go b/vendor/github.com/Shopify/sarama/record.go deleted file mode 100644 index cdccfe32..00000000 --- a/vendor/github.com/Shopify/sarama/record.go +++ /dev/null @@ -1,116 +0,0 @@ -package sarama - -import ( - "encoding/binary" - "time" -) - -const ( - isTransactionalMask = 0x10 - controlMask = 0x20 - maximumRecordOverhead = 5*binary.MaxVarintLen32 + binary.MaxVarintLen64 + 1 -) - -//RecordHeader stores key and value for a record header -type RecordHeader struct { - Key []byte - Value []byte -} - -func (h *RecordHeader) encode(pe packetEncoder) error { - if err := pe.putVarintBytes(h.Key); err != nil { - return err - } - return pe.putVarintBytes(h.Value) -} - -func (h *RecordHeader) decode(pd packetDecoder) (err error) { - if h.Key, err = pd.getVarintBytes(); err != nil { - return err - } - - if h.Value, err = pd.getVarintBytes(); err != nil { - return err - } - return nil -} - -//Record is kafka record type -type Record struct { - Headers []*RecordHeader - - Attributes int8 - TimestampDelta time.Duration - OffsetDelta int64 - Key []byte - Value []byte - length varintLengthField -} - -func (r *Record) encode(pe packetEncoder) error { - pe.push(&r.length) - pe.putInt8(r.Attributes) - pe.putVarint(int64(r.TimestampDelta / time.Millisecond)) - pe.putVarint(r.OffsetDelta) - if err := pe.putVarintBytes(r.Key); err != nil { - return err - } - if err := pe.putVarintBytes(r.Value); err != nil { - return err - } - pe.putVarint(int64(len(r.Headers))) - - for _, h := range r.Headers { - if err := h.encode(pe); err != nil { - return err - } - } - - return pe.pop() -} - -func (r *Record) decode(pd packetDecoder) (err error) { - if err = pd.push(&r.length); err != nil { - return err - } - - if r.Attributes, err = pd.getInt8(); err != nil { - return err - } - - timestamp, err := pd.getVarint() - if err != nil { - return err - } - r.TimestampDelta = time.Duration(timestamp) * time.Millisecond - - if r.OffsetDelta, err = pd.getVarint(); err != nil { - return err - } - - if r.Key, err = pd.getVarintBytes(); err != nil { - return err - } - - if r.Value, err = pd.getVarintBytes(); err != nil { - return err - } - - numHeaders, err := pd.getVarint() - if err != nil { - return err - } - - if numHeaders >= 0 { - r.Headers = make([]*RecordHeader, numHeaders) - } - for i := int64(0); i < numHeaders; i++ { - hdr := new(RecordHeader) - if err := hdr.decode(pd); err != nil { - return err - } - r.Headers[i] = hdr - } - - return pd.pop() -} diff --git a/vendor/github.com/Shopify/sarama/record_batch.go b/vendor/github.com/Shopify/sarama/record_batch.go deleted file mode 100644 index c653763e..00000000 --- a/vendor/github.com/Shopify/sarama/record_batch.go +++ /dev/null @@ -1,225 +0,0 @@ -package sarama - -import ( - "fmt" - "time" -) - -const recordBatchOverhead = 49 - -type recordsArray []*Record - -func (e recordsArray) encode(pe packetEncoder) error { - for _, r := range e { - if err := r.encode(pe); err != nil { - return err - } - } - return nil -} - -func (e recordsArray) decode(pd packetDecoder) error { - for i := range e { - rec := &Record{} - if err := rec.decode(pd); err != nil { - return err - } - e[i] = rec - } - return nil -} - -type RecordBatch struct { - FirstOffset int64 - PartitionLeaderEpoch int32 - Version int8 - Codec CompressionCodec - CompressionLevel int - Control bool - LogAppendTime bool - LastOffsetDelta int32 - FirstTimestamp time.Time - MaxTimestamp time.Time - ProducerID int64 - ProducerEpoch int16 - FirstSequence int32 - Records []*Record - PartialTrailingRecord bool - IsTransactional bool - - compressedRecords []byte - recordsLen int // uncompressed records size -} - -func (b *RecordBatch) LastOffset() int64 { - return b.FirstOffset + int64(b.LastOffsetDelta) -} - -func (b *RecordBatch) encode(pe packetEncoder) error { - if b.Version != 2 { - return PacketEncodingError{fmt.Sprintf("unsupported compression codec (%d)", b.Codec)} - } - pe.putInt64(b.FirstOffset) - pe.push(&lengthField{}) - pe.putInt32(b.PartitionLeaderEpoch) - pe.putInt8(b.Version) - pe.push(newCRC32Field(crcCastagnoli)) - pe.putInt16(b.computeAttributes()) - pe.putInt32(b.LastOffsetDelta) - - if err := (Timestamp{&b.FirstTimestamp}).encode(pe); err != nil { - return err - } - - if err := (Timestamp{&b.MaxTimestamp}).encode(pe); err != nil { - return err - } - - pe.putInt64(b.ProducerID) - pe.putInt16(b.ProducerEpoch) - pe.putInt32(b.FirstSequence) - - if err := pe.putArrayLength(len(b.Records)); err != nil { - return err - } - - if b.compressedRecords == nil { - if err := b.encodeRecords(pe); err != nil { - return err - } - } - if err := pe.putRawBytes(b.compressedRecords); err != nil { - return err - } - - if err := pe.pop(); err != nil { - return err - } - return pe.pop() -} - -func (b *RecordBatch) decode(pd packetDecoder) (err error) { - if b.FirstOffset, err = pd.getInt64(); err != nil { - return err - } - - batchLen, err := pd.getInt32() - if err != nil { - return err - } - - if b.PartitionLeaderEpoch, err = pd.getInt32(); err != nil { - return err - } - - if b.Version, err = pd.getInt8(); err != nil { - return err - } - - crc32Decoder := acquireCrc32Field(crcCastagnoli) - defer releaseCrc32Field(crc32Decoder) - - if err = pd.push(crc32Decoder); err != nil { - return err - } - - attributes, err := pd.getInt16() - if err != nil { - return err - } - b.Codec = CompressionCodec(int8(attributes) & compressionCodecMask) - b.Control = attributes&controlMask == controlMask - b.LogAppendTime = attributes×tampTypeMask == timestampTypeMask - b.IsTransactional = attributes&isTransactionalMask == isTransactionalMask - - if b.LastOffsetDelta, err = pd.getInt32(); err != nil { - return err - } - - if err = (Timestamp{&b.FirstTimestamp}).decode(pd); err != nil { - return err - } - - if err = (Timestamp{&b.MaxTimestamp}).decode(pd); err != nil { - return err - } - - if b.ProducerID, err = pd.getInt64(); err != nil { - return err - } - - if b.ProducerEpoch, err = pd.getInt16(); err != nil { - return err - } - - if b.FirstSequence, err = pd.getInt32(); err != nil { - return err - } - - numRecs, err := pd.getArrayLength() - if err != nil { - return err - } - if numRecs >= 0 { - b.Records = make([]*Record, numRecs) - } - - bufSize := int(batchLen) - recordBatchOverhead - recBuffer, err := pd.getRawBytes(bufSize) - if err != nil { - if err == ErrInsufficientData { - b.PartialTrailingRecord = true - b.Records = nil - return nil - } - return err - } - - if err = pd.pop(); err != nil { - return err - } - - recBuffer, err = decompress(b.Codec, recBuffer) - if err != nil { - return err - } - - b.recordsLen = len(recBuffer) - err = decode(recBuffer, recordsArray(b.Records)) - if err == ErrInsufficientData { - b.PartialTrailingRecord = true - b.Records = nil - return nil - } - return err -} - -func (b *RecordBatch) encodeRecords(pe packetEncoder) error { - var raw []byte - var err error - if raw, err = encode(recordsArray(b.Records), pe.metricRegistry()); err != nil { - return err - } - b.recordsLen = len(raw) - - b.compressedRecords, err = compress(b.Codec, b.CompressionLevel, raw) - return err -} - -func (b *RecordBatch) computeAttributes() int16 { - attr := int16(b.Codec) & int16(compressionCodecMask) - if b.Control { - attr |= controlMask - } - if b.LogAppendTime { - attr |= timestampTypeMask - } - if b.IsTransactional { - attr |= isTransactionalMask - } - return attr -} - -func (b *RecordBatch) addRecord(r *Record) { - b.Records = append(b.Records, r) -} diff --git a/vendor/github.com/Shopify/sarama/records.go b/vendor/github.com/Shopify/sarama/records.go deleted file mode 100644 index f4c5e95f..00000000 --- a/vendor/github.com/Shopify/sarama/records.go +++ /dev/null @@ -1,203 +0,0 @@ -package sarama - -import "fmt" - -const ( - unknownRecords = iota - legacyRecords - defaultRecords - - magicOffset = 16 -) - -// Records implements a union type containing either a RecordBatch or a legacy MessageSet. -type Records struct { - recordsType int - MsgSet *MessageSet - RecordBatch *RecordBatch -} - -func newLegacyRecords(msgSet *MessageSet) Records { - return Records{recordsType: legacyRecords, MsgSet: msgSet} -} - -func newDefaultRecords(batch *RecordBatch) Records { - return Records{recordsType: defaultRecords, RecordBatch: batch} -} - -// setTypeFromFields sets type of Records depending on which of MsgSet or RecordBatch is not nil. -// The first return value indicates whether both fields are nil (and the type is not set). -// If both fields are not nil, it returns an error. -func (r *Records) setTypeFromFields() (bool, error) { - if r.MsgSet == nil && r.RecordBatch == nil { - return true, nil - } - if r.MsgSet != nil && r.RecordBatch != nil { - return false, fmt.Errorf("both MsgSet and RecordBatch are set, but record type is unknown") - } - r.recordsType = defaultRecords - if r.MsgSet != nil { - r.recordsType = legacyRecords - } - return false, nil -} - -func (r *Records) encode(pe packetEncoder) error { - if r.recordsType == unknownRecords { - if empty, err := r.setTypeFromFields(); err != nil || empty { - return err - } - } - - switch r.recordsType { - case legacyRecords: - if r.MsgSet == nil { - return nil - } - return r.MsgSet.encode(pe) - case defaultRecords: - if r.RecordBatch == nil { - return nil - } - return r.RecordBatch.encode(pe) - } - - return fmt.Errorf("unknown records type: %v", r.recordsType) -} - -func (r *Records) setTypeFromMagic(pd packetDecoder) error { - magic, err := magicValue(pd) - if err != nil { - return err - } - - r.recordsType = defaultRecords - if magic < 2 { - r.recordsType = legacyRecords - } - - return nil -} - -func (r *Records) decode(pd packetDecoder) error { - if r.recordsType == unknownRecords { - if err := r.setTypeFromMagic(pd); err != nil { - return err - } - } - - switch r.recordsType { - case legacyRecords: - r.MsgSet = &MessageSet{} - return r.MsgSet.decode(pd) - case defaultRecords: - r.RecordBatch = &RecordBatch{} - return r.RecordBatch.decode(pd) - } - return fmt.Errorf("unknown records type: %v", r.recordsType) -} - -func (r *Records) numRecords() (int, error) { - if r.recordsType == unknownRecords { - if empty, err := r.setTypeFromFields(); err != nil || empty { - return 0, err - } - } - - switch r.recordsType { - case legacyRecords: - if r.MsgSet == nil { - return 0, nil - } - return len(r.MsgSet.Messages), nil - case defaultRecords: - if r.RecordBatch == nil { - return 0, nil - } - return len(r.RecordBatch.Records), nil - } - return 0, fmt.Errorf("unknown records type: %v", r.recordsType) -} - -func (r *Records) isPartial() (bool, error) { - if r.recordsType == unknownRecords { - if empty, err := r.setTypeFromFields(); err != nil || empty { - return false, err - } - } - - switch r.recordsType { - case unknownRecords: - return false, nil - case legacyRecords: - if r.MsgSet == nil { - return false, nil - } - return r.MsgSet.PartialTrailingMessage, nil - case defaultRecords: - if r.RecordBatch == nil { - return false, nil - } - return r.RecordBatch.PartialTrailingRecord, nil - } - return false, fmt.Errorf("unknown records type: %v", r.recordsType) -} - -func (r *Records) isControl() (bool, error) { - if r.recordsType == unknownRecords { - if empty, err := r.setTypeFromFields(); err != nil || empty { - return false, err - } - } - - switch r.recordsType { - case legacyRecords: - return false, nil - case defaultRecords: - if r.RecordBatch == nil { - return false, nil - } - return r.RecordBatch.Control, nil - } - return false, fmt.Errorf("unknown records type: %v", r.recordsType) -} - -func (r *Records) isOverflow() (bool, error) { - if r.recordsType == unknownRecords { - if empty, err := r.setTypeFromFields(); err != nil || empty { - return false, err - } - } - - switch r.recordsType { - case unknownRecords: - return false, nil - case legacyRecords: - if r.MsgSet == nil { - return false, nil - } - return r.MsgSet.OverflowMessage, nil - case defaultRecords: - return false, nil - } - return false, fmt.Errorf("unknown records type: %v", r.recordsType) -} - -func magicValue(pd packetDecoder) (int8, error) { - return pd.peekInt8(magicOffset) -} - -func (r *Records) getControlRecord() (ControlRecord, error) { - if r.RecordBatch == nil || len(r.RecordBatch.Records) <= 0 { - return ControlRecord{}, fmt.Errorf("cannot get control record, record batch is empty") - } - - firstRecord := r.RecordBatch.Records[0] - controlRecord := ControlRecord{} - err := controlRecord.decode(&realDecoder{raw: firstRecord.Key}, &realDecoder{raw: firstRecord.Value}) - if err != nil { - return ControlRecord{}, err - } - - return controlRecord, nil -} diff --git a/vendor/github.com/Shopify/sarama/request.go b/vendor/github.com/Shopify/sarama/request.go deleted file mode 100644 index dcfd3946..00000000 --- a/vendor/github.com/Shopify/sarama/request.go +++ /dev/null @@ -1,191 +0,0 @@ -package sarama - -import ( - "encoding/binary" - "fmt" - "io" -) - -type protocolBody interface { - encoder - versionedDecoder - key() int16 - version() int16 - headerVersion() int16 - requiredVersion() KafkaVersion -} - -type request struct { - correlationID int32 - clientID string - body protocolBody -} - -func (r *request) encode(pe packetEncoder) error { - pe.push(&lengthField{}) - pe.putInt16(r.body.key()) - pe.putInt16(r.body.version()) - pe.putInt32(r.correlationID) - - if r.body.headerVersion() >= 1 { - err := pe.putString(r.clientID) - if err != nil { - return err - } - } - - if r.body.headerVersion() >= 2 { - // we don't use tag headers at the moment so we just put an array length of 0 - pe.putUVarint(0) - } - - err := r.body.encode(pe) - if err != nil { - return err - } - - return pe.pop() -} - -func (r *request) decode(pd packetDecoder) (err error) { - key, err := pd.getInt16() - if err != nil { - return err - } - - version, err := pd.getInt16() - if err != nil { - return err - } - - r.correlationID, err = pd.getInt32() - if err != nil { - return err - } - - r.clientID, err = pd.getString() - if err != nil { - return err - } - - r.body = allocateBody(key, version) - if r.body == nil { - return PacketDecodingError{fmt.Sprintf("unknown request key (%d)", key)} - } - - if r.body.headerVersion() >= 2 { - // tagged field - _, err = pd.getUVarint() - if err != nil { - return err - } - } - - return r.body.decode(pd, version) -} - -func decodeRequest(r io.Reader) (*request, int, error) { - var ( - bytesRead int - lengthBytes = make([]byte, 4) - ) - - if _, err := io.ReadFull(r, lengthBytes); err != nil { - return nil, bytesRead, err - } - - bytesRead += len(lengthBytes) - length := int32(binary.BigEndian.Uint32(lengthBytes)) - - if length <= 4 || length > MaxRequestSize { - return nil, bytesRead, PacketDecodingError{fmt.Sprintf("message of length %d too large or too small", length)} - } - - encodedReq := make([]byte, length) - if _, err := io.ReadFull(r, encodedReq); err != nil { - return nil, bytesRead, err - } - - bytesRead += len(encodedReq) - - req := &request{} - if err := decode(encodedReq, req); err != nil { - return nil, bytesRead, err - } - - return req, bytesRead, nil -} - -func allocateBody(key, version int16) protocolBody { - switch key { - case 0: - return &ProduceRequest{} - case 1: - return &FetchRequest{Version: version} - case 2: - return &OffsetRequest{Version: version} - case 3: - return &MetadataRequest{} - case 8: - return &OffsetCommitRequest{Version: version} - case 9: - return &OffsetFetchRequest{} - case 10: - return &FindCoordinatorRequest{} - case 11: - return &JoinGroupRequest{} - case 12: - return &HeartbeatRequest{} - case 13: - return &LeaveGroupRequest{} - case 14: - return &SyncGroupRequest{} - case 15: - return &DescribeGroupsRequest{} - case 16: - return &ListGroupsRequest{} - case 17: - return &SaslHandshakeRequest{} - case 18: - return &ApiVersionsRequest{} - case 19: - return &CreateTopicsRequest{} - case 20: - return &DeleteTopicsRequest{} - case 21: - return &DeleteRecordsRequest{} - case 22: - return &InitProducerIDRequest{} - case 24: - return &AddPartitionsToTxnRequest{} - case 25: - return &AddOffsetsToTxnRequest{} - case 26: - return &EndTxnRequest{} - case 28: - return &TxnOffsetCommitRequest{} - case 29: - return &DescribeAclsRequest{} - case 30: - return &CreateAclsRequest{} - case 31: - return &DeleteAclsRequest{} - case 32: - return &DescribeConfigsRequest{} - case 33: - return &AlterConfigsRequest{} - case 35: - return &DescribeLogDirsRequest{} - case 36: - return &SaslAuthenticateRequest{} - case 37: - return &CreatePartitionsRequest{} - case 42: - return &DeleteGroupsRequest{} - case 45: - return &AlterPartitionReassignmentsRequest{} - case 46: - return &ListPartitionReassignmentsRequest{} - } - return nil -} diff --git a/vendor/github.com/Shopify/sarama/response_header.go b/vendor/github.com/Shopify/sarama/response_header.go deleted file mode 100644 index 5dffb75b..00000000 --- a/vendor/github.com/Shopify/sarama/response_header.go +++ /dev/null @@ -1,31 +0,0 @@ -package sarama - -import "fmt" - -const responseLengthSize = 4 -const correlationIDSize = 4 - -type responseHeader struct { - length int32 - correlationID int32 -} - -func (r *responseHeader) decode(pd packetDecoder, version int16) (err error) { - r.length, err = pd.getInt32() - if err != nil { - return err - } - if r.length <= 4 || r.length > MaxResponseSize { - return PacketDecodingError{fmt.Sprintf("message of length %d too large or too small", r.length)} - } - - r.correlationID, err = pd.getInt32() - - if version >= 1 { - if _, err := pd.getEmptyTaggedFieldArray(); err != nil { - return err - } - } - - return err -} diff --git a/vendor/github.com/Shopify/sarama/sarama.go b/vendor/github.com/Shopify/sarama/sarama.go deleted file mode 100644 index 48f362d2..00000000 --- a/vendor/github.com/Shopify/sarama/sarama.go +++ /dev/null @@ -1,110 +0,0 @@ -/* -Package sarama is a pure Go client library for dealing with Apache Kafka (versions 0.8 and later). It includes a high-level -API for easily producing and consuming messages, and a low-level API for controlling bytes on the wire when the high-level -API is insufficient. Usage examples for the high-level APIs are provided inline with their full documentation. - -To produce messages, use either the AsyncProducer or the SyncProducer. The AsyncProducer accepts messages on a channel -and produces them asynchronously in the background as efficiently as possible; it is preferred in most cases. -The SyncProducer provides a method which will block until Kafka acknowledges the message as produced. This can be -useful but comes with two caveats: it will generally be less efficient, and the actual durability guarantees -depend on the configured value of `Producer.RequiredAcks`. There are configurations where a message acknowledged by the -SyncProducer can still sometimes be lost. - -To consume messages, use Consumer or Consumer-Group API. - -For lower-level needs, the Broker and Request/Response objects permit precise control over each connection -and message sent on the wire; the Client provides higher-level metadata management that is shared between -the producers and the consumer. The Request/Response objects and properties are mostly undocumented, as they line up -exactly with the protocol fields documented by Kafka at -https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol - -Metrics are exposed through https://github.com/rcrowley/go-metrics library in a local registry. - -Broker related metrics: - - +----------------------------------------------+------------+---------------------------------------------------------------+ - | Name | Type | Description | - +----------------------------------------------+------------+---------------------------------------------------------------+ - | incoming-byte-rate | meter | Bytes/second read off all brokers | - | incoming-byte-rate-for-broker- | meter | Bytes/second read off a given broker | - | outgoing-byte-rate | meter | Bytes/second written off all brokers | - | outgoing-byte-rate-for-broker- | meter | Bytes/second written off a given broker | - | request-rate | meter | Requests/second sent to all brokers | - | request-rate-for-broker- | meter | Requests/second sent to a given broker | - | request-size | histogram | Distribution of the request size in bytes for all brokers | - | request-size-for-broker- | histogram | Distribution of the request size in bytes for a given broker | - | request-latency-in-ms | histogram | Distribution of the request latency in ms for all brokers | - | request-latency-in-ms-for-broker- | histogram | Distribution of the request latency in ms for a given broker | - | response-rate | meter | Responses/second received from all brokers | - | response-rate-for-broker- | meter | Responses/second received from a given broker | - | response-size | histogram | Distribution of the response size in bytes for all brokers | - | response-size-for-broker- | histogram | Distribution of the response size in bytes for a given broker | - | requests-in-flight | counter | The current number of in-flight requests awaiting a response | - | | | for all brokers | - | requests-in-flight-for-broker- | counter | The current number of in-flight requests awaiting a response | - | | | for a given broker | - +----------------------------------------------+------------+---------------------------------------------------------------+ - -Note that we do not gather specific metrics for seed brokers but they are part of the "all brokers" metrics. - -Producer related metrics: - - +-------------------------------------------+------------+--------------------------------------------------------------------------------------+ - | Name | Type | Description | - +-------------------------------------------+------------+--------------------------------------------------------------------------------------+ - | batch-size | histogram | Distribution of the number of bytes sent per partition per request for all topics | - | batch-size-for-topic- | histogram | Distribution of the number of bytes sent per partition per request for a given topic | - | record-send-rate | meter | Records/second sent to all topics | - | record-send-rate-for-topic- | meter | Records/second sent to a given topic | - | records-per-request | histogram | Distribution of the number of records sent per request for all topics | - | records-per-request-for-topic- | histogram | Distribution of the number of records sent per request for a given topic | - | compression-ratio | histogram | Distribution of the compression ratio times 100 of record batches for all topics | - | compression-ratio-for-topic- | histogram | Distribution of the compression ratio times 100 of record batches for a given topic | - +-------------------------------------------+------------+--------------------------------------------------------------------------------------+ - -Consumer related metrics: - - +-------------------------------------------+------------+--------------------------------------------------------------------------------------+ - | Name | Type | Description | - +-------------------------------------------+------------+--------------------------------------------------------------------------------------+ - | consumer-batch-size | histogram | Distribution of the number of messages in a batch | - +-------------------------------------------+------------+--------------------------------------------------------------------------------------+ - -*/ -package sarama - -import ( - "io/ioutil" - "log" -) - -var ( - // Logger is the instance of a StdLogger interface that Sarama writes connection - // management events to. By default it is set to discard all log messages via ioutil.Discard, - // but you can set it to redirect wherever you want. - Logger StdLogger = log.New(ioutil.Discard, "[Sarama] ", log.LstdFlags) - - // PanicHandler is called for recovering from panics spawned internally to the library (and thus - // not recoverable by the caller's goroutine). Defaults to nil, which means panics are not recovered. - PanicHandler func(interface{}) - - // MaxRequestSize is the maximum size (in bytes) of any request that Sarama will attempt to send. Trying - // to send a request larger than this will result in an PacketEncodingError. The default of 100 MiB is aligned - // with Kafka's default `socket.request.max.bytes`, which is the largest request the broker will attempt - // to process. - MaxRequestSize int32 = 100 * 1024 * 1024 - - // MaxResponseSize is the maximum size (in bytes) of any response that Sarama will attempt to parse. If - // a broker returns a response message larger than this value, Sarama will return a PacketDecodingError to - // protect the client from running out of memory. Please note that brokers do not have any natural limit on - // the size of responses they send. In particular, they can send arbitrarily large fetch responses to consumers - // (see https://issues.apache.org/jira/browse/KAFKA-2063). - MaxResponseSize int32 = 100 * 1024 * 1024 -) - -// StdLogger is used to log error messages. -type StdLogger interface { - Print(v ...interface{}) - Printf(format string, v ...interface{}) - Println(v ...interface{}) -} diff --git a/vendor/github.com/Shopify/sarama/sasl_authenticate_request.go b/vendor/github.com/Shopify/sarama/sasl_authenticate_request.go deleted file mode 100644 index 90504df6..00000000 --- a/vendor/github.com/Shopify/sarama/sasl_authenticate_request.go +++ /dev/null @@ -1,33 +0,0 @@ -package sarama - -type SaslAuthenticateRequest struct { - SaslAuthBytes []byte -} - -// APIKeySASLAuth is the API key for the SaslAuthenticate Kafka API -const APIKeySASLAuth = 36 - -func (r *SaslAuthenticateRequest) encode(pe packetEncoder) error { - return pe.putBytes(r.SaslAuthBytes) -} - -func (r *SaslAuthenticateRequest) decode(pd packetDecoder, version int16) (err error) { - r.SaslAuthBytes, err = pd.getBytes() - return err -} - -func (r *SaslAuthenticateRequest) key() int16 { - return APIKeySASLAuth -} - -func (r *SaslAuthenticateRequest) version() int16 { - return 0 -} - -func (r *SaslAuthenticateRequest) headerVersion() int16 { - return 1 -} - -func (r *SaslAuthenticateRequest) requiredVersion() KafkaVersion { - return V1_0_0_0 -} diff --git a/vendor/github.com/Shopify/sarama/sasl_authenticate_response.go b/vendor/github.com/Shopify/sarama/sasl_authenticate_response.go deleted file mode 100644 index 3ef57b5a..00000000 --- a/vendor/github.com/Shopify/sarama/sasl_authenticate_response.go +++ /dev/null @@ -1,48 +0,0 @@ -package sarama - -type SaslAuthenticateResponse struct { - Err KError - ErrorMessage *string - SaslAuthBytes []byte -} - -func (r *SaslAuthenticateResponse) encode(pe packetEncoder) error { - pe.putInt16(int16(r.Err)) - if err := pe.putNullableString(r.ErrorMessage); err != nil { - return err - } - return pe.putBytes(r.SaslAuthBytes) -} - -func (r *SaslAuthenticateResponse) decode(pd packetDecoder, version int16) error { - kerr, err := pd.getInt16() - if err != nil { - return err - } - - r.Err = KError(kerr) - - if r.ErrorMessage, err = pd.getNullableString(); err != nil { - return err - } - - r.SaslAuthBytes, err = pd.getBytes() - - return err -} - -func (r *SaslAuthenticateResponse) key() int16 { - return APIKeySASLAuth -} - -func (r *SaslAuthenticateResponse) version() int16 { - return 0 -} - -func (r *SaslAuthenticateResponse) headerVersion() int16 { - return 0 -} - -func (r *SaslAuthenticateResponse) requiredVersion() KafkaVersion { - return V1_0_0_0 -} diff --git a/vendor/github.com/Shopify/sarama/sasl_handshake_request.go b/vendor/github.com/Shopify/sarama/sasl_handshake_request.go deleted file mode 100644 index 74dc3072..00000000 --- a/vendor/github.com/Shopify/sarama/sasl_handshake_request.go +++ /dev/null @@ -1,38 +0,0 @@ -package sarama - -type SaslHandshakeRequest struct { - Mechanism string - Version int16 -} - -func (r *SaslHandshakeRequest) encode(pe packetEncoder) error { - if err := pe.putString(r.Mechanism); err != nil { - return err - } - - return nil -} - -func (r *SaslHandshakeRequest) decode(pd packetDecoder, version int16) (err error) { - if r.Mechanism, err = pd.getString(); err != nil { - return err - } - - return nil -} - -func (r *SaslHandshakeRequest) key() int16 { - return 17 -} - -func (r *SaslHandshakeRequest) version() int16 { - return r.Version -} - -func (r *SaslHandshakeRequest) headerVersion() int16 { - return 1 -} - -func (r *SaslHandshakeRequest) requiredVersion() KafkaVersion { - return V0_10_0_0 -} diff --git a/vendor/github.com/Shopify/sarama/sasl_handshake_response.go b/vendor/github.com/Shopify/sarama/sasl_handshake_response.go deleted file mode 100644 index 69dfc317..00000000 --- a/vendor/github.com/Shopify/sarama/sasl_handshake_response.go +++ /dev/null @@ -1,42 +0,0 @@ -package sarama - -type SaslHandshakeResponse struct { - Err KError - EnabledMechanisms []string -} - -func (r *SaslHandshakeResponse) encode(pe packetEncoder) error { - pe.putInt16(int16(r.Err)) - return pe.putStringArray(r.EnabledMechanisms) -} - -func (r *SaslHandshakeResponse) decode(pd packetDecoder, version int16) error { - kerr, err := pd.getInt16() - if err != nil { - return err - } - - r.Err = KError(kerr) - - if r.EnabledMechanisms, err = pd.getStringArray(); err != nil { - return err - } - - return nil -} - -func (r *SaslHandshakeResponse) key() int16 { - return 17 -} - -func (r *SaslHandshakeResponse) version() int16 { - return 0 -} - -func (r *SaslHandshakeResponse) headerVersion() int16 { - return 0 -} - -func (r *SaslHandshakeResponse) requiredVersion() KafkaVersion { - return V0_10_0_0 -} diff --git a/vendor/github.com/Shopify/sarama/sticky_assignor_user_data.go b/vendor/github.com/Shopify/sarama/sticky_assignor_user_data.go deleted file mode 100644 index bb0c82c3..00000000 --- a/vendor/github.com/Shopify/sarama/sticky_assignor_user_data.go +++ /dev/null @@ -1,124 +0,0 @@ -package sarama - -type topicPartitionAssignment struct { - Topic string - Partition int32 -} - -type StickyAssignorUserData interface { - partitions() []topicPartitionAssignment - hasGeneration() bool - generation() int -} - -//StickyAssignorUserDataV0 holds topic partition information for an assignment -type StickyAssignorUserDataV0 struct { - Topics map[string][]int32 - - topicPartitions []topicPartitionAssignment -} - -func (m *StickyAssignorUserDataV0) encode(pe packetEncoder) error { - if err := pe.putArrayLength(len(m.Topics)); err != nil { - return err - } - - for topic, partitions := range m.Topics { - if err := pe.putString(topic); err != nil { - return err - } - if err := pe.putInt32Array(partitions); err != nil { - return err - } - } - return nil -} - -func (m *StickyAssignorUserDataV0) decode(pd packetDecoder) (err error) { - var topicLen int - if topicLen, err = pd.getArrayLength(); err != nil { - return - } - - m.Topics = make(map[string][]int32, topicLen) - for i := 0; i < topicLen; i++ { - var topic string - if topic, err = pd.getString(); err != nil { - return - } - if m.Topics[topic], err = pd.getInt32Array(); err != nil { - return - } - } - m.topicPartitions = populateTopicPartitions(m.Topics) - return nil -} - -func (m *StickyAssignorUserDataV0) partitions() []topicPartitionAssignment { return m.topicPartitions } -func (m *StickyAssignorUserDataV0) hasGeneration() bool { return false } -func (m *StickyAssignorUserDataV0) generation() int { return defaultGeneration } - -//StickyAssignorUserDataV1 holds topic partition information for an assignment -type StickyAssignorUserDataV1 struct { - Topics map[string][]int32 - Generation int32 - - topicPartitions []topicPartitionAssignment -} - -func (m *StickyAssignorUserDataV1) encode(pe packetEncoder) error { - if err := pe.putArrayLength(len(m.Topics)); err != nil { - return err - } - - for topic, partitions := range m.Topics { - if err := pe.putString(topic); err != nil { - return err - } - if err := pe.putInt32Array(partitions); err != nil { - return err - } - } - - pe.putInt32(m.Generation) - return nil -} - -func (m *StickyAssignorUserDataV1) decode(pd packetDecoder) (err error) { - var topicLen int - if topicLen, err = pd.getArrayLength(); err != nil { - return - } - - m.Topics = make(map[string][]int32, topicLen) - for i := 0; i < topicLen; i++ { - var topic string - if topic, err = pd.getString(); err != nil { - return - } - if m.Topics[topic], err = pd.getInt32Array(); err != nil { - return - } - } - - m.Generation, err = pd.getInt32() - if err != nil { - return err - } - m.topicPartitions = populateTopicPartitions(m.Topics) - return nil -} - -func (m *StickyAssignorUserDataV1) partitions() []topicPartitionAssignment { return m.topicPartitions } -func (m *StickyAssignorUserDataV1) hasGeneration() bool { return true } -func (m *StickyAssignorUserDataV1) generation() int { return int(m.Generation) } - -func populateTopicPartitions(topics map[string][]int32) []topicPartitionAssignment { - topicPartitions := make([]topicPartitionAssignment, 0) - for topic, partitions := range topics { - for _, partition := range partitions { - topicPartitions = append(topicPartitions, topicPartitionAssignment{Topic: topic, Partition: partition}) - } - } - return topicPartitions -} diff --git a/vendor/github.com/Shopify/sarama/sync_group_request.go b/vendor/github.com/Shopify/sarama/sync_group_request.go deleted file mode 100644 index ac6ecb13..00000000 --- a/vendor/github.com/Shopify/sarama/sync_group_request.go +++ /dev/null @@ -1,104 +0,0 @@ -package sarama - -type SyncGroupRequest struct { - GroupId string - GenerationId int32 - MemberId string - GroupAssignments map[string][]byte -} - -func (r *SyncGroupRequest) encode(pe packetEncoder) error { - if err := pe.putString(r.GroupId); err != nil { - return err - } - - pe.putInt32(r.GenerationId) - - if err := pe.putString(r.MemberId); err != nil { - return err - } - - if err := pe.putArrayLength(len(r.GroupAssignments)); err != nil { - return err - } - for memberId, memberAssignment := range r.GroupAssignments { - if err := pe.putString(memberId); err != nil { - return err - } - if err := pe.putBytes(memberAssignment); err != nil { - return err - } - } - - return nil -} - -func (r *SyncGroupRequest) decode(pd packetDecoder, version int16) (err error) { - if r.GroupId, err = pd.getString(); err != nil { - return - } - if r.GenerationId, err = pd.getInt32(); err != nil { - return - } - if r.MemberId, err = pd.getString(); err != nil { - return - } - - n, err := pd.getArrayLength() - if err != nil { - return err - } - if n == 0 { - return nil - } - - r.GroupAssignments = make(map[string][]byte) - for i := 0; i < n; i++ { - memberId, err := pd.getString() - if err != nil { - return err - } - memberAssignment, err := pd.getBytes() - if err != nil { - return err - } - - r.GroupAssignments[memberId] = memberAssignment - } - - return nil -} - -func (r *SyncGroupRequest) key() int16 { - return 14 -} - -func (r *SyncGroupRequest) version() int16 { - return 0 -} - -func (r *SyncGroupRequest) headerVersion() int16 { - return 1 -} - -func (r *SyncGroupRequest) requiredVersion() KafkaVersion { - return V0_9_0_0 -} - -func (r *SyncGroupRequest) AddGroupAssignment(memberId string, memberAssignment []byte) { - if r.GroupAssignments == nil { - r.GroupAssignments = make(map[string][]byte) - } - - r.GroupAssignments[memberId] = memberAssignment -} - -func (r *SyncGroupRequest) AddGroupAssignmentMember(memberId string, memberAssignment *ConsumerGroupMemberAssignment) error { - bin, err := encode(memberAssignment, nil) - if err != nil { - return err - } - - r.AddGroupAssignment(memberId, bin) - return nil -} diff --git a/vendor/github.com/Shopify/sarama/sync_group_response.go b/vendor/github.com/Shopify/sarama/sync_group_response.go deleted file mode 100644 index af019c42..00000000 --- a/vendor/github.com/Shopify/sarama/sync_group_response.go +++ /dev/null @@ -1,45 +0,0 @@ -package sarama - -type SyncGroupResponse struct { - Err KError - MemberAssignment []byte -} - -func (r *SyncGroupResponse) GetMemberAssignment() (*ConsumerGroupMemberAssignment, error) { - assignment := new(ConsumerGroupMemberAssignment) - err := decode(r.MemberAssignment, assignment) - return assignment, err -} - -func (r *SyncGroupResponse) encode(pe packetEncoder) error { - pe.putInt16(int16(r.Err)) - return pe.putBytes(r.MemberAssignment) -} - -func (r *SyncGroupResponse) decode(pd packetDecoder, version int16) (err error) { - kerr, err := pd.getInt16() - if err != nil { - return err - } - - r.Err = KError(kerr) - - r.MemberAssignment, err = pd.getBytes() - return -} - -func (r *SyncGroupResponse) key() int16 { - return 14 -} - -func (r *SyncGroupResponse) version() int16 { - return 0 -} - -func (r *SyncGroupResponse) headerVersion() int16 { - return 0 -} - -func (r *SyncGroupResponse) requiredVersion() KafkaVersion { - return V0_9_0_0 -} diff --git a/vendor/github.com/Shopify/sarama/sync_producer.go b/vendor/github.com/Shopify/sarama/sync_producer.go deleted file mode 100644 index 021c5a01..00000000 --- a/vendor/github.com/Shopify/sarama/sync_producer.go +++ /dev/null @@ -1,149 +0,0 @@ -package sarama - -import "sync" - -// SyncProducer publishes Kafka messages, blocking until they have been acknowledged. It routes messages to the correct -// broker, refreshing metadata as appropriate, and parses responses for errors. You must call Close() on a producer -// to avoid leaks, it may not be garbage-collected automatically when it passes out of scope. -// -// The SyncProducer comes with two caveats: it will generally be less efficient than the AsyncProducer, and the actual -// durability guarantee provided when a message is acknowledged depend on the configured value of `Producer.RequiredAcks`. -// There are configurations where a message acknowledged by the SyncProducer can still sometimes be lost. -// -// For implementation reasons, the SyncProducer requires `Producer.Return.Errors` and `Producer.Return.Successes` to -// be set to true in its configuration. -type SyncProducer interface { - - // SendMessage produces a given message, and returns only when it either has - // succeeded or failed to produce. It will return the partition and the offset - // of the produced message, or an error if the message failed to produce. - SendMessage(msg *ProducerMessage) (partition int32, offset int64, err error) - - // SendMessages produces a given set of messages, and returns only when all - // messages in the set have either succeeded or failed. Note that messages - // can succeed and fail individually; if some succeed and some fail, - // SendMessages will return an error. - SendMessages(msgs []*ProducerMessage) error - - // Close shuts down the producer and waits for any buffered messages to be - // flushed. You must call this function before a producer object passes out of - // scope, as it may otherwise leak memory. You must call this before calling - // Close on the underlying client. - Close() error -} - -type syncProducer struct { - producer *asyncProducer - wg sync.WaitGroup -} - -// NewSyncProducer creates a new SyncProducer using the given broker addresses and configuration. -func NewSyncProducer(addrs []string, config *Config) (SyncProducer, error) { - if config == nil { - config = NewConfig() - config.Producer.Return.Successes = true - } - - if err := verifyProducerConfig(config); err != nil { - return nil, err - } - - p, err := NewAsyncProducer(addrs, config) - if err != nil { - return nil, err - } - return newSyncProducerFromAsyncProducer(p.(*asyncProducer)), nil -} - -// NewSyncProducerFromClient creates a new SyncProducer using the given client. It is still -// necessary to call Close() on the underlying client when shutting down this producer. -func NewSyncProducerFromClient(client Client) (SyncProducer, error) { - if err := verifyProducerConfig(client.Config()); err != nil { - return nil, err - } - - p, err := NewAsyncProducerFromClient(client) - if err != nil { - return nil, err - } - return newSyncProducerFromAsyncProducer(p.(*asyncProducer)), nil -} - -func newSyncProducerFromAsyncProducer(p *asyncProducer) *syncProducer { - sp := &syncProducer{producer: p} - - sp.wg.Add(2) - go withRecover(sp.handleSuccesses) - go withRecover(sp.handleErrors) - - return sp -} - -func verifyProducerConfig(config *Config) error { - if !config.Producer.Return.Errors { - return ConfigurationError("Producer.Return.Errors must be true to be used in a SyncProducer") - } - if !config.Producer.Return.Successes { - return ConfigurationError("Producer.Return.Successes must be true to be used in a SyncProducer") - } - return nil -} - -func (sp *syncProducer) SendMessage(msg *ProducerMessage) (partition int32, offset int64, err error) { - expectation := make(chan *ProducerError, 1) - msg.expectation = expectation - sp.producer.Input() <- msg - - if err := <-expectation; err != nil { - return -1, -1, err.Err - } - - return msg.Partition, msg.Offset, nil -} - -func (sp *syncProducer) SendMessages(msgs []*ProducerMessage) error { - expectations := make(chan chan *ProducerError, len(msgs)) - go func() { - for _, msg := range msgs { - expectation := make(chan *ProducerError, 1) - msg.expectation = expectation - sp.producer.Input() <- msg - expectations <- expectation - } - close(expectations) - }() - - var errors ProducerErrors - for expectation := range expectations { - if err := <-expectation; err != nil { - errors = append(errors, err) - } - } - - if len(errors) > 0 { - return errors - } - return nil -} - -func (sp *syncProducer) handleSuccesses() { - defer sp.wg.Done() - for msg := range sp.producer.Successes() { - expectation := msg.expectation - expectation <- nil - } -} - -func (sp *syncProducer) handleErrors() { - defer sp.wg.Done() - for err := range sp.producer.Errors() { - expectation := err.Msg.expectation - expectation <- err - } -} - -func (sp *syncProducer) Close() error { - sp.producer.AsyncClose() - sp.wg.Wait() - return nil -} diff --git a/vendor/github.com/Shopify/sarama/timestamp.go b/vendor/github.com/Shopify/sarama/timestamp.go deleted file mode 100644 index 372278d0..00000000 --- a/vendor/github.com/Shopify/sarama/timestamp.go +++ /dev/null @@ -1,40 +0,0 @@ -package sarama - -import ( - "fmt" - "time" -) - -type Timestamp struct { - *time.Time -} - -func (t Timestamp) encode(pe packetEncoder) error { - timestamp := int64(-1) - - if !t.Before(time.Unix(0, 0)) { - timestamp = t.UnixNano() / int64(time.Millisecond) - } else if !t.IsZero() { - return PacketEncodingError{fmt.Sprintf("invalid timestamp (%v)", t)} - } - - pe.putInt64(timestamp) - return nil -} - -func (t Timestamp) decode(pd packetDecoder) error { - millis, err := pd.getInt64() - if err != nil { - return err - } - - // negative timestamps are invalid, in these cases we should return - // a zero time - timestamp := time.Time{} - if millis >= 0 { - timestamp = time.Unix(millis/1000, (millis%1000)*int64(time.Millisecond)) - } - - *t.Time = timestamp - return nil -} diff --git a/vendor/github.com/Shopify/sarama/txn_offset_commit_request.go b/vendor/github.com/Shopify/sarama/txn_offset_commit_request.go deleted file mode 100644 index c4043a33..00000000 --- a/vendor/github.com/Shopify/sarama/txn_offset_commit_request.go +++ /dev/null @@ -1,130 +0,0 @@ -package sarama - -type TxnOffsetCommitRequest struct { - TransactionalID string - GroupID string - ProducerID int64 - ProducerEpoch int16 - Topics map[string][]*PartitionOffsetMetadata -} - -func (t *TxnOffsetCommitRequest) encode(pe packetEncoder) error { - if err := pe.putString(t.TransactionalID); err != nil { - return err - } - if err := pe.putString(t.GroupID); err != nil { - return err - } - pe.putInt64(t.ProducerID) - pe.putInt16(t.ProducerEpoch) - - if err := pe.putArrayLength(len(t.Topics)); err != nil { - return err - } - for topic, partitions := range t.Topics { - if err := pe.putString(topic); err != nil { - return err - } - if err := pe.putArrayLength(len(partitions)); err != nil { - return err - } - for _, partition := range partitions { - if err := partition.encode(pe); err != nil { - return err - } - } - } - - return nil -} - -func (t *TxnOffsetCommitRequest) decode(pd packetDecoder, version int16) (err error) { - if t.TransactionalID, err = pd.getString(); err != nil { - return err - } - if t.GroupID, err = pd.getString(); err != nil { - return err - } - if t.ProducerID, err = pd.getInt64(); err != nil { - return err - } - if t.ProducerEpoch, err = pd.getInt16(); err != nil { - return err - } - - n, err := pd.getArrayLength() - if err != nil { - return err - } - - t.Topics = make(map[string][]*PartitionOffsetMetadata) - for i := 0; i < n; i++ { - topic, err := pd.getString() - if err != nil { - return err - } - - m, err := pd.getArrayLength() - if err != nil { - return err - } - - t.Topics[topic] = make([]*PartitionOffsetMetadata, m) - - for j := 0; j < m; j++ { - partitionOffsetMetadata := new(PartitionOffsetMetadata) - if err := partitionOffsetMetadata.decode(pd, version); err != nil { - return err - } - t.Topics[topic][j] = partitionOffsetMetadata - } - } - - return nil -} - -func (a *TxnOffsetCommitRequest) key() int16 { - return 28 -} - -func (a *TxnOffsetCommitRequest) version() int16 { - return 0 -} - -func (a *TxnOffsetCommitRequest) headerVersion() int16 { - return 1 -} - -func (a *TxnOffsetCommitRequest) requiredVersion() KafkaVersion { - return V0_11_0_0 -} - -type PartitionOffsetMetadata struct { - Partition int32 - Offset int64 - Metadata *string -} - -func (p *PartitionOffsetMetadata) encode(pe packetEncoder) error { - pe.putInt32(p.Partition) - pe.putInt64(p.Offset) - if err := pe.putNullableString(p.Metadata); err != nil { - return err - } - - return nil -} - -func (p *PartitionOffsetMetadata) decode(pd packetDecoder, version int16) (err error) { - if p.Partition, err = pd.getInt32(); err != nil { - return err - } - if p.Offset, err = pd.getInt64(); err != nil { - return err - } - if p.Metadata, err = pd.getNullableString(); err != nil { - return err - } - - return nil -} diff --git a/vendor/github.com/Shopify/sarama/txn_offset_commit_response.go b/vendor/github.com/Shopify/sarama/txn_offset_commit_response.go deleted file mode 100644 index 94d8029d..00000000 --- a/vendor/github.com/Shopify/sarama/txn_offset_commit_response.go +++ /dev/null @@ -1,87 +0,0 @@ -package sarama - -import ( - "time" -) - -type TxnOffsetCommitResponse struct { - ThrottleTime time.Duration - Topics map[string][]*PartitionError -} - -func (t *TxnOffsetCommitResponse) encode(pe packetEncoder) error { - pe.putInt32(int32(t.ThrottleTime / time.Millisecond)) - if err := pe.putArrayLength(len(t.Topics)); err != nil { - return err - } - - for topic, e := range t.Topics { - if err := pe.putString(topic); err != nil { - return err - } - if err := pe.putArrayLength(len(e)); err != nil { - return err - } - for _, partitionError := range e { - if err := partitionError.encode(pe); err != nil { - return err - } - } - } - - return nil -} - -func (t *TxnOffsetCommitResponse) decode(pd packetDecoder, version int16) (err error) { - throttleTime, err := pd.getInt32() - if err != nil { - return err - } - t.ThrottleTime = time.Duration(throttleTime) * time.Millisecond - - n, err := pd.getArrayLength() - if err != nil { - return err - } - - t.Topics = make(map[string][]*PartitionError) - - for i := 0; i < n; i++ { - topic, err := pd.getString() - if err != nil { - return err - } - - m, err := pd.getArrayLength() - if err != nil { - return err - } - - t.Topics[topic] = make([]*PartitionError, m) - - for j := 0; j < m; j++ { - t.Topics[topic][j] = new(PartitionError) - if err := t.Topics[topic][j].decode(pd, version); err != nil { - return err - } - } - } - - return nil -} - -func (a *TxnOffsetCommitResponse) key() int16 { - return 28 -} - -func (a *TxnOffsetCommitResponse) version() int16 { - return 0 -} - -func (a *TxnOffsetCommitResponse) headerVersion() int16 { - return 0 -} - -func (a *TxnOffsetCommitResponse) requiredVersion() KafkaVersion { - return V0_11_0_0 -} diff --git a/vendor/github.com/Shopify/sarama/utils.go b/vendor/github.com/Shopify/sarama/utils.go deleted file mode 100644 index 93bdeefe..00000000 --- a/vendor/github.com/Shopify/sarama/utils.go +++ /dev/null @@ -1,229 +0,0 @@ -package sarama - -import ( - "bufio" - "fmt" - "net" - "regexp" -) - -type none struct{} - -// make []int32 sortable so we can sort partition numbers -type int32Slice []int32 - -func (slice int32Slice) Len() int { - return len(slice) -} - -func (slice int32Slice) Less(i, j int) bool { - return slice[i] < slice[j] -} - -func (slice int32Slice) Swap(i, j int) { - slice[i], slice[j] = slice[j], slice[i] -} - -func dupInt32Slice(input []int32) []int32 { - ret := make([]int32, 0, len(input)) - ret = append(ret, input...) - return ret -} - -func withRecover(fn func()) { - defer func() { - handler := PanicHandler - if handler != nil { - if err := recover(); err != nil { - handler(err) - } - } - }() - - fn() -} - -func safeAsyncClose(b *Broker) { - tmp := b // local var prevents clobbering in goroutine - go withRecover(func() { - if connected, _ := tmp.Connected(); connected { - if err := tmp.Close(); err != nil { - Logger.Println("Error closing broker", tmp.ID(), ":", err) - } - } - }) -} - -// Encoder is a simple interface for any type that can be encoded as an array of bytes -// in order to be sent as the key or value of a Kafka message. Length() is provided as an -// optimization, and must return the same as len() on the result of Encode(). -type Encoder interface { - Encode() ([]byte, error) - Length() int -} - -// make strings and byte slices encodable for convenience so they can be used as keys -// and/or values in kafka messages - -// StringEncoder implements the Encoder interface for Go strings so that they can be used -// as the Key or Value in a ProducerMessage. -type StringEncoder string - -func (s StringEncoder) Encode() ([]byte, error) { - return []byte(s), nil -} - -func (s StringEncoder) Length() int { - return len(s) -} - -// ByteEncoder implements the Encoder interface for Go byte slices so that they can be used -// as the Key or Value in a ProducerMessage. -type ByteEncoder []byte - -func (b ByteEncoder) Encode() ([]byte, error) { - return b, nil -} - -func (b ByteEncoder) Length() int { - return len(b) -} - -// bufConn wraps a net.Conn with a buffer for reads to reduce the number of -// reads that trigger syscalls. -type bufConn struct { - net.Conn - buf *bufio.Reader -} - -func newBufConn(conn net.Conn) *bufConn { - return &bufConn{ - Conn: conn, - buf: bufio.NewReader(conn), - } -} - -func (bc *bufConn) Read(b []byte) (n int, err error) { - return bc.buf.Read(b) -} - -// KafkaVersion instances represent versions of the upstream Kafka broker. -type KafkaVersion struct { - // it's a struct rather than just typing the array directly to make it opaque and stop people - // generating their own arbitrary versions - version [4]uint -} - -func newKafkaVersion(major, minor, veryMinor, patch uint) KafkaVersion { - return KafkaVersion{ - version: [4]uint{major, minor, veryMinor, patch}, - } -} - -// IsAtLeast return true if and only if the version it is called on is -// greater than or equal to the version passed in: -// V1.IsAtLeast(V2) // false -// V2.IsAtLeast(V1) // true -func (v KafkaVersion) IsAtLeast(other KafkaVersion) bool { - for i := range v.version { - if v.version[i] > other.version[i] { - return true - } else if v.version[i] < other.version[i] { - return false - } - } - return true -} - -// Effective constants defining the supported kafka versions. -var ( - V0_8_2_0 = newKafkaVersion(0, 8, 2, 0) - V0_8_2_1 = newKafkaVersion(0, 8, 2, 1) - V0_8_2_2 = newKafkaVersion(0, 8, 2, 2) - V0_9_0_0 = newKafkaVersion(0, 9, 0, 0) - V0_9_0_1 = newKafkaVersion(0, 9, 0, 1) - V0_10_0_0 = newKafkaVersion(0, 10, 0, 0) - V0_10_0_1 = newKafkaVersion(0, 10, 0, 1) - V0_10_1_0 = newKafkaVersion(0, 10, 1, 0) - V0_10_1_1 = newKafkaVersion(0, 10, 1, 1) - V0_10_2_0 = newKafkaVersion(0, 10, 2, 0) - V0_10_2_1 = newKafkaVersion(0, 10, 2, 1) - V0_11_0_0 = newKafkaVersion(0, 11, 0, 0) - V0_11_0_1 = newKafkaVersion(0, 11, 0, 1) - V0_11_0_2 = newKafkaVersion(0, 11, 0, 2) - V1_0_0_0 = newKafkaVersion(1, 0, 0, 0) - V1_1_0_0 = newKafkaVersion(1, 1, 0, 0) - V1_1_1_0 = newKafkaVersion(1, 1, 1, 0) - V2_0_0_0 = newKafkaVersion(2, 0, 0, 0) - V2_0_1_0 = newKafkaVersion(2, 0, 1, 0) - V2_1_0_0 = newKafkaVersion(2, 1, 0, 0) - V2_2_0_0 = newKafkaVersion(2, 2, 0, 0) - V2_3_0_0 = newKafkaVersion(2, 3, 0, 0) - V2_4_0_0 = newKafkaVersion(2, 4, 0, 0) - V2_5_0_0 = newKafkaVersion(2, 5, 0, 0) - V2_6_0_0 = newKafkaVersion(2, 6, 0, 0) - - SupportedVersions = []KafkaVersion{ - V0_8_2_0, - V0_8_2_1, - V0_8_2_2, - V0_9_0_0, - V0_9_0_1, - V0_10_0_0, - V0_10_0_1, - V0_10_1_0, - V0_10_1_1, - V0_10_2_0, - V0_10_2_1, - V0_11_0_0, - V0_11_0_1, - V0_11_0_2, - V1_0_0_0, - V1_1_0_0, - V1_1_1_0, - V2_0_0_0, - V2_0_1_0, - V2_1_0_0, - V2_2_0_0, - V2_3_0_0, - V2_4_0_0, - V2_5_0_0, - V2_6_0_0, - } - MinVersion = V0_8_2_0 - MaxVersion = V2_6_0_0 -) - -//ParseKafkaVersion parses and returns kafka version or error from a string -func ParseKafkaVersion(s string) (KafkaVersion, error) { - if len(s) < 5 { - return MinVersion, fmt.Errorf("invalid version `%s`", s) - } - var major, minor, veryMinor, patch uint - var err error - if s[0] == '0' { - err = scanKafkaVersion(s, `^0\.\d+\.\d+\.\d+$`, "0.%d.%d.%d", [3]*uint{&minor, &veryMinor, &patch}) - } else { - err = scanKafkaVersion(s, `^\d+\.\d+\.\d+$`, "%d.%d.%d", [3]*uint{&major, &minor, &veryMinor}) - } - if err != nil { - return MinVersion, err - } - return newKafkaVersion(major, minor, veryMinor, patch), nil -} - -func scanKafkaVersion(s string, pattern string, format string, v [3]*uint) error { - if !regexp.MustCompile(pattern).MatchString(s) { - return fmt.Errorf("invalid version `%s`", s) - } - _, err := fmt.Sscanf(s, format, v[0], v[1], v[2]) - return err -} - -func (v KafkaVersion) String() string { - if v.version[0] == 0 { - return fmt.Sprintf("0.%d.%d.%d", v.version[1], v.version[2], v.version[3]) - } - - return fmt.Sprintf("%d.%d.%d", v.version[0], v.version[1], v.version[2]) -} diff --git a/vendor/github.com/Shopify/sarama/zstd.go b/vendor/github.com/Shopify/sarama/zstd.go deleted file mode 100644 index 7c9951ac..00000000 --- a/vendor/github.com/Shopify/sarama/zstd.go +++ /dev/null @@ -1,28 +0,0 @@ -package sarama - -import ( - "sync" - - "github.com/klauspost/compress/zstd" -) - -var ( - zstdDec *zstd.Decoder - zstdEnc *zstd.Encoder - - zstdEncOnce, zstdDecOnce sync.Once -) - -func zstdDecompress(dst, src []byte) ([]byte, error) { - zstdDecOnce.Do(func() { - zstdDec, _ = zstd.NewReader(nil) - }) - return zstdDec.DecodeAll(src, dst) -} - -func zstdCompress(dst, src []byte) ([]byte, error) { - zstdEncOnce.Do(func() { - zstdEnc, _ = zstd.NewWriter(nil, zstd.WithZeroFrames(true)) - }) - return zstdEnc.EncodeAll(src, dst), nil -} diff --git a/vendor/github.com/cloudevents/sdk-go/protocol/kafka_sarama/v2/LICENSE b/vendor/github.com/cloudevents/sdk-go/protocol/kafka_sarama/v2/LICENSE deleted file mode 100644 index 261eeb9e..00000000 --- a/vendor/github.com/cloudevents/sdk-go/protocol/kafka_sarama/v2/LICENSE +++ /dev/null @@ -1,201 +0,0 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/vendor/github.com/cloudevents/sdk-go/protocol/kafka_sarama/v2/doc.go b/vendor/github.com/cloudevents/sdk-go/protocol/kafka_sarama/v2/doc.go deleted file mode 100644 index 12dfff9f..00000000 --- a/vendor/github.com/cloudevents/sdk-go/protocol/kafka_sarama/v2/doc.go +++ /dev/null @@ -1,4 +0,0 @@ -/* -Package kafka_sarama implements a Kafka binding using github.com/Shopify/sarama module -*/ -package kafka_sarama diff --git a/vendor/github.com/cloudevents/sdk-go/protocol/kafka_sarama/v2/go.mod b/vendor/github.com/cloudevents/sdk-go/protocol/kafka_sarama/v2/go.mod deleted file mode 100644 index 88d5736b..00000000 --- a/vendor/github.com/cloudevents/sdk-go/protocol/kafka_sarama/v2/go.mod +++ /dev/null @@ -1,15 +0,0 @@ -module github.com/cloudevents/sdk-go/protocol/kafka_sarama/v2 - -go 1.13 - -replace github.com/cloudevents/sdk-go/v2 => ../../../v2 - -require ( - github.com/Shopify/sarama v1.25.0 - github.com/cloudevents/sdk-go/v2 v2.0.0 - github.com/eapache/go-resiliency v1.2.0 // indirect - github.com/frankban/quicktest v1.10.0 // indirect - github.com/pierrec/lz4 v2.5.2+incompatible // indirect - github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 // indirect - github.com/stretchr/testify v1.5.1 -) diff --git a/vendor/github.com/cloudevents/sdk-go/protocol/kafka_sarama/v2/go.sum b/vendor/github.com/cloudevents/sdk-go/protocol/kafka_sarama/v2/go.sum deleted file mode 100644 index 1034ac06..00000000 --- a/vendor/github.com/cloudevents/sdk-go/protocol/kafka_sarama/v2/go.sum +++ /dev/null @@ -1,156 +0,0 @@ -cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/Shopify/sarama v1.25.0 h1:ch1ywjRLjfJtU+EaiJ+l0rWffQ6TRpyYmW4DX7Cb2SU= -github.com/Shopify/sarama v1.25.0/go.mod h1:y/CFFTO9eaMTNriwu/Q+W4eioLqiDMGkA1W+gmdfj8w= -github.com/Shopify/toxiproxy v2.1.4+incompatible h1:TKdv8HiTLgE5wdJuEML90aBgNWsokNbMijUGhmcoBJc= -github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= -github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= -github.com/eapache/go-resiliency v1.2.0 h1:v7g92e/KSN71Rq7vSThKaWIq68fL4YHvWyiUKorFR1Q= -github.com/eapache/go-resiliency v1.2.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= -github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 h1:YEetp8/yCZMuEPMUDHG0CW/brkkEp8mzqk2+ODEitlw= -github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= -github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= -github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= -github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= -github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= -github.com/frankban/quicktest v1.4.1/go.mod h1:36zfPVQyHxymz4cH7wlDmVwDrJuljRB60qkgn7rorfQ= -github.com/frankban/quicktest v1.10.0 h1:Gfh+GAJZOAoKZsIZeZbdn2JF10kN1XHNvjsvQK8gVkE= -github.com/frankban/quicktest v1.10.0/go.mod h1:ui7WezCLWMWxVWr1GETZY3smRy0G4KWq9vcPtJmFl7Y= -github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= -github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= -github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= -github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= -github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= -github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/hashicorp/go-uuid v1.0.1 h1:fv1ep09latC32wFoVwnqcnKJGnMSdBanPczbHAYm1BE= -github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.3 h1:YPkqC67at8FYaadspW/6uE0COsBxS2656RLEr8Bppgk= -github.com/hashicorp/golang-lru v0.5.3/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= -github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= -github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/jcmturner/gofork v0.0.0-20190328161633-dc7c13fece03 h1:FUwcHNlEqkqLjLBdCp5PRlCFijNjvcYANOZXzCfXwCM= -github.com/jcmturner/gofork v0.0.0-20190328161633-dc7c13fece03/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o= -github.com/klauspost/compress v1.9.7 h1:hYW1gP94JUmAhBtJ+LNz5My+gBobDxPR1iVuKug26aA= -github.com/klauspost/compress v1.9.7/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= -github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= -github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/lightstep/tracecontext.go v0.0.0-20181129014701-1757c391b1ac h1:+2b6iGRJe3hvV/yVXrd41yVEjxuFHxasJqDhkIjS4gk= -github.com/lightstep/tracecontext.go v0.0.0-20181129014701-1757c391b1ac/go.mod h1:Frd2bnT3w5FB5q49ENTfVlztJES+1k/7lyWX2+9gq/M= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= -github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.10.2 h1:uqH7bpe+ERSiDa34FDOF7RikN6RzXgduUF8yarlZp94= -github.com/onsi/ginkgo v1.10.2/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME= -github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/pierrec/lz4 v2.2.6+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= -github.com/pierrec/lz4 v2.5.2+incompatible h1:WCjObylUIOlKy/+7Abdn34TLIkXiA4UWUMhxq9m9ZXI= -github.com/pierrec/lz4 v2.5.2+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= -github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= -github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= -github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 h1:MkV+77GLUNo5oJ0jf870itWm3D0Sjh7+Za9gazKc5LQ= -github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= -github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= -github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= -github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= -go.opencensus.io v0.22.0 h1:C9hSCOW830chIVkdja34wa6Ky+IzWllkUinR+BtRZd4= -go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= -go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU= -go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI= -go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= -go.uber.org/zap v1.10.0 h1:ORx85nbTijNz8ljznvCMR1ZBIPKFn3jQrag10X2AsuM= -go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190404164418-38d8ce5564a5 h1:bselrhR0Or1vomJZC8ZIjWtbDmn9OYFLX5Ik9alpJpE= -golang.org/x/crypto v0.0.0-20190404164418-38d8ce5564a5/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= -golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjutRKlBEZQ6wTn8ozI/nI= -golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5 h1:LfCXLvNmTYH9kEmVgqbnsWfruoXZIrh4YBgqVHtDvw0= -golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= -golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= -golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= -gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= -gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= -gopkg.in/jcmturner/aescts.v1 v1.0.1 h1:cVVZBK2b1zY26haWB4vbBiZrfFQnfbTVrE3xZq6hrEw= -gopkg.in/jcmturner/aescts.v1 v1.0.1/go.mod h1:nsR8qBOg+OucoIW+WMhB3GspUQXq9XorLnQb9XtvcOo= -gopkg.in/jcmturner/dnsutils.v1 v1.0.1 h1:cIuC1OLRGZrld+16ZJvvZxVJeKPsvd5eUIvxfoN5hSM= -gopkg.in/jcmturner/dnsutils.v1 v1.0.1/go.mod h1:m3v+5svpVOhtFAP/wSz+yzh4Mc0Fg7eRhxkJMWSIz9Q= -gopkg.in/jcmturner/goidentity.v3 v3.0.0 h1:1duIyWiTaYvVx3YX2CYtpJbUFd7/UuPYCfgXtQ3VTbI= -gopkg.in/jcmturner/goidentity.v3 v3.0.0/go.mod h1:oG2kH0IvSYNIu80dVAyu/yoefjq1mNfM5bm88whjWx4= -gopkg.in/jcmturner/gokrb5.v7 v7.2.3 h1:hHMV/yKPwMnJhPuPx7pH2Uw/3Qyf+thJYlisUc44010= -gopkg.in/jcmturner/gokrb5.v7 v7.2.3/go.mod h1:l8VISx+WGYp+Fp7KRbsiUuXTTOnxIc3Tuvyavf11/WM= -gopkg.in/jcmturner/rpc.v1 v1.1.0 h1:QHIUxTX1ISuAv9dD2wJ9HWQVuWDX/Zc0PfeC2tjc4rU= -gopkg.in/jcmturner/rpc.v1 v1.1.0/go.mod h1:YIdkC4XfD6GXbzje11McwsDuOlZQSb9W4vfLvuNnlv8= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= -gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= -gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/vendor/github.com/cloudevents/sdk-go/protocol/kafka_sarama/v2/message.go b/vendor/github.com/cloudevents/sdk-go/protocol/kafka_sarama/v2/message.go deleted file mode 100644 index c6bf2ee6..00000000 --- a/vendor/github.com/cloudevents/sdk-go/protocol/kafka_sarama/v2/message.go +++ /dev/null @@ -1,138 +0,0 @@ -package kafka_sarama - -import ( - "bytes" - "context" - "strings" - - "github.com/cloudevents/sdk-go/v2/binding" - "github.com/cloudevents/sdk-go/v2/binding/format" - "github.com/cloudevents/sdk-go/v2/binding/spec" - - "github.com/Shopify/sarama" -) - -const ( - prefix = "ce_" - contentTypeHeader = "content-type" -) - -var specs = spec.WithPrefix(prefix) - -// Message holds a Kafka Message. -// This message *can* be read several times safely -type Message struct { - Value []byte - Headers map[string][]byte - ContentType string - format format.Format - version spec.Version -} - -// Check if http.Message implements binding.Message -var _ binding.Message = (*Message)(nil) -var _ binding.MessageMetadataReader = (*Message)(nil) - -// NewMessageFromConsumerMessage returns a binding.Message that holds the provided ConsumerMessage. -// The returned binding.Message *can* be read several times safely -// This function *doesn't* guarantee that the returned binding.Message is always a kafka_sarama.Message instance -func NewMessageFromConsumerMessage(cm *sarama.ConsumerMessage) *Message { - var contentType string - headers := make(map[string][]byte, len(cm.Headers)) - for _, r := range cm.Headers { - k := strings.ToLower(string(r.Key)) - if k == contentTypeHeader { - contentType = string(r.Value) - } - headers[k] = r.Value - } - return NewMessage(cm.Value, contentType, headers) -} - -// NewMessage returns a binding.Message that holds the provided kafka message components. -// The returned binding.Message *can* be read several times safely -// This function *doesn't* guarantee that the returned binding.Message is always a kafka_sarama.Message instance -func NewMessage(value []byte, contentType string, headers map[string][]byte) *Message { - if ft := format.Lookup(contentType); ft != nil { - return &Message{ - Value: value, - ContentType: contentType, - Headers: headers, - format: ft, - } - } else if v := specs.Version(string(headers[specs.PrefixedSpecVersionName()])); v != nil { - return &Message{ - Value: value, - ContentType: contentType, - Headers: headers, - version: v, - } - } - - return &Message{ - Value: value, - ContentType: contentType, - Headers: headers, - } -} - -func (m *Message) ReadEncoding() binding.Encoding { - if m.version != nil { - return binding.EncodingBinary - } - if m.format != nil { - return binding.EncodingStructured - } - return binding.EncodingUnknown -} - -func (m *Message) ReadStructured(ctx context.Context, encoder binding.StructuredWriter) error { - if m.format != nil { - return encoder.SetStructuredEvent(ctx, m.format, bytes.NewReader(m.Value)) - } - return binding.ErrNotStructured -} - -func (m *Message) ReadBinary(ctx context.Context, encoder binding.BinaryWriter) (err error) { - if m.version == nil { - return binding.ErrNotBinary - } - - for k, v := range m.Headers { - if strings.HasPrefix(k, prefix) { - attr := m.version.Attribute(k) - if attr != nil { - err = encoder.SetAttribute(attr, string(v)) - } else { - err = encoder.SetExtension(strings.TrimPrefix(k, prefix), string(v)) - } - } else if k == contentTypeHeader { - err = encoder.SetAttribute(m.version.AttributeFromKind(spec.DataContentType), string(v)) - } - if err != nil { - return - } - } - - if m.Value != nil { - err = encoder.SetData(bytes.NewReader(m.Value)) - } - - return -} - -func (m *Message) GetAttribute(k spec.Kind) (spec.Attribute, interface{}) { - attr := m.version.AttributeFromKind(k) - if attr != nil { - return attr, string(m.Headers[attr.PrefixedName()]) - } - return nil, nil -} - -func (m *Message) GetExtension(name string) interface{} { - return string(m.Headers[prefix+name]) -} - -func (m *Message) Finish(error) error { - return nil -} diff --git a/vendor/github.com/cloudevents/sdk-go/protocol/kafka_sarama/v2/option.go b/vendor/github.com/cloudevents/sdk-go/protocol/kafka_sarama/v2/option.go deleted file mode 100644 index f14b3522..00000000 --- a/vendor/github.com/cloudevents/sdk-go/protocol/kafka_sarama/v2/option.go +++ /dev/null @@ -1,23 +0,0 @@ -package kafka_sarama - -import ( - "context" -) - -// SenderOptionFunc is the type of kafka_sarama.Sender options -type SenderOptionFunc func(sender *Sender) - -// ProtocolOptionFunc is the type of kafka_sarama.Protocol options -type ProtocolOptionFunc func(protocol *Protocol) - -func WithReceiverGroupId(groupId string) ProtocolOptionFunc { - return func(protocol *Protocol) { - protocol.receiverGroupId = groupId - } -} - -func WithSenderContextDecorators(decorator func(context.Context) context.Context) ProtocolOptionFunc { - return func(protocol *Protocol) { - protocol.SenderContextDecorators = append(protocol.SenderContextDecorators, decorator) - } -} diff --git a/vendor/github.com/cloudevents/sdk-go/protocol/kafka_sarama/v2/protocol.go b/vendor/github.com/cloudevents/sdk-go/protocol/kafka_sarama/v2/protocol.go deleted file mode 100644 index b8db98fd..00000000 --- a/vendor/github.com/cloudevents/sdk-go/protocol/kafka_sarama/v2/protocol.go +++ /dev/null @@ -1,133 +0,0 @@ -package kafka_sarama - -import ( - "context" - "errors" - "sync" - - "github.com/Shopify/sarama" - - "github.com/cloudevents/sdk-go/v2/binding" - cecontext "github.com/cloudevents/sdk-go/v2/context" - "github.com/cloudevents/sdk-go/v2/protocol" -) - -const ( - defaultGroupId = "cloudevents-sdk-go" -) - -type Protocol struct { - // Kafka - Client sarama.Client - ownsClient bool - - // Sender - Sender *Sender - - // Sender options - SenderContextDecorators []func(context.Context) context.Context - senderTopic string - - // Consumer - Consumer *Consumer - consumerMux sync.Mutex - - // Consumer options - receiverTopic string - receiverGroupId string -} - -// NewProtocol creates a new kafka transport. -func NewProtocol(brokers []string, saramaConfig *sarama.Config, sendToTopic string, receiveFromTopic string, opts ...ProtocolOptionFunc) (*Protocol, error) { - // Force this setting because it's required by sarama SyncProducer - saramaConfig.Producer.Return.Successes = true - client, err := sarama.NewClient(brokers, saramaConfig) - if err != nil { - return nil, err - } - - p, err := NewProtocolFromClient(client, sendToTopic, receiveFromTopic, opts...) - if err != nil { - return nil, err - } - p.ownsClient = true - return p, nil -} - -// NewProtocolFromClient creates a new kafka transport starting from a sarama.Client -func NewProtocolFromClient(client sarama.Client, sendToTopic string, receiveFromTopic string, opts ...ProtocolOptionFunc) (*Protocol, error) { - p := &Protocol{ - Client: client, - SenderContextDecorators: make([]func(context.Context) context.Context, 0), - receiverGroupId: defaultGroupId, - senderTopic: sendToTopic, - receiverTopic: receiveFromTopic, - ownsClient: false, - } - - var err error - if err = p.applyOptions(opts...); err != nil { - return nil, err - } - - if p.senderTopic == "" { - return nil, errors.New("you didn't specify the topic to send to") - } - p.Sender, err = NewSenderFromClient(p.Client, p.senderTopic) - if err != nil { - return nil, err - } - - if p.receiverTopic == "" { - return nil, errors.New("you didn't specify the topic to receive from") - } - p.Consumer = NewConsumerFromClient(p.Client, p.receiverGroupId, p.receiverTopic) - - return p, nil -} - -func (p *Protocol) applyOptions(opts ...ProtocolOptionFunc) error { - for _, fn := range opts { - fn(p) - } - return nil -} - -// OpenInbound implements Opener.OpenInbound -// NOTE: This is a blocking call. -func (p *Protocol) OpenInbound(ctx context.Context) error { - p.consumerMux.Lock() - defer p.consumerMux.Unlock() - - logger := cecontext.LoggerFrom(ctx) - logger.Infof("Starting consumer group to topic %s and group id %s", p.receiverTopic, p.receiverGroupId) - - return p.Consumer.OpenInbound(ctx) -} - -func (p *Protocol) Send(ctx context.Context, in binding.Message, transformers ...binding.Transformer) error { - for _, f := range p.SenderContextDecorators { - ctx = f(ctx) - } - return p.Sender.Send(ctx, in, transformers...) -} - -func (p *Protocol) Receive(ctx context.Context) (binding.Message, error) { - return p.Consumer.Receive(ctx) -} - -func (p *Protocol) Close(ctx context.Context) error { - if p.ownsClient { - // Just closing the client here closes at cascade consumer and producer - return p.Client.Close() - } - if err := p.Consumer.Close(ctx); err != nil { - return err - } - return p.Sender.Close(ctx) -} - -// Kafka protocol implements Sender, Receiver -var _ protocol.Sender = (*Protocol)(nil) -var _ protocol.Receiver = (*Protocol)(nil) -var _ protocol.Closer = (*Protocol)(nil) diff --git a/vendor/github.com/cloudevents/sdk-go/protocol/kafka_sarama/v2/receiver.go b/vendor/github.com/cloudevents/sdk-go/protocol/kafka_sarama/v2/receiver.go deleted file mode 100644 index 65f2c61d..00000000 --- a/vendor/github.com/cloudevents/sdk-go/protocol/kafka_sarama/v2/receiver.go +++ /dev/null @@ -1,175 +0,0 @@ -package kafka_sarama - -import ( - "context" - "io" - "sync" - - "github.com/Shopify/sarama" - - "github.com/cloudevents/sdk-go/v2/binding" - "github.com/cloudevents/sdk-go/v2/protocol" -) - -type msgErr struct { - msg binding.Message - err error -} - -// Receiver which implements sarama.ConsumerGroupHandler -// After the first invocation of Receiver.Receive(), the sarama.ConsumerGroup is created and started. -type Receiver struct { - once sync.Once - incoming chan msgErr -} - -// NewReceiver creates a Receiver which implements sarama.ConsumerGroupHandler -// The sarama.ConsumerGroup must be started invoking. If you need a Receiver which also manage the ConsumerGroup, use NewConsumer -// After the first invocation of Receiver.Receive(), the sarama.ConsumerGroup is created and started. -func NewReceiver() *Receiver { - return &Receiver{ - incoming: make(chan msgErr), - } -} - -func (r *Receiver) Setup(sess sarama.ConsumerGroupSession) error { - return nil -} - -func (r *Receiver) Cleanup(sarama.ConsumerGroupSession) error { - r.once.Do(func() { - close(r.incoming) - }) - return nil -} - -func (r *Receiver) ConsumeClaim(session sarama.ConsumerGroupSession, claim sarama.ConsumerGroupClaim) error { - for message := range claim.Messages() { - m := NewMessageFromConsumerMessage(message) - - r.incoming <- msgErr{ - msg: binding.WithFinish(m, func(err error) { - if protocol.IsACK(err) { - session.MarkMessage(message, "") - } - }), - } - } - return nil -} - -func (r *Receiver) Receive(ctx context.Context) (binding.Message, error) { - select { - case <-ctx.Done(): - return nil, io.EOF - case msgErr, ok := <-r.incoming: - if !ok { - return nil, io.EOF - } - return msgErr.msg, msgErr.err - } -} - -var _ protocol.Receiver = (*Receiver)(nil) - -type Consumer struct { - Receiver - - client sarama.Client - ownClient bool - - topic string - groupId string - - cgMtx sync.Mutex -} - -func NewConsumer(brokers []string, saramaConfig *sarama.Config, groupId string, topic string) (*Consumer, error) { - client, err := sarama.NewClient(brokers, saramaConfig) - if err != nil { - return nil, err - } - - consumer := NewConsumerFromClient(client, groupId, topic) - consumer.ownClient = true - - return consumer, nil -} - -func NewConsumerFromClient(client sarama.Client, groupId string, topic string) *Consumer { - return &Consumer{ - Receiver: Receiver{ - incoming: make(chan msgErr), - }, - client: client, - topic: topic, - groupId: groupId, - ownClient: false, - } -} - -func (c *Consumer) OpenInbound(ctx context.Context) error { - c.cgMtx.Lock() - defer c.cgMtx.Unlock() - cg, err := sarama.NewConsumerGroupFromClient(c.groupId, c.client) - if err != nil { - return err - } - - errCh := make(chan error) - - go c.startConsumerGroupLoop(cg, ctx, errCh) - - select { - case <-ctx.Done(): - return cg.Close() - case err = <-errCh: - // We still need to close this thing - err2 := cg.Close() - if err == nil { - err = err2 - } - // Somebody else closed the client, so no problem here - if err == sarama.ErrClosedClient || err == sarama.ErrClosedConsumerGroup { - return nil - } - return err - } -} - -func (c *Consumer) startConsumerGroupLoop(cg sarama.ConsumerGroup, ctx context.Context, errs chan<- error) { - // Need to be wrapped in a for loop - // https://godoc.org/github.com/Shopify/sarama#ConsumerGroup - for { - err := cg.Consume(context.Background(), []string{c.topic}, c) - - select { - // If context is closed, then consumer group session was closed by the user - case <-ctx.Done(): - if err != nil { - errs <- err - } - return - // Something else happened - default: - if err == nil || err == sarama.ErrClosedClient || err == sarama.ErrClosedConsumerGroup { - // Consumer group closed correctly, we can close that loop - return - } else { - // Another error happened (eg a disconnection to the cluster) - // We need to loop again - errs <- err - } - } - } -} - -func (c *Consumer) Close(ctx context.Context) error { - if c.ownClient { - return c.client.Close() - } - return nil -} - -var _ protocol.Opener = (*Consumer)(nil) -var _ protocol.Closer = (*Consumer)(nil) diff --git a/vendor/github.com/cloudevents/sdk-go/protocol/kafka_sarama/v2/sender.go b/vendor/github.com/cloudevents/sdk-go/protocol/kafka_sarama/v2/sender.go deleted file mode 100644 index 0c92399a..00000000 --- a/vendor/github.com/cloudevents/sdk-go/protocol/kafka_sarama/v2/sender.go +++ /dev/null @@ -1,83 +0,0 @@ -package kafka_sarama - -import ( - "context" - - "github.com/Shopify/sarama" - - "github.com/cloudevents/sdk-go/v2/binding" -) - -// Sender implements binding.Sender that sends messages to a specific receiverTopic using sarama.SyncProducer -type Sender struct { - topic string - syncProducer sarama.SyncProducer -} - -// NewSender returns a binding.Sender that sends messages to a specific receiverTopic using sarama.SyncProducer -func NewSender(brokers []string, saramaConfig *sarama.Config, topic string, options ...SenderOptionFunc) (*Sender, error) { - // Force this setting because it's required by sarama SyncProducer - saramaConfig.Producer.Return.Successes = true - producer, err := sarama.NewSyncProducer(brokers, saramaConfig) - if err != nil { - return nil, err - } - - return makeSender(producer, topic, options...), nil -} - -// NewSenderFromClient returns a binding.Sender that sends messages to a specific receiverTopic using sarama.SyncProducer -func NewSenderFromClient(client sarama.Client, topic string, options ...SenderOptionFunc) (*Sender, error) { - producer, err := sarama.NewSyncProducerFromClient(client) - if err != nil { - return nil, err - } - - return makeSender(producer, topic, options...), nil -} - -func makeSender(syncProducer sarama.SyncProducer, topic string, options ...SenderOptionFunc) *Sender { - s := &Sender{ - topic: topic, - syncProducer: syncProducer, - } - for _, o := range options { - o(s) - } - return s -} - -func (s *Sender) Send(ctx context.Context, m binding.Message, transformers ...binding.Transformer) error { - var err error - defer m.Finish(err) - - kafkaMessage := sarama.ProducerMessage{Topic: s.topic} - - if k := ctx.Value(withMessageKey{}); k != nil { - kafkaMessage.Key = k.(sarama.Encoder) - } - - if err = WriteProducerMessage(ctx, m, &kafkaMessage, transformers...); err != nil { - return err - } - - _, _, err = s.syncProducer.SendMessage(&kafkaMessage) - // Somebody closed the client while sending the message, so no problem here - if err == sarama.ErrClosedClient { - return nil - } - return err -} - -func (s *Sender) Close(ctx context.Context) error { - // If the Sender was built with NewSenderFromClient, this Close will close only the producer, - // otherwise it will close the whole client - return s.syncProducer.Close() -} - -type withMessageKey struct{} - -// WithMessageKey allows to set the key used when sending the producer message -func WithMessageKey(ctx context.Context, key sarama.Encoder) context.Context { - return context.WithValue(ctx, withMessageKey{}, key) -} diff --git a/vendor/github.com/cloudevents/sdk-go/protocol/kafka_sarama/v2/write_producer_message.go b/vendor/github.com/cloudevents/sdk-go/protocol/kafka_sarama/v2/write_producer_message.go deleted file mode 100644 index 3c786347..00000000 --- a/vendor/github.com/cloudevents/sdk-go/protocol/kafka_sarama/v2/write_producer_message.go +++ /dev/null @@ -1,158 +0,0 @@ -package kafka_sarama - -import ( - "bytes" - "context" - "io" - - "github.com/Shopify/sarama" - - "github.com/cloudevents/sdk-go/v2/binding" - "github.com/cloudevents/sdk-go/v2/binding/format" - "github.com/cloudevents/sdk-go/v2/binding/spec" - "github.com/cloudevents/sdk-go/v2/types" -) - -const ( - partitionKey = "partitionkey" -) - -// WriteProducerMessage fills the provided producerMessage with the message m. -// Using context you can tweak the encoding processing (more details on binding.Write documentation). -// By default, this function implements the key mapping, trying to set the key of the message based on partitionKey extension. -// If you want to disable the Key Mapping, decorate the context with `WithSkipKeyMapping` -func WriteProducerMessage(ctx context.Context, m binding.Message, producerMessage *sarama.ProducerMessage, transformers ...binding.Transformer) error { - writer := (*kafkaProducerMessageWriter)(producerMessage) - - skipKey := binding.GetOrDefaultFromCtx(ctx, skipKeyKey{}, false).(bool) - - var key string - - // If skipKey = false, then we add a transformer that extracts the key - if !skipKey { - transformers = append(transformers, binding.TransformerFunc(func(r binding.MessageMetadataReader, w binding.MessageMetadataWriter) error { - ext := r.GetExtension(partitionKey) - if !types.IsZero(ext) { - extStr, err := types.Format(ext) - if err != nil { - return err - } - key = extStr - } - return nil - })) - } - - _, err := binding.Write( - ctx, - m, - writer, - writer, - transformers..., - ) - if key != "" { - producerMessage.Key = sarama.StringEncoder(key) - } - return err -} - -type kafkaProducerMessageWriter sarama.ProducerMessage - -func (b *kafkaProducerMessageWriter) SetStructuredEvent(ctx context.Context, format format.Format, event io.Reader) error { - b.Headers = []sarama.RecordHeader{{ - Key: []byte(contentTypeHeader), - Value: []byte(format.MediaType()), - }} - - var buf bytes.Buffer - _, err := io.Copy(&buf, event) - if err != nil { - return err - } - - b.Value = sarama.ByteEncoder(buf.Bytes()) - return nil -} - -func (b *kafkaProducerMessageWriter) Start(ctx context.Context) error { - b.Headers = []sarama.RecordHeader{} - return nil -} - -func (b *kafkaProducerMessageWriter) End(ctx context.Context) error { - return nil -} - -func (b *kafkaProducerMessageWriter) SetData(reader io.Reader) error { - var buf bytes.Buffer - _, err := io.Copy(&buf, reader) - if err != nil { - return err - } - - b.Value = sarama.ByteEncoder(buf.Bytes()) - return nil -} - -func (b *kafkaProducerMessageWriter) SetAttribute(attribute spec.Attribute, value interface{}) error { - if attribute.Kind() == spec.DataContentType { - if value == nil { - b.removeHeader(contentTypeHeader) - return nil - } - - // Everything is a string here - s, err := types.Format(value) - if err != nil { - return err - } - b.Headers = append(b.Headers, sarama.RecordHeader{Key: []byte(contentTypeHeader), Value: []byte(s)}) - } else { - if value == nil { - b.removeHeader(prefix + attribute.Name()) - return nil - } - - // Everything is a string here - s, err := types.Format(value) - if err != nil { - return err - } - b.Headers = append(b.Headers, sarama.RecordHeader{Key: []byte(prefix + attribute.Name()), Value: []byte(s)}) - } - return nil -} - -func (b *kafkaProducerMessageWriter) SetExtension(name string, value interface{}) error { - if value == nil { - b.removeHeader(prefix + name) - return nil - } - - // Kafka headers, everything is a string! - s, err := types.Format(value) - if err != nil { - return err - } - b.Headers = append(b.Headers, sarama.RecordHeader{Key: []byte(prefix + name), Value: []byte(s)}) - return nil -} - -func (b *kafkaProducerMessageWriter) removeHeader(name string) { - k := []byte(name) - for index, h := range b.Headers { - if bytes.Equal(k, h.Key) { - b.Headers = append(b.Headers[:index], b.Headers[index+1:]...) - return - } - } -} - -type skipKeyKey struct{} - -func WithSkipKeyMapping(ctx context.Context) context.Context { - return context.WithValue(ctx, skipKeyKey{}, true) -} - -var _ binding.StructuredWriter = (*kafkaProducerMessageWriter)(nil) // Test it conforms to the interface -var _ binding.BinaryWriter = (*kafkaProducerMessageWriter)(nil) // Test it conforms to the interface diff --git a/vendor/github.com/cloudevents/sdk-go/v2/LICENSE b/vendor/github.com/cloudevents/sdk-go/v2/LICENSE deleted file mode 100644 index 261eeb9e..00000000 --- a/vendor/github.com/cloudevents/sdk-go/v2/LICENSE +++ /dev/null @@ -1,201 +0,0 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/vendor/github.com/cloudevents/sdk-go/v2/alias.go b/vendor/github.com/cloudevents/sdk-go/v2/alias.go deleted file mode 100644 index d60c10b2..00000000 --- a/vendor/github.com/cloudevents/sdk-go/v2/alias.go +++ /dev/null @@ -1,173 +0,0 @@ -// Package v2 reexports a subset of the SDK v2 API. -package v2 - -// Package cloudevents alias' common functions and types to improve discoverability and reduce -// the number of imports for simple HTTP clients. - -import ( - "github.com/cloudevents/sdk-go/v2/binding" - "github.com/cloudevents/sdk-go/v2/client" - "github.com/cloudevents/sdk-go/v2/context" - "github.com/cloudevents/sdk-go/v2/event" - "github.com/cloudevents/sdk-go/v2/observability" - "github.com/cloudevents/sdk-go/v2/protocol" - "github.com/cloudevents/sdk-go/v2/protocol/http" - "github.com/cloudevents/sdk-go/v2/types" -) - -// Client - -type ClientOption client.Option -type Client = client.Client - -// Event - -type Event = event.Event -type Result = protocol.Result - -// Context - -type EventContext = event.EventContext -type EventContextV1 = event.EventContextV1 -type EventContextV03 = event.EventContextV03 - -// Custom Types - -type Timestamp = types.Timestamp -type URIRef = types.URIRef - -// HTTP Protocol - -type HTTPOption http.Option - -type HTTPProtocol = http.Protocol - -// Encoding - -type Encoding = binding.Encoding - -// Message - -type Message = binding.Message - -const ( - // ReadEncoding - - ApplicationXML = event.ApplicationXML - ApplicationJSON = event.ApplicationJSON - TextPlain = event.TextPlain - ApplicationCloudEventsJSON = event.ApplicationCloudEventsJSON - ApplicationCloudEventsBatchJSON = event.ApplicationCloudEventsBatchJSON - Base64 = event.Base64 - - // Event Versions - - VersionV1 = event.CloudEventsVersionV1 - VersionV03 = event.CloudEventsVersionV03 - - // Encoding - - EncodingBinary = binding.EncodingBinary - EncodingStructured = binding.EncodingStructured -) - -var ( - - // ContentType Helpers - - StringOfApplicationJSON = event.StringOfApplicationJSON - StringOfApplicationXML = event.StringOfApplicationXML - StringOfTextPlain = event.StringOfTextPlain - StringOfApplicationCloudEventsJSON = event.StringOfApplicationCloudEventsJSON - StringOfApplicationCloudEventsBatchJSON = event.StringOfApplicationCloudEventsBatchJSON - StringOfBase64 = event.StringOfBase64 - - // Client Creation - - NewClient = client.New - NewClientObserved = client.NewObserved - NewDefaultClient = client.NewDefault - NewHTTPReceiveHandler = client.NewHTTPReceiveHandler - - // Client Options - - WithEventDefaulter = client.WithEventDefaulter - WithUUIDs = client.WithUUIDs - WithTimeNow = client.WithTimeNow - WithTracePropagation = client.WithTracePropagation() - - // Event Creation - - NewEvent = event.New - - // Results - - NewResult = protocol.NewResult - ResultIs = protocol.ResultIs - ResultAs = protocol.ResultAs - - // Receipt helpers - - NewReceipt = protocol.NewReceipt - - ResultACK = protocol.ResultACK - ResultNACK = protocol.ResultNACK - - IsACK = protocol.IsACK - IsNACK = protocol.IsNACK - IsUndelivered = protocol.IsUndelivered - - // HTTP Results - - NewHTTPResult = http.NewResult - NewHTTPRetriesResult = http.NewRetriesResult - - // Message Creation - - ToMessage = binding.ToMessage - - // HTTP Messages - - WriteHTTPRequest = http.WriteRequest - - // Tracing - - EnableTracing = observability.EnableTracing - - // Context - - ContextWithTarget = context.WithTarget - TargetFromContext = context.TargetFrom - ContextWithRetriesConstantBackoff = context.WithRetriesConstantBackoff - ContextWithRetriesLinearBackoff = context.WithRetriesLinearBackoff - ContextWithRetriesExponentialBackoff = context.WithRetriesExponentialBackoff - - WithEncodingBinary = binding.WithForceBinary - WithEncodingStructured = binding.WithForceStructured - - // Custom Types - - ParseTimestamp = types.ParseTimestamp - ParseURIRef = types.ParseURIRef - ParseURI = types.ParseURI - - // HTTP Protocol - - NewHTTP = http.New - - // HTTP Protocol Options - - WithTarget = http.WithTarget - WithHeader = http.WithHeader - WithShutdownTimeout = http.WithShutdownTimeout - //WithEncoding = http.WithEncoding - //WithStructuredEncoding = http.WithStructuredEncoding // TODO: expose new way - WithPort = http.WithPort - WithPath = http.WithPath - WithMiddleware = http.WithMiddleware - WithListener = http.WithListener - WithRoundTripper = http.WithRoundTripper - WithGetHandlerFunc = http.WithGetHandlerFunc - WithOptionsHandlerFunc = http.WithOptionsHandlerFunc - WithDefaultOptionsHandlerFunc = http.WithDefaultOptionsHandlerFunc -) diff --git a/vendor/github.com/cloudevents/sdk-go/v2/binding/binary_writer.go b/vendor/github.com/cloudevents/sdk-go/v2/binding/binary_writer.go deleted file mode 100644 index a99cd0b7..00000000 --- a/vendor/github.com/cloudevents/sdk-go/v2/binding/binary_writer.go +++ /dev/null @@ -1,47 +0,0 @@ -package binding - -import ( - "context" - "io" - - "github.com/cloudevents/sdk-go/v2/binding/spec" -) - -// MessageMetadataWriter is used to set metadata when a binary Message is visited. -type MessageMetadataWriter interface { - // Set a standard attribute. - // - // The value can either be the correct golang type for the attribute, or a canonical - // string encoding, or nil. If value is nil, then the attribute should be deleted. - // See package types to perform the needed conversions. - SetAttribute(attribute spec.Attribute, value interface{}) error - - // Set an extension attribute. - // - // The value can either be the correct golang type for the attribute, or a canonical - // string encoding, or nil. If value is nil, then the extension should be deleted. - // See package types to perform the needed conversions. - SetExtension(name string, value interface{}) error -} - -// BinaryWriter is used to visit a binary Message and generate a new representation. -// -// Protocols that supports binary encoding should implement this interface to implement direct -// binary to binary encoding and event to binary encoding. -// -// Start() and End() methods must be invoked by the caller of Message.ReadBinary() every time -// the BinaryWriter implementation is used to visit a Message. -type BinaryWriter interface { - MessageMetadataWriter - - // Method invoked at the beginning of the visit. Useful to perform initial memory allocations - Start(ctx context.Context) error - - // SetData receives an io.Reader for the data attribute. - // io.Reader is not invoked when the data attribute is empty - SetData(data io.Reader) error - - // End method is invoked only after the whole encoding process ends successfully. - // If it fails, it's never invoked. It can be used to finalize the message. - End(ctx context.Context) error -} diff --git a/vendor/github.com/cloudevents/sdk-go/v2/binding/doc.go b/vendor/github.com/cloudevents/sdk-go/v2/binding/doc.go deleted file mode 100644 index 1176fad8..00000000 --- a/vendor/github.com/cloudevents/sdk-go/v2/binding/doc.go +++ /dev/null @@ -1,63 +0,0 @@ -/* - -Package binding defines interfaces for protocol bindings. - -NOTE: Most applications that emit or consume events should use the ../client -package, which provides a simpler API to the underlying binding. - -The interfaces in this package provide extra encoding and protocol information -to allow efficient forwarding and end-to-end reliable delivery between a -Receiver and a Sender belonging to different bindings. This is useful for -intermediary applications that route or forward events, but not necessary for -most "endpoint" applications that emit or consume events. - -Protocol Bindings - -A protocol binding usually implements a Message, a Sender and Receiver, a StructuredWriter and a BinaryWriter (depending on the supported encodings of the protocol) and an Write[ProtocolMessage] method. - -Read and write events - -The core of this package is the binding.Message interface. -Through binding.MessageReader It defines how to read a protocol specific message for an -encoded event in structured mode or binary mode. -The entity who receives a protocol specific data structure representing a message -(e.g. an HttpRequest) encapsulates it in a binding.Message implementation using a NewMessage method (e.g. http.NewMessage). -Then the entity that wants to send the binding.Message back on the wire, -translates it back to the protocol specific data structure (e.g. a Kafka ConsumerMessage), using -the writers BinaryWriter and StructuredWriter specific to that protocol. -Binding implementations exposes their writers -through a specific Write[ProtocolMessage] function (e.g. kafka.EncodeProducerMessage), -in order to simplify the encoding process. - -The encoding process can be customized in order to mutate the final result with binding.TransformerFactory. -A bunch of these are provided directly by the binding/transformer module. - -Usually binding.Message implementations can be encoded only one time, because the encoding process drain the message itself. -In order to consume a message several times, the binding/buffering package provides several APIs to buffer the Message. - -A message can be converted to an event.Event using binding.ToEvent() method. -An event.Event can be used as Message casting it to binding.EventMessage. - -In order to simplify the encoding process for each protocol, this package provide several utility methods like binding.Write and binding.DirectWrite. -The binding.Write method tries to preserve the structured/binary encoding, in order to be as much efficient as possible. - -Messages can be eventually wrapped to change their behaviours and binding their lifecycle, like the binding.FinishMessage. -Every Message wrapper implements the MessageWrapper interface - -Sender and Receiver - -A Receiver receives protocol specific messages and wraps them to into binding.Message implementations. - -A Sender converts arbitrary Message implementations to a protocol-specific form using the protocol specific Write method -and sends them. - -Message and ExactlyOnceMessage provide methods to allow acknowledgments to -propagate when a reliable messages is forwarded from a Receiver to a Sender. -QoS 0 (unreliable), 1 (at-least-once) and 2 (exactly-once) are supported. - -Transport - -A binding implementation providing Sender and Receiver implementations can be used as a Transport through the BindingTransport adapter. - -*/ -package binding diff --git a/vendor/github.com/cloudevents/sdk-go/v2/binding/encoding.go b/vendor/github.com/cloudevents/sdk-go/v2/binding/encoding.go deleted file mode 100644 index 0b6efe63..00000000 --- a/vendor/github.com/cloudevents/sdk-go/v2/binding/encoding.go +++ /dev/null @@ -1,40 +0,0 @@ -package binding - -import "errors" - -// Encoding enum specifies the type of encodings supported by binding interfaces -type Encoding int - -const ( - // Binary encoding as specified in https://github.com/cloudevents/spec/blob/master/spec.md#message - EncodingBinary Encoding = iota - // Structured encoding as specified in https://github.com/cloudevents/spec/blob/master/spec.md#message - EncodingStructured - // Message is an instance of EventMessage or it contains EventMessage nested (through MessageWrapper) - EncodingEvent - // When the encoding is unknown (which means that the message is a non-event) - EncodingUnknown -) - -func (e Encoding) String() string { - switch e { - case EncodingBinary: - return "binary" - case EncodingStructured: - return "structured" - case EncodingEvent: - return "event" - case EncodingUnknown: - return "unknown" - } - return "" -} - -// ErrUnknownEncoding specifies that the Message is not an event or it is encoded with an unknown encoding -var ErrUnknownEncoding = errors.New("unknown Message encoding") - -// ErrNotStructured returned by Message.Structured for non-structured messages. -var ErrNotStructured = errors.New("message is not in structured mode") - -// ErrNotBinary returned by Message.Binary for non-binary messages. -var ErrNotBinary = errors.New("message is not in binary mode") diff --git a/vendor/github.com/cloudevents/sdk-go/v2/binding/event_message.go b/vendor/github.com/cloudevents/sdk-go/v2/binding/event_message.go deleted file mode 100644 index 7afc13d1..00000000 --- a/vendor/github.com/cloudevents/sdk-go/v2/binding/event_message.go +++ /dev/null @@ -1,103 +0,0 @@ -package binding - -import ( - "bytes" - "context" - - "github.com/cloudevents/sdk-go/v2/binding/format" - "github.com/cloudevents/sdk-go/v2/binding/spec" - "github.com/cloudevents/sdk-go/v2/event" -) - -type eventFormatKey int - -const ( - formatEventStructured eventFormatKey = iota -) - -// EventMessage type-converts a event.Event object to implement Message. -// This allows local event.Event objects to be sent directly via Sender.Send() -// s.Send(ctx, binding.EventMessage(e)) -// When an event is wrapped into a EventMessage, the original event could be -// potentially mutated. If you need to use the Event again, after wrapping it into -// an Event message, you should copy it before -type EventMessage event.Event - -func ToMessage(e *event.Event) Message { - return (*EventMessage)(e) -} - -func (m *EventMessage) ReadEncoding() Encoding { - return EncodingEvent -} - -func (m *EventMessage) ReadStructured(ctx context.Context, builder StructuredWriter) error { - f := GetOrDefaultFromCtx(ctx, formatEventStructured, format.JSON).(format.Format) - b, err := f.Marshal((*event.Event)(m)) - if err != nil { - return err - } - return builder.SetStructuredEvent(ctx, f, bytes.NewReader(b)) -} - -func (m *EventMessage) ReadBinary(ctx context.Context, b BinaryWriter) (err error) { - err = eventContextToBinaryWriter(m.Context, b) - if err != nil { - return err - } - // Pass the body - body := (*event.Event)(m).Data() - if len(body) > 0 { - err = b.SetData(bytes.NewReader(body)) - if err != nil { - return err - } - } - return nil -} - -func (m *EventMessage) GetAttribute(k spec.Kind) (spec.Attribute, interface{}) { - sv := spec.VS.Version(m.Context.GetSpecVersion()) - a := sv.AttributeFromKind(k) - if a != nil { - return a, a.Get(m.Context) - } - return nil, nil -} - -func (m *EventMessage) GetExtension(name string) interface{} { - ext, _ := m.Context.GetExtension(name) - return ext -} - -func eventContextToBinaryWriter(c event.EventContext, b BinaryWriter) (err error) { - // Pass all attributes - sv := spec.VS.Version(c.GetSpecVersion()) - for _, a := range sv.Attributes() { - value := a.Get(c) - if value != nil { - err = b.SetAttribute(a, value) - } - if err != nil { - return err - } - } - // Pass all extensions - for k, v := range c.GetExtensions() { - err = b.SetExtension(k, v) - if err != nil { - return err - } - } - return nil -} - -func (*EventMessage) Finish(error) error { return nil } - -var _ Message = (*EventMessage)(nil) // Test it conforms to the interface -var _ MessageMetadataReader = (*EventMessage)(nil) // Test it conforms to the interface - -// UseFormatForEvent configures which format to use when marshalling the event to structured mode -func UseFormatForEvent(ctx context.Context, f format.Format) context.Context { - return context.WithValue(ctx, formatEventStructured, f) -} diff --git a/vendor/github.com/cloudevents/sdk-go/v2/binding/finish_message.go b/vendor/github.com/cloudevents/sdk-go/v2/binding/finish_message.go deleted file mode 100644 index 17445bfe..00000000 --- a/vendor/github.com/cloudevents/sdk-go/v2/binding/finish_message.go +++ /dev/null @@ -1,37 +0,0 @@ -package binding - -import "github.com/cloudevents/sdk-go/v2/binding/spec" - -type finishMessage struct { - Message - finish func(error) -} - -func (m *finishMessage) GetAttribute(k spec.Kind) (spec.Attribute, interface{}) { - return m.Message.(MessageMetadataReader).GetAttribute(k) -} - -func (m *finishMessage) GetExtension(s string) interface{} { - return m.Message.(MessageMetadataReader).GetExtension(s) -} - -func (m *finishMessage) GetWrappedMessage() Message { - return m.Message -} - -func (m *finishMessage) Finish(err error) error { - err2 := m.Message.Finish(err) // Finish original message first - if m.finish != nil { - m.finish(err) // Notify callback - } - return err2 -} - -var _ MessageWrapper = (*finishMessage)(nil) - -// WithFinish returns a wrapper for m that calls finish() and -// m.Finish() in its Finish(). -// Allows code to be notified when a message is Finished. -func WithFinish(m Message, finish func(error)) Message { - return &finishMessage{Message: m, finish: finish} -} diff --git a/vendor/github.com/cloudevents/sdk-go/v2/binding/format/doc.go b/vendor/github.com/cloudevents/sdk-go/v2/binding/format/doc.go deleted file mode 100644 index ab153afb..00000000 --- a/vendor/github.com/cloudevents/sdk-go/v2/binding/format/doc.go +++ /dev/null @@ -1,7 +0,0 @@ -/* -Package format formats structured events. - -The "application/cloudevents+json" format is built-in and always -available. Other formats may be added. -*/ -package format diff --git a/vendor/github.com/cloudevents/sdk-go/v2/binding/format/format.go b/vendor/github.com/cloudevents/sdk-go/v2/binding/format/format.go deleted file mode 100644 index 9e2b1ec6..00000000 --- a/vendor/github.com/cloudevents/sdk-go/v2/binding/format/format.go +++ /dev/null @@ -1,78 +0,0 @@ -package format - -import ( - "encoding/json" - "fmt" - "strings" - - "github.com/cloudevents/sdk-go/v2/event" -) - -// Format marshals and unmarshals structured events to bytes. -type Format interface { - // MediaType identifies the format - MediaType() string - // Marshal event to bytes - Marshal(*event.Event) ([]byte, error) - // Unmarshal bytes to event - Unmarshal([]byte, *event.Event) error -} - -// Prefix for event-format media types. -const Prefix = "application/cloudevents" - -// IsFormat returns true if mediaType begins with "application/cloudevents" -func IsFormat(mediaType string) bool { return strings.HasPrefix(mediaType, Prefix) } - -// JSON is the built-in "application/cloudevents+json" format. -var JSON = jsonFmt{} - -type jsonFmt struct{} - -func (jsonFmt) MediaType() string { return event.ApplicationCloudEventsJSON } - -func (jsonFmt) Marshal(e *event.Event) ([]byte, error) { return json.Marshal(e) } -func (jsonFmt) Unmarshal(b []byte, e *event.Event) error { - return json.Unmarshal(b, e) -} - -// built-in formats -var formats map[string]Format - -func init() { - formats = map[string]Format{} - Add(JSON) -} - -// Lookup returns the format for contentType, or nil if not found. -func Lookup(contentType string) Format { - i := strings.IndexRune(contentType, ';') - if i == -1 { - i = len(contentType) - } - contentType = strings.TrimSpace(strings.ToLower(contentType[0:i])) - return formats[contentType] -} - -func unknown(mediaType string) error { - return fmt.Errorf("unknown event format media-type %#v", mediaType) -} - -// Add a new Format. It can be retrieved by Lookup(f.MediaType()) -func Add(f Format) { formats[f.MediaType()] = f } - -// Marshal an event to bytes using the mediaType event format. -func Marshal(mediaType string, e *event.Event) ([]byte, error) { - if f := formats[mediaType]; f != nil { - return f.Marshal(e) - } - return nil, unknown(mediaType) -} - -// Unmarshal bytes to an event using the mediaType event format. -func Unmarshal(mediaType string, b []byte, e *event.Event) error { - if f := formats[mediaType]; f != nil { - return f.Unmarshal(b, e) - } - return unknown(mediaType) -} diff --git a/vendor/github.com/cloudevents/sdk-go/v2/binding/message.go b/vendor/github.com/cloudevents/sdk-go/v2/binding/message.go deleted file mode 100644 index 7222f715..00000000 --- a/vendor/github.com/cloudevents/sdk-go/v2/binding/message.go +++ /dev/null @@ -1,141 +0,0 @@ -package binding - -import ( - "context" - - "github.com/cloudevents/sdk-go/v2/binding/spec" -) - -// MessageReader defines the read-related portion of the Message interface. -// -// The ReadStructured and ReadBinary methods allows to perform an optimized encoding of a Message to a specific data structure. -// -// If MessageReader.ReadEncoding() can be equal to EncodingBinary, then the implementation of MessageReader -// MUST also implement MessageMetadataReader. -// -// A Sender should try each method of interest and fall back to binding.ToEvent() if none are supported. -// An out of the box algorithm is provided for writing a message: binding.Write(). -type MessageReader interface { - // Return the type of the message Encoding. - // The encoding should be preferably computed when the message is constructed. - ReadEncoding() Encoding - - // ReadStructured transfers a structured-mode event to a StructuredWriter. - // It must return ErrNotStructured if message is not in structured mode. - // - // Returns a different err if something wrong happened while trying to read the structured event. - // In this case, the caller must Finish the message with appropriate error. - // - // This allows Senders to avoid re-encoding messages that are - // already in suitable structured form. - ReadStructured(context.Context, StructuredWriter) error - - // ReadBinary transfers a binary-mode event to an BinaryWriter. - // It must return ErrNotBinary if message is not in binary mode. - // - // The implementation of ReadBinary must not control the lifecycle with BinaryWriter.Start() and BinaryWriter.End(), - // because the caller must control the lifecycle. - // - // Returns a different err if something wrong happened while trying to read the binary event - // In this case, the caller must Finish the message with appropriate error - // - // This allows Senders to avoid re-encoding messages that are - // already in suitable binary form. - ReadBinary(context.Context, BinaryWriter) error -} - -// MessageMetadataReader defines how to read metadata from a binary/event message -// -// If a message implementing MessageReader is encoded as binary (MessageReader.ReadEncoding() == EncodingBinary) -// or it's an EventMessage, then it's safe to assume that it also implements this interface -type MessageMetadataReader interface { - // GetAttribute returns: - // - // * attribute, value: if the message contains an attribute of that attribute kind - // * attribute, nil: if the message spec version supports the attribute kind, but doesn't have any value - // * nil, nil: if the message spec version doesn't support the attribute kind - GetAttribute(attributeKind spec.Kind) (spec.Attribute, interface{}) - // GetExtension returns the value of that extension, if any. - GetExtension(name string) interface{} -} - -// Message is the interface to a binding-specific message containing an event. -// -// Reliable Delivery -// -// There are 3 reliable qualities of service for messages: -// -// 0/at-most-once/unreliable: messages can be dropped silently. -// -// 1/at-least-once: messages are not dropped without signaling an error -// to the sender, but they may be duplicated in the event of a re-send. -// -// 2/exactly-once: messages are never dropped (without error) or -// duplicated, as long as both sending and receiving ends maintain -// some binding-specific delivery state. Whether this is persisted -// depends on the configuration of the binding implementations. -// -// The Message interface supports QoS 0 and 1, the ExactlyOnceMessage interface -// supports QoS 2 -// -// Message includes the MessageReader interface to read messages. Every binding.Message implementation *must* specify if the message can be accessed one or more times. -// -// When a Message can be forgotten by the entity who produced the message, Message.Finish() *must* be invoked. -type Message interface { - MessageReader - - // Finish *must* be called when message from a Receiver can be forgotten by - // the receiver. A QoS 1 sender should not call Finish() until it gets an acknowledgment of - // receipt on the underlying transport. For QoS 2 see ExactlyOnceMessage. - // - // Note that, depending on the Message implementation, forgetting to Finish the message - // could produce memory/resources leaks! - // - // Passing a non-nil err indicates sending or processing failed. - // A non-nil return indicates that the message was not accepted - // by the receivers peer. - Finish(error) error -} - -// ExactlyOnceMessage is implemented by received Messages -// that support QoS 2. Only transports that support QoS 2 need to -// implement or use this interface. -type ExactlyOnceMessage interface { - Message - - // Received is called by a forwarding QoS2 Sender when it gets - // acknowledgment of receipt (e.g. AMQP 'accept' or MQTT PUBREC) - // - // The receiver must call settle(nil) when it get's the ack-of-ack - // (e.g. AMQP 'settle' or MQTT PUBCOMP) or settle(err) if the - // transfer fails. - // - // Finally the Sender calls Finish() to indicate the message can be - // discarded. - // - // If sending fails, or if the sender does not support QoS 2, then - // Finish() may be called without any call to Received() - Received(settle func(error)) -} - -// MessageWrapper interface is used to walk through a decorated Message and unwrap it. -type MessageWrapper interface { - Message - MessageMetadataReader - - // Method to get the wrapped message - GetWrappedMessage() Message -} - -func UnwrapMessage(message Message) Message { - m := message - for m != nil { - switch mt := m.(type) { - case MessageWrapper: - m = mt.GetWrappedMessage() - default: - return m - } - } - return m -} diff --git a/vendor/github.com/cloudevents/sdk-go/v2/binding/spec/attributes.go b/vendor/github.com/cloudevents/sdk-go/v2/binding/spec/attributes.go deleted file mode 100644 index 20ec1ce9..00000000 --- a/vendor/github.com/cloudevents/sdk-go/v2/binding/spec/attributes.go +++ /dev/null @@ -1,136 +0,0 @@ -package spec - -import ( - "fmt" - "time" - - "github.com/cloudevents/sdk-go/v2/event" - - "github.com/cloudevents/sdk-go/v2/types" -) - -// Kind is a version-independent identifier for a CloudEvent context attribute. -type Kind uint8 - -const ( - // Required cloudevents attributes - ID Kind = iota - Source - SpecVersion - Type - // Optional cloudevents attributes - DataContentType - DataSchema - Subject - Time -) -const nAttrs = int(Time) + 1 - -var kindNames = [nAttrs]string{ - "id", - "source", - "specversion", - "type", - "datacontenttype", - "dataschema", - "subject", - "time", -} - -// String is a human-readable string, for a valid attribute name use Attribute.Name -func (k Kind) String() string { return kindNames[k] } - -// IsRequired returns true for attributes defined as "required" by the CE spec. -func (k Kind) IsRequired() bool { return k < DataContentType } - -// Attribute is a named attribute accessor. -// The attribute name is specific to a Version. -type Attribute interface { - Kind() Kind - // Name of the attribute with respect to the current spec Version() with prefix - PrefixedName() string - // Name of the attribute with respect to the current spec Version() - Name() string - // Version of the spec that this attribute belongs to - Version() Version - // Get the value of this attribute from an event context - Get(event.EventContextReader) interface{} - // Set the value of this attribute on an event context - Set(event.EventContextWriter, interface{}) error - // Delete this attribute from and event context, when possible - Delete(event.EventContextWriter) error -} - -// accessor provides Kind, Get, Set. -type accessor interface { - Kind() Kind - Get(event.EventContextReader) interface{} - Set(event.EventContextWriter, interface{}) error - Delete(event.EventContextWriter) error -} - -var acc = [nAttrs]accessor{ - &aStr{aKind(ID), event.EventContextReader.GetID, event.EventContextWriter.SetID}, - &aStr{aKind(Source), event.EventContextReader.GetSource, event.EventContextWriter.SetSource}, - &aStr{aKind(SpecVersion), event.EventContextReader.GetSpecVersion, func(writer event.EventContextWriter, s string) error { return nil }}, - &aStr{aKind(Type), event.EventContextReader.GetType, event.EventContextWriter.SetType}, - &aStr{aKind(DataContentType), event.EventContextReader.GetDataContentType, event.EventContextWriter.SetDataContentType}, - &aStr{aKind(DataSchema), event.EventContextReader.GetDataSchema, event.EventContextWriter.SetDataSchema}, - &aStr{aKind(Subject), event.EventContextReader.GetSubject, event.EventContextWriter.SetSubject}, - &aTime{aKind(Time), event.EventContextReader.GetTime, event.EventContextWriter.SetTime}, -} - -// aKind implements Kind() -type aKind Kind - -func (kind aKind) Kind() Kind { return Kind(kind) } - -type aStr struct { - aKind - get func(event.EventContextReader) string - set func(event.EventContextWriter, string) error -} - -func (a *aStr) Get(c event.EventContextReader) interface{} { - if s := a.get(c); s != "" { - return s - } - return nil // Treat blank as missing -} - -func (a *aStr) Set(c event.EventContextWriter, v interface{}) error { - s, err := types.ToString(v) - if err != nil { - return fmt.Errorf("invalid value for %s: %#v", a.Kind(), v) - } - return a.set(c, s) -} - -func (a *aStr) Delete(c event.EventContextWriter) error { - return a.set(c, "") -} - -type aTime struct { - aKind - get func(event.EventContextReader) time.Time - set func(event.EventContextWriter, time.Time) error -} - -func (a *aTime) Get(c event.EventContextReader) interface{} { - if v := a.get(c); !v.IsZero() { - return v - } - return nil // Treat zero time as missing. -} - -func (a *aTime) Set(c event.EventContextWriter, v interface{}) error { - t, err := types.ToTime(v) - if err != nil { - return fmt.Errorf("invalid value for %s: %#v", a.Kind(), v) - } - return a.set(c, t) -} - -func (a *aTime) Delete(c event.EventContextWriter) error { - return a.set(c, time.Time{}) -} diff --git a/vendor/github.com/cloudevents/sdk-go/v2/binding/spec/doc.go b/vendor/github.com/cloudevents/sdk-go/v2/binding/spec/doc.go deleted file mode 100644 index 38d6fddf..00000000 --- a/vendor/github.com/cloudevents/sdk-go/v2/binding/spec/doc.go +++ /dev/null @@ -1,8 +0,0 @@ -/* -Package spec provides spec-version metadata. - -For use by code that maps events using (prefixed) attribute name strings. -Supports handling multiple spec versions uniformly. - -*/ -package spec diff --git a/vendor/github.com/cloudevents/sdk-go/v2/binding/spec/match_exact_version.go b/vendor/github.com/cloudevents/sdk-go/v2/binding/spec/match_exact_version.go deleted file mode 100644 index 5976faf1..00000000 --- a/vendor/github.com/cloudevents/sdk-go/v2/binding/spec/match_exact_version.go +++ /dev/null @@ -1,76 +0,0 @@ -package spec - -import ( - "github.com/cloudevents/sdk-go/v2/event" -) - -type matchExactVersion struct { - version -} - -func (v *matchExactVersion) Attribute(name string) Attribute { return v.attrMap[name] } - -var _ Version = (*matchExactVersion)(nil) - -func newMatchExactVersionVersion( - prefix string, - attributeNameMatchMapper func(string) string, - context event.EventContext, - convert func(event.EventContextConverter) event.EventContext, - attrs ...*attribute, -) *matchExactVersion { - v := &matchExactVersion{ - version: version{ - prefix: prefix, - context: context, - convert: convert, - attrMap: map[string]Attribute{}, - attrs: make([]Attribute, len(attrs)), - }, - } - for i, a := range attrs { - a.version = v - v.attrs[i] = a - v.attrMap[attributeNameMatchMapper(a.name)] = a - } - return v -} - -// WithPrefixMatchExact returns a set of versions with prefix added to all attribute names. -func WithPrefixMatchExact(attributeNameMatchMapper func(string) string, prefix string) *Versions { - attr := func(name string, kind Kind) *attribute { - return &attribute{accessor: acc[kind], name: name} - } - vs := &Versions{ - m: map[string]Version{}, - prefix: prefix, - all: []Version{ - newMatchExactVersionVersion(prefix, attributeNameMatchMapper, event.EventContextV1{}.AsV1(), - func(c event.EventContextConverter) event.EventContext { return c.AsV1() }, - attr("id", ID), - attr("source", Source), - attr("specversion", SpecVersion), - attr("type", Type), - attr("datacontenttype", DataContentType), - attr("dataschema", DataSchema), - attr("subject", Subject), - attr("time", Time), - ), - newMatchExactVersionVersion(prefix, attributeNameMatchMapper, event.EventContextV03{}.AsV03(), - func(c event.EventContextConverter) event.EventContext { return c.AsV03() }, - attr("specversion", SpecVersion), - attr("type", Type), - attr("source", Source), - attr("schemaurl", DataSchema), - attr("subject", Subject), - attr("id", ID), - attr("time", Time), - attr("datacontenttype", DataContentType), - ), - }, - } - for _, v := range vs.all { - vs.m[v.String()] = v - } - return vs -} diff --git a/vendor/github.com/cloudevents/sdk-go/v2/binding/spec/spec.go b/vendor/github.com/cloudevents/sdk-go/v2/binding/spec/spec.go deleted file mode 100644 index 4de58918..00000000 --- a/vendor/github.com/cloudevents/sdk-go/v2/binding/spec/spec.go +++ /dev/null @@ -1,184 +0,0 @@ -package spec - -import ( - "strings" - - "github.com/cloudevents/sdk-go/v2/event" -) - -// Version provides meta-data for a single spec-version. -type Version interface { - // String name of the version, e.g. "1.0" - String() string - // Prefix for attribute names. - Prefix() string - // Attribute looks up a prefixed attribute name (case insensitive). - // Returns nil if not found. - Attribute(prefixedName string) Attribute - // Attribute looks up the attribute from kind. - // Returns nil if not found. - AttributeFromKind(kind Kind) Attribute - // Attributes returns all the context attributes for this version. - Attributes() []Attribute - // Convert translates a context to this version. - Convert(event.EventContextConverter) event.EventContext - // NewContext returns a new context for this version. - NewContext() event.EventContext - // SetAttribute sets named attribute to value. - // - // Name is case insensitive. - // Does nothing if name does not start with prefix. - SetAttribute(context event.EventContextWriter, name string, value interface{}) error -} - -// Versions contains all known versions with the same attribute prefix. -type Versions struct { - prefix string - all []Version - m map[string]Version -} - -// Versions returns the list of all known versions, most recent first. -func (vs *Versions) Versions() []Version { return vs.all } - -// Version returns the named version. -func (vs *Versions) Version(name string) Version { - return vs.m[name] -} - -// Latest returns the latest Version -func (vs *Versions) Latest() Version { return vs.all[0] } - -// PrefixedSpecVersionName returns the specversion attribute PrefixedName -func (vs *Versions) PrefixedSpecVersionName() string { return vs.prefix + "specversion" } - -// Prefix is the lowercase attribute name prefix. -func (vs *Versions) Prefix() string { return vs.prefix } - -type attribute struct { - accessor - name string - version Version -} - -func (a *attribute) PrefixedName() string { return a.version.Prefix() + a.name } -func (a *attribute) Name() string { return a.name } -func (a *attribute) Version() Version { return a.version } - -type version struct { - prefix string - context event.EventContext - convert func(event.EventContextConverter) event.EventContext - attrMap map[string]Attribute - attrs []Attribute -} - -func (v *version) Attribute(name string) Attribute { return v.attrMap[strings.ToLower(name)] } -func (v *version) Attributes() []Attribute { return v.attrs } -func (v *version) String() string { return v.context.GetSpecVersion() } -func (v *version) Prefix() string { return v.prefix } -func (v *version) NewContext() event.EventContext { return v.context.Clone() } - -// HasPrefix is a case-insensitive prefix check. -func (v *version) HasPrefix(name string) bool { - return strings.HasPrefix(strings.ToLower(name), v.prefix) -} - -func (v *version) Convert(c event.EventContextConverter) event.EventContext { return v.convert(c) } - -func (v *version) SetAttribute(c event.EventContextWriter, name string, value interface{}) error { - if a := v.Attribute(name); a != nil { // Standard attribute - return a.Set(c, value) - } - name = strings.ToLower(name) - var err error - if v.HasPrefix(name) { // Extension attribute - return c.SetExtension(strings.TrimPrefix(name, v.prefix), value) - } - return err -} - -func (v *version) AttributeFromKind(kind Kind) Attribute { - for _, a := range v.Attributes() { - if a.Kind() == kind { - return a - } - } - return nil -} - -func newVersion( - prefix string, - context event.EventContext, - convert func(event.EventContextConverter) event.EventContext, - attrs ...*attribute, -) *version { - v := &version{ - prefix: strings.ToLower(prefix), - context: context, - convert: convert, - attrMap: map[string]Attribute{}, - attrs: make([]Attribute, len(attrs)), - } - for i, a := range attrs { - a.version = v - v.attrs[i] = a - v.attrMap[strings.ToLower(a.PrefixedName())] = a - } - return v -} - -// WithPrefix returns a set of versions with prefix added to all attribute names. -func WithPrefix(prefix string) *Versions { - attr := func(name string, kind Kind) *attribute { - return &attribute{accessor: acc[kind], name: name} - } - vs := &Versions{ - m: map[string]Version{}, - prefix: prefix, - all: []Version{ - newVersion(prefix, event.EventContextV1{}.AsV1(), - func(c event.EventContextConverter) event.EventContext { return c.AsV1() }, - attr("id", ID), - attr("source", Source), - attr("specversion", SpecVersion), - attr("type", Type), - attr("datacontenttype", DataContentType), - attr("dataschema", DataSchema), - attr("subject", Subject), - attr("time", Time), - ), - newVersion(prefix, event.EventContextV03{}.AsV03(), - func(c event.EventContextConverter) event.EventContext { return c.AsV03() }, - attr("specversion", SpecVersion), - attr("type", Type), - attr("source", Source), - attr("schemaurl", DataSchema), - attr("subject", Subject), - attr("id", ID), - attr("time", Time), - attr("datacontenttype", DataContentType), - ), - }, - } - for _, v := range vs.all { - vs.m[v.String()] = v - } - return vs -} - -// New returns a set of versions -func New() *Versions { return WithPrefix("") } - -// Built-in un-prefixed versions. -var ( - VS *Versions - V03 Version - V1 Version -) - -func init() { - VS = New() - V03 = VS.Version("0.3") - V1 = VS.Version("1.0") -} diff --git a/vendor/github.com/cloudevents/sdk-go/v2/binding/structured_writer.go b/vendor/github.com/cloudevents/sdk-go/v2/binding/structured_writer.go deleted file mode 100644 index 8cf2bbe3..00000000 --- a/vendor/github.com/cloudevents/sdk-go/v2/binding/structured_writer.go +++ /dev/null @@ -1,17 +0,0 @@ -package binding - -import ( - "context" - "io" - - "github.com/cloudevents/sdk-go/v2/binding/format" -) - -// StructuredWriter is used to visit a structured Message and generate a new representation. -// -// Protocols that supports structured encoding should implement this interface to implement direct -// structured to structured encoding and event to structured encoding. -type StructuredWriter interface { - // Event receives an io.Reader for the whole event. - SetStructuredEvent(ctx context.Context, format format.Format, event io.Reader) error -} diff --git a/vendor/github.com/cloudevents/sdk-go/v2/binding/to_event.go b/vendor/github.com/cloudevents/sdk-go/v2/binding/to_event.go deleted file mode 100644 index d22a32e9..00000000 --- a/vendor/github.com/cloudevents/sdk-go/v2/binding/to_event.go +++ /dev/null @@ -1,126 +0,0 @@ -package binding - -import ( - "bytes" - "context" - "errors" - "fmt" - "io" - - "github.com/cloudevents/sdk-go/v2/binding/format" - "github.com/cloudevents/sdk-go/v2/binding/spec" - "github.com/cloudevents/sdk-go/v2/event" - "github.com/cloudevents/sdk-go/v2/types" -) - -// ErrCannotConvertToEvent is a generic error when a conversion of a Message to an Event fails -var ErrCannotConvertToEvent = errors.New("cannot convert message to event") - -// ToEvent translates a Message with a valid Structured or Binary representation to an Event. -// This function returns the Event generated from the Message and the original encoding of the message or -// an error that points the conversion error. -// transformers can be nil and this function guarantees that they are invoked only once during the encoding process. -func ToEvent(ctx context.Context, message MessageReader, transformers ...Transformer) (*event.Event, error) { - if message == nil { - return nil, nil - } - - messageEncoding := message.ReadEncoding() - if messageEncoding == EncodingEvent { - m := message - for m != nil { - switch mt := m.(type) { - case *EventMessage: - e := (*event.Event)(mt) - return e, Transformers(transformers).Transform(mt, (*messageToEventBuilder)(e)) - case MessageWrapper: - m = mt.GetWrappedMessage() - default: - break - } - } - return nil, ErrCannotConvertToEvent - } - - e := event.New() - encoder := (*messageToEventBuilder)(&e) - _, err := DirectWrite( - context.Background(), - message, - encoder, - encoder, - ) - if err != nil { - return nil, err - } - return &e, Transformers(transformers).Transform((*EventMessage)(&e), encoder) -} - -type messageToEventBuilder event.Event - -var _ StructuredWriter = (*messageToEventBuilder)(nil) -var _ BinaryWriter = (*messageToEventBuilder)(nil) - -func (b *messageToEventBuilder) SetStructuredEvent(ctx context.Context, format format.Format, ev io.Reader) error { - var buf bytes.Buffer - _, err := io.Copy(&buf, ev) - if err != nil { - return err - } - return format.Unmarshal(buf.Bytes(), (*event.Event)(b)) -} - -func (b *messageToEventBuilder) Start(ctx context.Context) error { - return nil -} - -func (b *messageToEventBuilder) End(ctx context.Context) error { - return nil -} - -func (b *messageToEventBuilder) SetData(data io.Reader) error { - var buf bytes.Buffer - w, err := io.Copy(&buf, data) - if err != nil { - return err - } - if w != 0 { - b.DataEncoded = buf.Bytes() - } - return nil -} - -func (b *messageToEventBuilder) SetAttribute(attribute spec.Attribute, value interface{}) error { - if value == nil { - _ = attribute.Delete(b.Context) - return nil - } - // If spec version we need to change to right context struct - if attribute.Kind() == spec.SpecVersion { - str, err := types.ToString(value) - if err != nil { - return err - } - switch str { - case event.CloudEventsVersionV03: - b.Context = b.Context.AsV03() - case event.CloudEventsVersionV1: - b.Context = b.Context.AsV1() - default: - return fmt.Errorf("unrecognized event version %s", str) - } - return nil - } - return attribute.Set(b.Context, value) -} - -func (b *messageToEventBuilder) SetExtension(name string, value interface{}) error { - if value == nil { - return b.Context.SetExtension(name, nil) - } - value, err := types.Validate(value) - if err != nil { - return err - } - return b.Context.SetExtension(name, value) -} diff --git a/vendor/github.com/cloudevents/sdk-go/v2/binding/transformer.go b/vendor/github.com/cloudevents/sdk-go/v2/binding/transformer.go deleted file mode 100644 index 6ab4f1e5..00000000 --- a/vendor/github.com/cloudevents/sdk-go/v2/binding/transformer.go +++ /dev/null @@ -1,37 +0,0 @@ -package binding - -// Transformer is an interface that implements a transformation -// process while transferring the event from the Message -// implementation to the provided encoder -// -// When a write function (binding.Write, binding.ToEvent, buffering.CopyMessage, etc.) -// takes Transformer(s) as parameter, it eventually converts the message to a form -// which correctly implements MessageMetadataReader, in order to guarantee that transformation -// is applied -type Transformer interface { - Transform(MessageMetadataReader, MessageMetadataWriter) error -} - -// TransformerFunc is a type alias to implement a Transformer through a function pointer -type TransformerFunc func(MessageMetadataReader, MessageMetadataWriter) error - -func (t TransformerFunc) Transform(r MessageMetadataReader, w MessageMetadataWriter) error { - return t(r, w) -} - -var _ Transformer = (TransformerFunc)(nil) - -// Transformers is a utility alias to run several Transformer -type Transformers []Transformer - -func (t Transformers) Transform(r MessageMetadataReader, w MessageMetadataWriter) error { - for _, transformer := range t { - err := transformer.Transform(r, w) - if err != nil { - return err - } - } - return nil -} - -var _ Transformer = (Transformers)(nil) diff --git a/vendor/github.com/cloudevents/sdk-go/v2/binding/write.go b/vendor/github.com/cloudevents/sdk-go/v2/binding/write.go deleted file mode 100644 index ff7cf5fb..00000000 --- a/vendor/github.com/cloudevents/sdk-go/v2/binding/write.go +++ /dev/null @@ -1,174 +0,0 @@ -package binding - -import ( - "context" - - "github.com/cloudevents/sdk-go/v2/event" -) - -type eventEncodingKey int - -const ( - skipDirectStructuredEncoding eventEncodingKey = iota - skipDirectBinaryEncoding - preferredEventEncoding -) - -// DirectWrite invokes the encoders. structuredWriter and binaryWriter could be nil if the protocol doesn't support it. -// transformers can be nil and this function guarantees that they are invoked only once during the encoding process. -// This function MUST be invoked only if message.ReadEncoding() == EncodingBinary or message.ReadEncoding() == EncodingStructured -// -// Returns: -// * EncodingStructured, nil if message is correctly encoded in structured encoding -// * EncodingBinary, nil if message is correctly encoded in binary encoding -// * EncodingStructured, err if message was structured but error happened during the encoding -// * EncodingBinary, err if message was binary but error happened during the encoding -// * EncodingUnknown, ErrUnknownEncoding if message is not a structured or a binary Message -func DirectWrite( - ctx context.Context, - message MessageReader, - structuredWriter StructuredWriter, - binaryWriter BinaryWriter, - transformers ...Transformer, -) (Encoding, error) { - if structuredWriter != nil && len(transformers) == 0 && !GetOrDefaultFromCtx(ctx, skipDirectStructuredEncoding, false).(bool) { - if err := message.ReadStructured(ctx, structuredWriter); err == nil { - return EncodingStructured, nil - } else if err != ErrNotStructured { - return EncodingStructured, err - } - } - - if binaryWriter != nil && !GetOrDefaultFromCtx(ctx, skipDirectBinaryEncoding, false).(bool) && message.ReadEncoding() == EncodingBinary { - return EncodingBinary, writeBinaryWithTransformer(ctx, message, binaryWriter, transformers) - } - - return EncodingUnknown, ErrUnknownEncoding -} - -// Write executes the full algorithm to encode a Message using transformers: -// 1. It first tries direct encoding using DirectWrite -// 2. If no direct encoding is possible, it uses ToEvent to generate an Event representation -// 3. From the Event, the message is encoded back to the provided structured or binary encoders -// You can tweak the encoding process using the context decorators WithForceStructured, WithForceStructured, etc. -// transformers can be nil and this function guarantees that they are invoked only once during the encoding process. -// Returns: -// * EncodingStructured, nil if message is correctly encoded in structured encoding -// * EncodingBinary, nil if message is correctly encoded in binary encoding -// * EncodingUnknown, ErrUnknownEncoding if message.ReadEncoding() == EncodingUnknown -// * _, err if error happened during the encoding -func Write( - ctx context.Context, - message MessageReader, - structuredWriter StructuredWriter, - binaryWriter BinaryWriter, - transformers ...Transformer, -) (Encoding, error) { - enc := message.ReadEncoding() - var err error - // Skip direct encoding if the event is an event message - if enc != EncodingEvent { - enc, err = DirectWrite(ctx, message, structuredWriter, binaryWriter, transformers...) - if enc != EncodingUnknown { - // Message directly encoded, nothing else to do here - return enc, err - } - } - - var e *event.Event - e, err = ToEvent(ctx, message, transformers...) - if err != nil { - return enc, err - } - - message = (*EventMessage)(e) - - if GetOrDefaultFromCtx(ctx, preferredEventEncoding, EncodingBinary).(Encoding) == EncodingStructured { - if structuredWriter != nil { - return EncodingStructured, message.ReadStructured(ctx, structuredWriter) - } - if binaryWriter != nil { - return EncodingBinary, writeBinary(ctx, message, binaryWriter) - } - } else { - if binaryWriter != nil { - return EncodingBinary, writeBinary(ctx, message, binaryWriter) - } - if structuredWriter != nil { - return EncodingStructured, message.ReadStructured(ctx, structuredWriter) - } - } - - return EncodingUnknown, ErrUnknownEncoding -} - -// WithSkipDirectStructuredEncoding skips direct structured to structured encoding during the encoding process -func WithSkipDirectStructuredEncoding(ctx context.Context, skip bool) context.Context { - return context.WithValue(ctx, skipDirectStructuredEncoding, skip) -} - -// WithSkipDirectBinaryEncoding skips direct binary to binary encoding during the encoding process -func WithSkipDirectBinaryEncoding(ctx context.Context, skip bool) context.Context { - return context.WithValue(ctx, skipDirectBinaryEncoding, skip) -} - -// WithPreferredEventEncoding defines the preferred encoding from event to message during the encoding process -func WithPreferredEventEncoding(ctx context.Context, enc Encoding) context.Context { - return context.WithValue(ctx, preferredEventEncoding, enc) -} - -// WithForceStructured forces structured encoding during the encoding process -func WithForceStructured(ctx context.Context) context.Context { - return context.WithValue(context.WithValue(ctx, preferredEventEncoding, EncodingStructured), skipDirectBinaryEncoding, true) -} - -// WithForceBinary forces binary encoding during the encoding process -func WithForceBinary(ctx context.Context) context.Context { - return context.WithValue(context.WithValue(ctx, preferredEventEncoding, EncodingBinary), skipDirectStructuredEncoding, true) -} - -// GetOrDefaultFromCtx gets a configuration value from the provided context -func GetOrDefaultFromCtx(ctx context.Context, key interface{}, def interface{}) interface{} { - if val := ctx.Value(key); val != nil { - return val - } else { - return def - } -} - -func writeBinaryWithTransformer( - ctx context.Context, - message MessageReader, - binaryWriter BinaryWriter, - transformers Transformers, -) error { - err := binaryWriter.Start(ctx) - if err != nil { - return err - } - err = message.ReadBinary(ctx, binaryWriter) - if err != nil { - return err - } - err = transformers.Transform(message.(MessageMetadataReader), binaryWriter) - if err != nil { - return err - } - return binaryWriter.End(ctx) -} - -func writeBinary( - ctx context.Context, - message MessageReader, - binaryWriter BinaryWriter, -) error { - err := binaryWriter.Start(ctx) - if err != nil { - return err - } - err = message.ReadBinary(ctx, binaryWriter) - if err != nil { - return err - } - return binaryWriter.End(ctx) -} diff --git a/vendor/github.com/cloudevents/sdk-go/v2/client/client.go b/vendor/github.com/cloudevents/sdk-go/v2/client/client.go deleted file mode 100644 index 062799fb..00000000 --- a/vendor/github.com/cloudevents/sdk-go/v2/client/client.go +++ /dev/null @@ -1,249 +0,0 @@ -package client - -import ( - "context" - "errors" - "fmt" - "io" - "runtime" - "sync" - - "go.uber.org/zap" - - "github.com/cloudevents/sdk-go/v2/binding" - cecontext "github.com/cloudevents/sdk-go/v2/context" - "github.com/cloudevents/sdk-go/v2/event" - "github.com/cloudevents/sdk-go/v2/protocol" -) - -// Client interface defines the runtime contract the CloudEvents client supports. -type Client interface { - // Send will transmit the given event over the client's configured transport. - Send(ctx context.Context, event event.Event) protocol.Result - - // Request will transmit the given event over the client's configured - // transport and return any response event. - Request(ctx context.Context, event event.Event) (*event.Event, protocol.Result) - - // StartReceiver will register the provided function for callback on receipt - // of a cloudevent. It will also start the underlying protocol as it has - // been configured. - // This call is blocking. - // Valid fn signatures are: - // * func() - // * func() error - // * func(context.Context) - // * func(context.Context) protocol.Result - // * func(event.Event) - // * func(event.Event) protocol.Result - // * func(context.Context, event.Event) - // * func(context.Context, event.Event) protocol.Result - // * func(event.Event) *event.Event - // * func(event.Event) (*event.Event, protocol.Result) - // * func(context.Context, event.Event) *event.Event - // * func(context.Context, event.Event) (*event.Event, protocol.Result) - StartReceiver(ctx context.Context, fn interface{}) error -} - -// New produces a new client with the provided transport object and applied -// client options. -func New(obj interface{}, opts ...Option) (Client, error) { - c := &ceClient{ - // Running runtime.GOMAXPROCS(0) doesn't update the value, just returns the current one - pollGoroutines: runtime.GOMAXPROCS(0), - } - - if p, ok := obj.(protocol.Sender); ok { - c.sender = p - } - if p, ok := obj.(protocol.Requester); ok { - c.requester = p - } - if p, ok := obj.(protocol.Responder); ok { - c.responder = p - } - if p, ok := obj.(protocol.Receiver); ok { - c.receiver = p - } - if p, ok := obj.(protocol.Opener); ok { - c.opener = p - } - - if err := c.applyOptions(opts...); err != nil { - return nil, err - } - return c, nil -} - -type ceClient struct { - sender protocol.Sender - requester protocol.Requester - receiver protocol.Receiver - responder protocol.Responder - // Optional. - opener protocol.Opener - - outboundContextDecorators []func(context.Context) context.Context - invoker Invoker - receiverMu sync.Mutex - eventDefaulterFns []EventDefaulter - pollGoroutines int -} - -func (c *ceClient) applyOptions(opts ...Option) error { - for _, fn := range opts { - if err := fn(c); err != nil { - return err - } - } - return nil -} - -func (c *ceClient) Send(ctx context.Context, e event.Event) protocol.Result { - if c.sender == nil { - return errors.New("sender not set") - } - - for _, f := range c.outboundContextDecorators { - ctx = f(ctx) - } - - if len(c.eventDefaulterFns) > 0 { - for _, fn := range c.eventDefaulterFns { - e = fn(ctx, e) - } - } - - if err := e.Validate(); err != nil { - return err - } - - return c.sender.Send(ctx, (*binding.EventMessage)(&e)) -} - -func (c *ceClient) Request(ctx context.Context, e event.Event) (*event.Event, protocol.Result) { - if c.requester == nil { - return nil, errors.New("requester not set") - } - for _, f := range c.outboundContextDecorators { - ctx = f(ctx) - } - - if len(c.eventDefaulterFns) > 0 { - for _, fn := range c.eventDefaulterFns { - e = fn(ctx, e) - } - } - - if err := e.Validate(); err != nil { - return nil, err - } - - // If provided a requester, use it to do request/response. - var resp *event.Event - msg, err := c.requester.Request(ctx, (*binding.EventMessage)(&e)) - if msg != nil { - defer func() { - if err := msg.Finish(err); err != nil { - cecontext.LoggerFrom(ctx).Warnw("failed calling message.Finish", zap.Error(err)) - } - }() - } - if protocol.IsUndelivered(err) { - return nil, err - } - - // try to turn msg into an event, it might not work and that is ok. - if rs, rserr := binding.ToEvent(ctx, msg); rserr != nil { - cecontext.LoggerFrom(ctx).Debugw("response: failed calling ToEvent", zap.Error(rserr), zap.Any("resp", msg)) - // If the protocol returns no error, it is an ACK on the request, but we had - // issues turning the response into an event, so make an ACK Result and pass - // down the ToEvent error as well. - err = protocol.NewReceipt(true, "failed to convert response into event: %s\n%w", rserr.Error(), err) - } else { - resp = rs - } - - return resp, err -} - -// StartReceiver sets up the given fn to handle Receive. -// See Client.StartReceiver for details. This is a blocking call. -func (c *ceClient) StartReceiver(ctx context.Context, fn interface{}) error { - c.receiverMu.Lock() - defer c.receiverMu.Unlock() - - if c.invoker != nil { - return fmt.Errorf("client already has a receiver") - } - - invoker, err := newReceiveInvoker(fn, c.eventDefaulterFns...) // TODO: this will have to pick between a observed invoker or not. - if err != nil { - return err - } - if invoker.IsReceiver() && c.receiver == nil { - return fmt.Errorf("mismatched receiver callback without protocol.Receiver supported by protocol") - } - if invoker.IsResponder() && c.responder == nil { - return fmt.Errorf("mismatched receiver callback without protocol.Responder supported by protocol") - } - c.invoker = invoker - - if c.responder == nil && c.receiver == nil { - return errors.New("responder nor receiver set") - } - - defer func() { - c.invoker = nil - }() - - // Start the opener, if set. - if c.opener != nil { - go func() { - if err := c.opener.OpenInbound(ctx); err != nil { - cecontext.LoggerFrom(ctx).Errorf("Error while opening the inbound connection: %s", err) - } - }() - } - - // Start Polling. - wg := sync.WaitGroup{} - for i := 0; i < c.pollGoroutines; i++ { - wg.Add(1) - go func() { - defer wg.Done() - for { - var msg binding.Message - var respFn protocol.ResponseFn - var err error - - if c.responder != nil { - msg, respFn, err = c.responder.Respond(ctx) - } else if c.receiver != nil { - msg, err = c.receiver.Receive(ctx) - respFn = noRespFn - } - - if err == io.EOF { // Normal close - return - } - - if err != nil { - cecontext.LoggerFrom(ctx).Warnf("Error while receiving a message: %s", err) - continue - } - - if err := c.invoker.Invoke(ctx, msg, respFn); err != nil { - cecontext.LoggerFrom(ctx).Warnf("Error while handling a message: %s", err) - } - } - }() - } - wg.Wait() - return nil -} - -// noRespFn is used to simply forward the protocol.Result for receivers that aren't responders -func noRespFn(_ context.Context, _ binding.Message, r protocol.Result, _ ...binding.Transformer) error { - return r -} diff --git a/vendor/github.com/cloudevents/sdk-go/v2/client/client_default.go b/vendor/github.com/cloudevents/sdk-go/v2/client/client_default.go deleted file mode 100644 index 82877d67..00000000 --- a/vendor/github.com/cloudevents/sdk-go/v2/client/client_default.go +++ /dev/null @@ -1,26 +0,0 @@ -package client - -import ( - "github.com/cloudevents/sdk-go/v2/protocol/http" -) - -// NewDefault provides the good defaults for the common case using an HTTP -// Protocol client. The http transport has had WithBinaryEncoding http -// transport option applied to it. The client will always send Binary -// encoding but will inspect the outbound event context and match the version. -// The WithTimeNow, and WithUUIDs client options are also applied to the -// client, all outbound events will have a time and id set if not already -// present. -func NewDefault() (Client, error) { - p, err := http.New() - if err != nil { - return nil, err - } - - c, err := NewObserved(p, WithTimeNow(), WithUUIDs()) - if err != nil { - return nil, err - } - - return c, nil -} diff --git a/vendor/github.com/cloudevents/sdk-go/v2/client/client_observed.go b/vendor/github.com/cloudevents/sdk-go/v2/client/client_observed.go deleted file mode 100644 index 5feb4f8d..00000000 --- a/vendor/github.com/cloudevents/sdk-go/v2/client/client_observed.go +++ /dev/null @@ -1,101 +0,0 @@ -package client - -import ( - "context" - "github.com/cloudevents/sdk-go/v2/event" - "github.com/cloudevents/sdk-go/v2/extensions" - "github.com/cloudevents/sdk-go/v2/observability" - "github.com/cloudevents/sdk-go/v2/protocol" - "go.opencensus.io/trace" -) - -// NewObserved produces a new client with the provided transport object and applied -// client options. -func NewObserved(protocol interface{}, opts ...Option) (Client, error) { - client, err := New(protocol, opts...) - if err != nil { - return nil, err - } - - c := &obsClient{client: client} - - if err := c.applyOptions(opts...); err != nil { - return nil, err - } - return c, nil -} - -type obsClient struct { - client Client - - addTracing bool -} - -func (c *obsClient) applyOptions(opts ...Option) error { - for _, fn := range opts { - if err := fn(c); err != nil { - return err - } - } - return nil -} - -// Send transmits the provided event on a preconfigured Protocol. Send returns -// an error if there was an an issue validating the outbound event or the -// transport returns an error. -func (c *obsClient) Send(ctx context.Context, e event.Event) protocol.Result { - ctx, r := observability.NewReporter(ctx, reportSend) - ctx, span := trace.StartSpan(ctx, observability.ClientSpanName, trace.WithSpanKind(trace.SpanKindClient)) - defer span.End() - if span.IsRecordingEvents() { - span.AddAttributes(EventTraceAttributes(&e)...) - } - - if c.addTracing { - e.Context = e.Context.Clone() - extensions.FromSpanContext(span.SpanContext()).AddTracingAttributes(&e) - } - - result := c.client.Send(ctx, e) - - if protocol.IsACK(result) { - r.OK() - } else { - r.Error() - } - return result -} - -func (c *obsClient) Request(ctx context.Context, e event.Event) (*event.Event, protocol.Result) { - ctx, r := observability.NewReporter(ctx, reportRequest) - ctx, span := trace.StartSpan(ctx, observability.ClientSpanName, trace.WithSpanKind(trace.SpanKindClient)) - defer span.End() - if span.IsRecordingEvents() { - span.AddAttributes(EventTraceAttributes(&e)...) - } - - resp, result := c.client.Request(ctx, e) - - if protocol.IsACK(result) { - r.OK() - } else { - r.Error() - } - - return resp, result -} - -// StartReceiver sets up the given fn to handle Receive. -// See Client.StartReceiver for details. This is a blocking call. -func (c *obsClient) StartReceiver(ctx context.Context, fn interface{}) error { - ctx, r := observability.NewReporter(ctx, reportStartReceiver) - - err := c.client.StartReceiver(ctx, fn) - - if err != nil { - r.Error() - } else { - r.OK() - } - return err -} diff --git a/vendor/github.com/cloudevents/sdk-go/v2/client/defaulters.go b/vendor/github.com/cloudevents/sdk-go/v2/client/defaulters.go deleted file mode 100644 index 5d0d7bc9..00000000 --- a/vendor/github.com/cloudevents/sdk-go/v2/client/defaulters.go +++ /dev/null @@ -1,52 +0,0 @@ -package client - -import ( - "context" - "time" - - "github.com/cloudevents/sdk-go/v2/event" - - "github.com/google/uuid" -) - -// EventDefaulter is the function signature for extensions that are able -// to perform event defaulting. -type EventDefaulter func(ctx context.Context, event event.Event) event.Event - -// DefaultIDToUUIDIfNotSet will inspect the provided event and assign a UUID to -// context.ID if it is found to be empty. -func DefaultIDToUUIDIfNotSet(ctx context.Context, event event.Event) event.Event { - if event.Context != nil { - if event.ID() == "" { - event.Context = event.Context.Clone() - event.SetID(uuid.New().String()) - } - } - return event -} - -// DefaultTimeToNowIfNotSet will inspect the provided event and assign a new -// Timestamp to context.Time if it is found to be nil or zero. -func DefaultTimeToNowIfNotSet(ctx context.Context, event event.Event) event.Event { - if event.Context != nil { - if event.Time().IsZero() { - event.Context = event.Context.Clone() - event.SetTime(time.Now()) - } - } - return event -} - -// NewDefaultDataContentTypeIfNotSet returns a defaulter that will inspect the -// provided event and set the provided content type if content type is found -// to be empty. -func NewDefaultDataContentTypeIfNotSet(contentType string) EventDefaulter { - return func(ctx context.Context, event event.Event) event.Event { - if event.Context != nil { - if event.DataContentType() == "" { - event.SetDataContentType(contentType) - } - } - return event - } -} diff --git a/vendor/github.com/cloudevents/sdk-go/v2/client/doc.go b/vendor/github.com/cloudevents/sdk-go/v2/client/doc.go deleted file mode 100644 index a6a602bb..00000000 --- a/vendor/github.com/cloudevents/sdk-go/v2/client/doc.go +++ /dev/null @@ -1,6 +0,0 @@ -/* -Package client holds the recommended entry points for interacting with the CloudEvents Golang SDK. The client wraps -a selected transport. The client adds validation and defaulting for sending events, and flexible receiver method -registration. For full details, read the `client.Client` documentation. -*/ -package client diff --git a/vendor/github.com/cloudevents/sdk-go/v2/client/http_receiver.go b/vendor/github.com/cloudevents/sdk-go/v2/client/http_receiver.go deleted file mode 100644 index 467cff9e..00000000 --- a/vendor/github.com/cloudevents/sdk-go/v2/client/http_receiver.go +++ /dev/null @@ -1,45 +0,0 @@ -package client - -import ( - "context" - "net/http" - "sync" - - thttp "github.com/cloudevents/sdk-go/v2/protocol/http" -) - -func NewHTTPReceiveHandler(ctx context.Context, p *thttp.Protocol, fn interface{}) (*EventReceiver, error) { - invoker, err := newReceiveInvoker(fn) - if err != nil { - return nil, err - } - - return &EventReceiver{ - p: p, - invoker: invoker, - }, nil -} - -type EventReceiver struct { - p *thttp.Protocol - invoker Invoker -} - -func (r *EventReceiver) ServeHTTP(rw http.ResponseWriter, req *http.Request) { - wg := sync.WaitGroup{} - wg.Add(1) - go func() { - r.p.ServeHTTP(rw, req) - wg.Done() - }() - - ctx := req.Context() - msg, respFn, err := r.p.Respond(ctx) - if err != nil { - //lint:ignore SA9003 TODO: Branch left empty - } else if err := r.invoker.Invoke(ctx, msg, respFn); err != nil { - // TODO - } - // Block until ServeHTTP has returned - wg.Wait() -} diff --git a/vendor/github.com/cloudevents/sdk-go/v2/client/invoker.go b/vendor/github.com/cloudevents/sdk-go/v2/client/invoker.go deleted file mode 100644 index 162ae27e..00000000 --- a/vendor/github.com/cloudevents/sdk-go/v2/client/invoker.go +++ /dev/null @@ -1,99 +0,0 @@ -package client - -import ( - "context" - - "github.com/cloudevents/sdk-go/v2/binding" - cecontext "github.com/cloudevents/sdk-go/v2/context" - "github.com/cloudevents/sdk-go/v2/event" - "github.com/cloudevents/sdk-go/v2/protocol" -) - -type Invoker interface { - Invoke(context.Context, binding.Message, protocol.ResponseFn) error - IsReceiver() bool - IsResponder() bool -} - -var _ Invoker = (*receiveInvoker)(nil) - -func newReceiveInvoker(fn interface{}, fns ...EventDefaulter) (Invoker, error) { - r := &receiveInvoker{ - eventDefaulterFns: fns, - } - - if fn, err := receiver(fn); err != nil { - return nil, err - } else { - r.fn = fn - } - - return r, nil -} - -type receiveInvoker struct { - fn *receiverFn - eventDefaulterFns []EventDefaulter -} - -func (r *receiveInvoker) Invoke(ctx context.Context, m binding.Message, respFn protocol.ResponseFn) (err error) { - defer func() { - err = m.Finish(err) - }() - - var respMsg binding.Message - var result protocol.Result - - e, eventErr := binding.ToEvent(ctx, m) - switch { - case eventErr != nil && r.fn.hasEventIn: - return respFn(ctx, nil, protocol.NewReceipt(false, "failed to convert Message to Event: %w", eventErr)) - case r.fn != nil: - // Check if event is valid before invoking the receiver function - if e != nil { - if validationErr := e.Validate(); validationErr != nil { - return respFn(ctx, nil, protocol.NewReceipt(false, "validation error in incoming event: %w", validationErr)) - } - } - - // Let's invoke the receiver fn - var resp *event.Event - resp, result = r.fn.invoke(ctx, e) - - if respFn == nil { - break - } - - // Apply the defaulter chain to the outgoing event. - if resp != nil && len(r.eventDefaulterFns) > 0 { - for _, fn := range r.eventDefaulterFns { - *resp = fn(ctx, *resp) - } - // Validate the event conforms to the CloudEvents Spec. - if vErr := resp.Validate(); vErr != nil { - cecontext.LoggerFrom(ctx).Errorf("cloudevent validation failed on response event: %w", vErr) - } - } - - // because binding.Message is an interface, casting a nil resp - // here would make future comparisons to nil false - if resp != nil { - respMsg = (*binding.EventMessage)(resp) - } - } - - if respFn == nil { - // let the protocol ACK based on the result - return result - } - - return respFn(ctx, respMsg, result) -} - -func (r *receiveInvoker) IsReceiver() bool { - return !r.fn.hasEventOut -} - -func (r *receiveInvoker) IsResponder() bool { - return r.fn.hasEventOut -} diff --git a/vendor/github.com/cloudevents/sdk-go/v2/client/observability.go b/vendor/github.com/cloudevents/sdk-go/v2/client/observability.go deleted file mode 100644 index 4c190595..00000000 --- a/vendor/github.com/cloudevents/sdk-go/v2/client/observability.go +++ /dev/null @@ -1,96 +0,0 @@ -package client - -import ( - "context" - "github.com/cloudevents/sdk-go/v2/event" - "github.com/cloudevents/sdk-go/v2/extensions" - "github.com/cloudevents/sdk-go/v2/observability" - "go.opencensus.io/stats" - "go.opencensus.io/stats/view" - "go.opencensus.io/trace" -) - -var ( - // LatencyMs measures the latency in milliseconds for the CloudEvents - // client methods. - LatencyMs = stats.Float64("cloudevents.io/sdk-go/client/latency", "The latency in milliseconds for the CloudEvents client methods.", "ms") -) - -var ( - // LatencyView is an OpenCensus view that shows client method latency. - LatencyView = &view.View{ - Name: "client/latency", - Measure: LatencyMs, - Description: "The distribution of latency inside of client for CloudEvents.", - Aggregation: view.Distribution(0, .01, .1, 1, 10, 100, 1000, 10000), - TagKeys: observability.LatencyTags(), - } -) - -type observed int32 - -// Adheres to Observable -var _ observability.Observable = observed(0) - -const ( - specversionAttr = "cloudevents.specversion" - idAttr = "cloudevents.id" - typeAttr = "cloudevents.type" - sourceAttr = "cloudevents.source" - subjectAttr = "cloudevents.subject" - datacontenttypeAttr = "cloudevents.datacontenttype" - - reportSend observed = iota - reportRequest - reportStartReceiver -) - -// MethodName implements Observable.MethodName -func (o observed) MethodName() string { - switch o { - case reportSend: - return "send" - case reportRequest: - return "request" - case reportStartReceiver: - return "start_receiver" - default: - return "unknown" - } -} - -// LatencyMs implements Observable.LatencyMs -func (o observed) LatencyMs() *stats.Float64Measure { - return LatencyMs -} - -func EventTraceAttributes(e event.EventReader) []trace.Attribute { - as := []trace.Attribute{ - trace.StringAttribute(specversionAttr, e.SpecVersion()), - trace.StringAttribute(idAttr, e.ID()), - trace.StringAttribute(typeAttr, e.Type()), - trace.StringAttribute(sourceAttr, e.Source()), - } - if sub := e.Subject(); sub != "" { - as = append(as, trace.StringAttribute(subjectAttr, sub)) - } - if dct := e.DataContentType(); dct != "" { - as = append(as, trace.StringAttribute(datacontenttypeAttr, dct)) - } - return as -} - -// TraceSpan returns context and trace.Span based on event. Caller must call span.End() -func TraceSpan(ctx context.Context, e event.Event) (context.Context, *trace.Span) { - var span *trace.Span - if ext, ok := extensions.GetDistributedTracingExtension(e); ok { - ctx, span = ext.StartChildSpan(ctx, observability.ClientSpanName, trace.WithSpanKind(trace.SpanKindServer)) - } - if span == nil { - ctx, span = trace.StartSpan(ctx, observability.ClientSpanName, trace.WithSpanKind(trace.SpanKindServer)) - } - if span.IsRecordingEvents() { - span.AddAttributes(EventTraceAttributes(&e)...) - } - return ctx, span -} diff --git a/vendor/github.com/cloudevents/sdk-go/v2/client/options.go b/vendor/github.com/cloudevents/sdk-go/v2/client/options.go deleted file mode 100644 index aeec1eb2..00000000 --- a/vendor/github.com/cloudevents/sdk-go/v2/client/options.go +++ /dev/null @@ -1,85 +0,0 @@ -package client - -import ( - "fmt" - "github.com/cloudevents/sdk-go/v2/binding" -) - -// Option is the function signature required to be considered an client.Option. -type Option func(interface{}) error - -// WithEventDefaulter adds an event defaulter to the end of the defaulter chain. -func WithEventDefaulter(fn EventDefaulter) Option { - return func(i interface{}) error { - if c, ok := i.(*ceClient); ok { - if fn == nil { - return fmt.Errorf("client option was given an nil event defaulter") - } - c.eventDefaulterFns = append(c.eventDefaulterFns, fn) - } - return nil - } -} - -func WithForceBinary() Option { - return func(i interface{}) error { - if c, ok := i.(*ceClient); ok { - c.outboundContextDecorators = append(c.outboundContextDecorators, binding.WithForceBinary) - } - return nil - } -} - -func WithForceStructured() Option { - return func(i interface{}) error { - if c, ok := i.(*ceClient); ok { - c.outboundContextDecorators = append(c.outboundContextDecorators, binding.WithForceStructured) - } - return nil - } -} - -// WithUUIDs adds DefaultIDToUUIDIfNotSet event defaulter to the end of the -// defaulter chain. -func WithUUIDs() Option { - return func(i interface{}) error { - if c, ok := i.(*ceClient); ok { - c.eventDefaulterFns = append(c.eventDefaulterFns, DefaultIDToUUIDIfNotSet) - } - return nil - } -} - -// WithTimeNow adds DefaultTimeToNowIfNotSet event defaulter to the end of the -// defaulter chain. -func WithTimeNow() Option { - return func(i interface{}) error { - if c, ok := i.(*ceClient); ok { - c.eventDefaulterFns = append(c.eventDefaulterFns, DefaultTimeToNowIfNotSet) - } - return nil - } -} - -// WithTracePropagation enables trace propagation via the distributed tracing -// extension. -func WithTracePropagation() Option { - return func(i interface{}) error { - if c, ok := i.(*obsClient); ok { - c.addTracing = true - } - return nil - } -} - -// WithPollGoroutines configures how much goroutines should be used to -// poll the Receiver/Responder/Protocol implementations. -// Default value is GOMAXPROCS -func WithPollGoroutines(pollGoroutines int) Option { - return func(i interface{}) error { - if c, ok := i.(*ceClient); ok { - c.pollGoroutines = pollGoroutines - } - return nil - } -} diff --git a/vendor/github.com/cloudevents/sdk-go/v2/client/receiver.go b/vendor/github.com/cloudevents/sdk-go/v2/client/receiver.go deleted file mode 100644 index e1d1544c..00000000 --- a/vendor/github.com/cloudevents/sdk-go/v2/client/receiver.go +++ /dev/null @@ -1,189 +0,0 @@ -package client - -import ( - "context" - "errors" - "fmt" - "reflect" - - "github.com/cloudevents/sdk-go/v2/event" - "github.com/cloudevents/sdk-go/v2/protocol" -) - -// ReceiveFull is the signature of a fn to be invoked for incoming cloudevents. -type ReceiveFull func(context.Context, event.Event) protocol.Result - -type receiverFn struct { - numIn int - numOut int - fnValue reflect.Value - - hasContextIn bool - hasEventIn bool - - hasEventOut bool - hasResultOut bool -} - -const ( - inParamUsage = "expected a function taking either no parameters, one or more of (context.Context, event.Event) ordered" - outParamUsage = "expected a function returning one or mode of (*event.Event, protocol.Result) ordered" -) - -var ( - contextType = reflect.TypeOf((*context.Context)(nil)).Elem() - eventType = reflect.TypeOf((*event.Event)(nil)).Elem() - eventPtrType = reflect.TypeOf((*event.Event)(nil)) // want the ptr type - resultType = reflect.TypeOf((*protocol.Result)(nil)).Elem() -) - -// receiver creates a receiverFn wrapper class that is used by the client to -// validate and invoke the provided function. -// Valid fn signatures are: -// * func() -// * func() error -// * func(context.Context) -// * func(context.Context) transport.Result -// * func(event.Event) -// * func(event.Event) transport.Result -// * func(context.Context, event.Event) -// * func(context.Context, event.Event) transport.Result -// * func(event.Event) *event.Event -// * func(event.Event) (*event.Event, transport.Result) -// * func(context.Context, event.Event, *event.Event -// * func(context.Context, event.Event) (*event.Event, transport.Result) -// -func receiver(fn interface{}) (*receiverFn, error) { - fnType := reflect.TypeOf(fn) - if fnType.Kind() != reflect.Func { - return nil, errors.New("must pass a function to handle events") - } - - r := &receiverFn{ - fnValue: reflect.ValueOf(fn), - numIn: fnType.NumIn(), - numOut: fnType.NumOut(), - } - - if err := r.validate(fnType); err != nil { - return nil, err - } - - return r, nil -} - -func (r *receiverFn) invoke(ctx context.Context, e *event.Event) (*event.Event, protocol.Result) { - args := make([]reflect.Value, 0, r.numIn) - - if r.numIn > 0 { - if r.hasContextIn { - args = append(args, reflect.ValueOf(ctx)) - } - if r.hasEventIn { - args = append(args, reflect.ValueOf(*e)) - } - } - v := r.fnValue.Call(args) - var respOut protocol.Result - var eOut *event.Event - if r.numOut > 0 { - i := 0 - if r.hasEventOut { - if eo, ok := v[i].Interface().(*event.Event); ok { - eOut = eo - } - i++ // <-- note, need to inc i. - } - if r.hasResultOut { - if resp, ok := v[i].Interface().(protocol.Result); ok { - respOut = resp - } - } - } - return eOut, respOut -} - -// Verifies that the inputs to a function have a valid signature -// Valid input is to be [0, all] of -// context.Context, event.Event in this order. -func (r *receiverFn) validateInParamSignature(fnType reflect.Type) error { - r.hasContextIn = false - r.hasEventIn = false - - switch fnType.NumIn() { - case 2: - // has to be (context.Context, event.Event) - if !fnType.In(1).ConvertibleTo(eventType) { - return fmt.Errorf("%s; cannot convert parameter 2 from %s to event.Event", inParamUsage, fnType.In(1)) - } else { - r.hasEventIn = true - } - fallthrough - case 1: - if !fnType.In(0).ConvertibleTo(contextType) { - if !fnType.In(0).ConvertibleTo(eventType) { - return fmt.Errorf("%s; cannot convert parameter 1 from %s to context.Context or event.Event", inParamUsage, fnType.In(0)) - } else if r.hasEventIn { - return fmt.Errorf("%s; duplicate parameter of type event.Event", inParamUsage) - } else { - r.hasEventIn = true - } - } else { - r.hasContextIn = true - } - fallthrough - case 0: - return nil - - default: - return fmt.Errorf("%s; function has too many parameters (%d)", inParamUsage, fnType.NumIn()) - } -} - -// Verifies that the outputs of a function have a valid signature -// Valid output signatures to be [0, all] of -// *event.Event, transport.Result in this order -func (r *receiverFn) validateOutParamSignature(fnType reflect.Type) error { - r.hasEventOut = false - r.hasResultOut = false - - switch fnType.NumOut() { - case 2: - // has to be (*event.Event, transport.Result) - if !fnType.Out(1).ConvertibleTo(resultType) { - return fmt.Errorf("%s; cannot convert parameter 2 from %s to event.Response", outParamUsage, fnType.Out(1)) - } else { - r.hasResultOut = true - } - fallthrough - case 1: - if !fnType.Out(0).ConvertibleTo(resultType) { - if !fnType.Out(0).ConvertibleTo(eventPtrType) { - return fmt.Errorf("%s; cannot convert parameter 1 from %s to *event.Event or transport.Result", outParamUsage, fnType.Out(0)) - } else { - r.hasEventOut = true - } - } else if r.hasResultOut { - return fmt.Errorf("%s; duplicate parameter of type event.Response", outParamUsage) - } else { - r.hasResultOut = true - } - fallthrough - case 0: - return nil - default: - return fmt.Errorf("%s; function has too many return types (%d)", outParamUsage, fnType.NumOut()) - } -} - -// validateReceiverFn validates that a function has the right number of in and -// out params and that they are of allowed types. -func (r *receiverFn) validate(fnType reflect.Type) error { - if err := r.validateInParamSignature(fnType); err != nil { - return err - } - if err := r.validateOutParamSignature(fnType); err != nil { - return err - } - return nil -} diff --git a/vendor/github.com/cloudevents/sdk-go/v2/context/context.go b/vendor/github.com/cloudevents/sdk-go/v2/context/context.go deleted file mode 100644 index f9843dd6..00000000 --- a/vendor/github.com/cloudevents/sdk-go/v2/context/context.go +++ /dev/null @@ -1,105 +0,0 @@ -package context - -import ( - "context" - "net/url" - "time" -) - -// Opaque key type used to store target -type targetKeyType struct{} - -var targetKey = targetKeyType{} - -// WithTarget returns back a new context with the given target. Target is intended to be transport dependent. -// For http transport, `target` should be a full URL and will be injected into the outbound http request. -func WithTarget(ctx context.Context, target string) context.Context { - return context.WithValue(ctx, targetKey, target) -} - -// TargetFrom looks in the given context and returns `target` as a parsed url if found and valid, otherwise nil. -func TargetFrom(ctx context.Context) *url.URL { - c := ctx.Value(targetKey) - if c != nil { - if s, ok := c.(string); ok && s != "" { - if target, err := url.Parse(s); err == nil { - return target - } - } - } - return nil -} - -// Opaque key type used to store topic -type topicKeyType struct{} - -var topicKey = topicKeyType{} - -// WithTopic returns back a new context with the given topic. Topic is intended to be transport dependent. -// For pubsub transport, `topic` should be a Pub/Sub Topic ID. -func WithTopic(ctx context.Context, topic string) context.Context { - return context.WithValue(ctx, topicKey, topic) -} - -// TopicFrom looks in the given context and returns `topic` as a string if found and valid, otherwise "". -func TopicFrom(ctx context.Context) string { - c := ctx.Value(topicKey) - if c != nil { - if s, ok := c.(string); ok { - return s - } - } - return "" -} - -// Opaque key type used to store retry parameters -type retriesKeyType struct{} - -var retriesKey = retriesKeyType{} - -// WithRetriesConstantBackoff returns back a new context with retries parameters using constant backoff strategy. -// MaxTries is the maximum number for retries and delay is the time interval between retries -func WithRetriesConstantBackoff(ctx context.Context, delay time.Duration, maxTries int) context.Context { - return WithRetryParams(ctx, &RetryParams{ - Strategy: BackoffStrategyConstant, - Period: delay, - MaxTries: maxTries, - }) -} - -// WithRetriesLinearBackoff returns back a new context with retries parameters using linear backoff strategy. -// MaxTries is the maximum number for retries and delay*tries is the time interval between retries -func WithRetriesLinearBackoff(ctx context.Context, delay time.Duration, maxTries int) context.Context { - return WithRetryParams(ctx, &RetryParams{ - Strategy: BackoffStrategyLinear, - Period: delay, - MaxTries: maxTries, - }) -} - -// WithRetriesExponentialBackoff returns back a new context with retries parameters using exponential backoff strategy. -// MaxTries is the maximum number for retries and period is the amount of time to wait, used as `period * 2^retries`. -func WithRetriesExponentialBackoff(ctx context.Context, period time.Duration, maxTries int) context.Context { - return WithRetryParams(ctx, &RetryParams{ - Strategy: BackoffStrategyExponential, - Period: period, - MaxTries: maxTries, - }) -} - -// WithRetryParams returns back a new context with retries parameters. -func WithRetryParams(ctx context.Context, rp *RetryParams) context.Context { - return context.WithValue(ctx, retriesKey, rp) -} - -// RetriesFrom looks in the given context and returns the retries parameters if found. -// Otherwise returns the default retries configuration (ie. no retries). -func RetriesFrom(ctx context.Context) *RetryParams { - c := ctx.Value(retriesKey) - if c != nil { - if s, ok := c.(*RetryParams); ok { - return s - } - } - return &DefaultRetryParams -} diff --git a/vendor/github.com/cloudevents/sdk-go/v2/context/doc.go b/vendor/github.com/cloudevents/sdk-go/v2/context/doc.go deleted file mode 100644 index 377cab85..00000000 --- a/vendor/github.com/cloudevents/sdk-go/v2/context/doc.go +++ /dev/null @@ -1,5 +0,0 @@ -/* -Package context holds the last resort overrides and fyi objects that can be passed to clients and transports added to -context.Context objects. -*/ -package context diff --git a/vendor/github.com/cloudevents/sdk-go/v2/context/logger.go b/vendor/github.com/cloudevents/sdk-go/v2/context/logger.go deleted file mode 100644 index 996f7205..00000000 --- a/vendor/github.com/cloudevents/sdk-go/v2/context/logger.go +++ /dev/null @@ -1,43 +0,0 @@ -package context - -import ( - "context" - - "go.uber.org/zap" -) - -// Opaque key type used to store logger -type loggerKeyType struct{} - -var loggerKey = loggerKeyType{} - -// fallbackLogger is the logger is used when there is no logger attached to the context. -var fallbackLogger *zap.SugaredLogger - -func init() { - if logger, err := zap.NewProduction(); err != nil { - // We failed to create a fallback logger. - fallbackLogger = zap.NewNop().Sugar() - } else { - fallbackLogger = logger.Named("fallback").Sugar() - } -} - -// WithLogger returns a new context with the logger injected into the given context. -func WithLogger(ctx context.Context, logger *zap.SugaredLogger) context.Context { - if logger == nil { - return context.WithValue(ctx, loggerKey, fallbackLogger) - } - return context.WithValue(ctx, loggerKey, logger) -} - -// LoggerFrom returns the logger stored in context. -func LoggerFrom(ctx context.Context) *zap.SugaredLogger { - l := ctx.Value(loggerKey) - if l != nil { - if logger, ok := l.(*zap.SugaredLogger); ok { - return logger - } - } - return fallbackLogger -} diff --git a/vendor/github.com/cloudevents/sdk-go/v2/context/retry.go b/vendor/github.com/cloudevents/sdk-go/v2/context/retry.go deleted file mode 100644 index f590d466..00000000 --- a/vendor/github.com/cloudevents/sdk-go/v2/context/retry.go +++ /dev/null @@ -1,71 +0,0 @@ -package context - -import ( - "context" - "errors" - "math" - "time" -) - -type BackoffStrategy string - -const ( - BackoffStrategyNone = "none" - BackoffStrategyConstant = "constant" - BackoffStrategyLinear = "linear" - BackoffStrategyExponential = "exponential" -) - -var DefaultRetryParams = RetryParams{Strategy: BackoffStrategyNone} - -// RetryParams holds parameters applied to retries -type RetryParams struct { - // Strategy is the backoff strategy to applies between retries - Strategy BackoffStrategy - - // MaxTries is the maximum number of times to retry request before giving up - MaxTries int - - // Period is - // - for none strategy: no delay - // - for constant strategy: the delay interval between retries - // - for linear strategy: interval between retries = Period * retries - // - for exponential strategy: interval between retries = Period * retries^2 - Period time.Duration -} - -// BackoffFor tries will return the time duration that should be used for this -// current try count. -// `tries` is assumed to be the number of times the caller has already retried. -func (r *RetryParams) BackoffFor(tries int) time.Duration { - switch r.Strategy { - case BackoffStrategyConstant: - return r.Period - case BackoffStrategyLinear: - return r.Period * time.Duration(tries) - case BackoffStrategyExponential: - exp := math.Exp2(float64(tries)) - return r.Period * time.Duration(exp) - case BackoffStrategyNone: - fallthrough // default - default: - return r.Period - } -} - -// Backoff is a blocking call to wait for the correct amount of time for the retry. -// `tries` is assumed to be the number of times the caller has already retried. -func (r *RetryParams) Backoff(ctx context.Context, tries int) error { - if tries > r.MaxTries { - return errors.New("too many retries") - } - ticker := time.NewTicker(r.BackoffFor(tries)) - select { - case <-ctx.Done(): - ticker.Stop() - return errors.New("context has been cancelled") - case <-ticker.C: - ticker.Stop() - } - return nil -} diff --git a/vendor/github.com/cloudevents/sdk-go/v2/event/content_type.go b/vendor/github.com/cloudevents/sdk-go/v2/event/content_type.go deleted file mode 100644 index 591878e5..00000000 --- a/vendor/github.com/cloudevents/sdk-go/v2/event/content_type.go +++ /dev/null @@ -1,42 +0,0 @@ -package event - -const ( - TextPlain = "text/plain" - TextJSON = "text/json" - ApplicationJSON = "application/json" - ApplicationXML = "application/xml" - ApplicationCloudEventsJSON = "application/cloudevents+json" - ApplicationCloudEventsBatchJSON = "application/cloudevents-batch+json" -) - -// StringOfApplicationJSON returns a string pointer to "application/json" -func StringOfApplicationJSON() *string { - a := ApplicationJSON - return &a -} - -// StringOfApplicationXML returns a string pointer to "application/xml" -func StringOfApplicationXML() *string { - a := ApplicationXML - return &a -} - -// StringOfTextPlain returns a string pointer to "text/plain" -func StringOfTextPlain() *string { - a := TextPlain - return &a -} - -// StringOfApplicationCloudEventsJSON returns a string pointer to -// "application/cloudevents+json" -func StringOfApplicationCloudEventsJSON() *string { - a := ApplicationCloudEventsJSON - return &a -} - -// StringOfApplicationCloudEventsBatchJSON returns a string pointer to -// "application/cloudevents-batch+json" -func StringOfApplicationCloudEventsBatchJSON() *string { - a := ApplicationCloudEventsBatchJSON - return &a -} diff --git a/vendor/github.com/cloudevents/sdk-go/v2/event/data_content_encoding.go b/vendor/github.com/cloudevents/sdk-go/v2/event/data_content_encoding.go deleted file mode 100644 index 24c4094f..00000000 --- a/vendor/github.com/cloudevents/sdk-go/v2/event/data_content_encoding.go +++ /dev/null @@ -1,11 +0,0 @@ -package event - -const ( - Base64 = "base64" -) - -// StringOfBase64 returns a string pointer to "Base64" -func StringOfBase64() *string { - a := Base64 - return &a -} diff --git a/vendor/github.com/cloudevents/sdk-go/v2/event/datacodec/codec.go b/vendor/github.com/cloudevents/sdk-go/v2/event/datacodec/codec.go deleted file mode 100644 index fd68ca55..00000000 --- a/vendor/github.com/cloudevents/sdk-go/v2/event/datacodec/codec.go +++ /dev/null @@ -1,73 +0,0 @@ -package datacodec - -import ( - "context" - "fmt" - - "github.com/cloudevents/sdk-go/v2/event/datacodec/json" - "github.com/cloudevents/sdk-go/v2/event/datacodec/text" - "github.com/cloudevents/sdk-go/v2/event/datacodec/xml" -) - -// Decoder is the expected function signature for decoding `in` to `out`. -// If Event sent the payload as base64, Decoder assumes that `in` is the -// decoded base64 byte array. -type Decoder func(ctx context.Context, in []byte, out interface{}) error - -// Encoder is the expected function signature for encoding `in` to bytes. -// Returns an error if the encoder has an issue encoding `in`. -type Encoder func(ctx context.Context, in interface{}) ([]byte, error) - -var decoder map[string]Decoder -var encoder map[string]Encoder - -func init() { - decoder = make(map[string]Decoder, 10) - encoder = make(map[string]Encoder, 10) - - AddDecoder("", json.Decode) - AddDecoder("application/json", json.Decode) - AddDecoder("text/json", json.Decode) - AddDecoder("application/xml", xml.Decode) - AddDecoder("text/xml", xml.Decode) - AddDecoder("text/plain", text.Decode) - - AddEncoder("", json.Encode) - AddEncoder("application/json", json.Encode) - AddEncoder("text/json", json.Encode) - AddEncoder("application/xml", xml.Encode) - AddEncoder("text/xml", xml.Encode) - AddEncoder("text/plain", text.Encode) -} - -// AddDecoder registers a decoder for a given content type. The codecs will use -// these to decode the data payload from a cloudevent.Event object. -func AddDecoder(contentType string, fn Decoder) { - decoder[contentType] = fn -} - -// AddEncoder registers an encoder for a given content type. The codecs will -// use these to encode the data payload for a cloudevent.Event object. -func AddEncoder(contentType string, fn Encoder) { - encoder[contentType] = fn -} - -// Decode looks up and invokes the decoder registered for the given content -// type. An error is returned if no decoder is registered for the given -// content type. -func Decode(ctx context.Context, contentType string, in []byte, out interface{}) error { - if fn, ok := decoder[contentType]; ok { - return fn(ctx, in, out) - } - return fmt.Errorf("[decode] unsupported content type: %q", contentType) -} - -// Encode looks up and invokes the encoder registered for the given content -// type. An error is returned if no encoder is registered for the given -// content type. -func Encode(ctx context.Context, contentType string, in interface{}) ([]byte, error) { - if fn, ok := encoder[contentType]; ok { - return fn(ctx, in) - } - return nil, fmt.Errorf("[encode] unsupported content type: %q", contentType) -} diff --git a/vendor/github.com/cloudevents/sdk-go/v2/event/datacodec/codec_observed.go b/vendor/github.com/cloudevents/sdk-go/v2/event/datacodec/codec_observed.go deleted file mode 100644 index b14e6f8b..00000000 --- a/vendor/github.com/cloudevents/sdk-go/v2/event/datacodec/codec_observed.go +++ /dev/null @@ -1,50 +0,0 @@ -package datacodec - -import ( - "context" - - "github.com/cloudevents/sdk-go/v2/event/datacodec/json" - "github.com/cloudevents/sdk-go/v2/event/datacodec/text" - "github.com/cloudevents/sdk-go/v2/event/datacodec/xml" - "github.com/cloudevents/sdk-go/v2/observability" -) - -func SetObservedCodecs() { - AddDecoder("", json.DecodeObserved) - AddDecoder("application/json", json.DecodeObserved) - AddDecoder("text/json", json.DecodeObserved) - AddDecoder("application/xml", xml.DecodeObserved) - AddDecoder("text/xml", xml.DecodeObserved) - AddDecoder("text/plain", text.DecodeObserved) - - AddEncoder("", json.Encode) - AddEncoder("application/json", json.EncodeObserved) - AddEncoder("text/json", json.EncodeObserved) - AddEncoder("application/xml", xml.EncodeObserved) - AddEncoder("text/xml", xml.EncodeObserved) - AddEncoder("text/plain", text.EncodeObserved) -} - -// DecodeObserved calls Decode and records the result. -func DecodeObserved(ctx context.Context, contentType string, in []byte, out interface{}) error { - _, r := observability.NewReporter(ctx, reportDecode) - err := Decode(ctx, contentType, in, out) - if err != nil { - r.Error() - } else { - r.OK() - } - return err -} - -// EncodeObserved calls Encode and records the result. -func EncodeObserved(ctx context.Context, contentType string, in interface{}) ([]byte, error) { - _, r := observability.NewReporter(ctx, reportEncode) - b, err := Encode(ctx, contentType, in) - if err != nil { - r.Error() - } else { - r.OK() - } - return b, err -} diff --git a/vendor/github.com/cloudevents/sdk-go/v2/event/datacodec/doc.go b/vendor/github.com/cloudevents/sdk-go/v2/event/datacodec/doc.go deleted file mode 100644 index 9e401534..00000000 --- a/vendor/github.com/cloudevents/sdk-go/v2/event/datacodec/doc.go +++ /dev/null @@ -1,5 +0,0 @@ -/* -Package datacodec holds the data codec registry and adds known encoders and decoders supporting media types such as -`application/json` and `application/xml`. -*/ -package datacodec diff --git a/vendor/github.com/cloudevents/sdk-go/v2/event/datacodec/json/data.go b/vendor/github.com/cloudevents/sdk-go/v2/event/datacodec/json/data.go deleted file mode 100644 index f40869b3..00000000 --- a/vendor/github.com/cloudevents/sdk-go/v2/event/datacodec/json/data.go +++ /dev/null @@ -1,51 +0,0 @@ -package json - -import ( - "context" - "encoding/json" - "fmt" - "reflect" -) - -// Decode takes `in` as []byte. -// If Event sent the payload as base64, Decoder assumes that `in` is the -// decoded base64 byte array. -func Decode(ctx context.Context, in []byte, out interface{}) error { - if in == nil { - return nil - } - if out == nil { - return fmt.Errorf("out is nil") - } - - if err := json.Unmarshal(in, out); err != nil { - return fmt.Errorf("[json] found bytes \"%s\", but failed to unmarshal: %s", string(in), err.Error()) - } - return nil -} - -// Encode attempts to json.Marshal `in` into bytes. Encode will inspect `in` -// and returns `in` unmodified if it is detected that `in` is already a []byte; -// Or json.Marshal errors. -func Encode(ctx context.Context, in interface{}) ([]byte, error) { - if in == nil { - return nil, nil - } - - it := reflect.TypeOf(in) - switch it.Kind() { - case reflect.Slice: - if it.Elem().Kind() == reflect.Uint8 { - - if b, ok := in.([]byte); ok && len(b) > 0 { - // check to see if it is a pre-encoded byte string. - if b[0] == byte('"') || b[0] == byte('{') || b[0] == byte('[') { - return b, nil - } - } - - } - } - - return json.Marshal(in) -} diff --git a/vendor/github.com/cloudevents/sdk-go/v2/event/datacodec/json/data_observed.go b/vendor/github.com/cloudevents/sdk-go/v2/event/datacodec/json/data_observed.go deleted file mode 100644 index 21308ce8..00000000 --- a/vendor/github.com/cloudevents/sdk-go/v2/event/datacodec/json/data_observed.go +++ /dev/null @@ -1,30 +0,0 @@ -package json - -import ( - "context" - "github.com/cloudevents/sdk-go/v2/observability" -) - -// DecodeObserved calls Decode and records the results. -func DecodeObserved(ctx context.Context, in []byte, out interface{}) error { - _, r := observability.NewReporter(ctx, reportDecode) - err := Decode(ctx, in, out) - if err != nil { - r.Error() - } else { - r.OK() - } - return err -} - -// EncodeObserved calls Encode and records the results. -func EncodeObserved(ctx context.Context, in interface{}) ([]byte, error) { - _, r := observability.NewReporter(ctx, reportEncode) - b, err := Encode(ctx, in) - if err != nil { - r.Error() - } else { - r.OK() - } - return b, err -} diff --git a/vendor/github.com/cloudevents/sdk-go/v2/event/datacodec/json/doc.go b/vendor/github.com/cloudevents/sdk-go/v2/event/datacodec/json/doc.go deleted file mode 100644 index 86772c2e..00000000 --- a/vendor/github.com/cloudevents/sdk-go/v2/event/datacodec/json/doc.go +++ /dev/null @@ -1,4 +0,0 @@ -/* -Package json holds the encoder/decoder implementation for `application/json`. -*/ -package json diff --git a/vendor/github.com/cloudevents/sdk-go/v2/event/datacodec/json/observability.go b/vendor/github.com/cloudevents/sdk-go/v2/event/datacodec/json/observability.go deleted file mode 100644 index 7ff79659..00000000 --- a/vendor/github.com/cloudevents/sdk-go/v2/event/datacodec/json/observability.go +++ /dev/null @@ -1,51 +0,0 @@ -package json - -import ( - "github.com/cloudevents/sdk-go/v2/observability" - "go.opencensus.io/stats" - "go.opencensus.io/stats/view" -) - -var ( - // LatencyMs measures the latency in milliseconds for the CloudEvents json - // data codec methods. - LatencyMs = stats.Float64("cloudevents.io/sdk-go/datacodec/json/latency", "The latency in milliseconds for the CloudEvents json data codec methods.", "ms") -) - -var ( - // LatencyView is an OpenCensus view that shows data codec json method latency. - LatencyView = &view.View{ - Name: "datacodec/json/latency", - Measure: LatencyMs, - Description: "The distribution of latency inside of the json data codec for CloudEvents.", - Aggregation: view.Distribution(0, .01, .1, 1, 10, 100, 1000, 10000), - TagKeys: observability.LatencyTags(), - } -) - -type observed int32 - -// Adheres to Observable -var _ observability.Observable = observed(0) - -const ( - reportEncode observed = iota - reportDecode -) - -// MethodName implements Observable.MethodName -func (o observed) MethodName() string { - switch o { - case reportEncode: - return "encode" - case reportDecode: - return "decode" - default: - return "unknown" - } -} - -// LatencyMs implements Observable.LatencyMs -func (o observed) LatencyMs() *stats.Float64Measure { - return LatencyMs -} diff --git a/vendor/github.com/cloudevents/sdk-go/v2/event/datacodec/observability.go b/vendor/github.com/cloudevents/sdk-go/v2/event/datacodec/observability.go deleted file mode 100644 index 870ec5df..00000000 --- a/vendor/github.com/cloudevents/sdk-go/v2/event/datacodec/observability.go +++ /dev/null @@ -1,51 +0,0 @@ -package datacodec - -import ( - "github.com/cloudevents/sdk-go/v2/observability" - "go.opencensus.io/stats" - "go.opencensus.io/stats/view" -) - -var ( - // LatencyMs measures the latency in milliseconds for the CloudEvents generic - // codec data methods. - LatencyMs = stats.Float64("cloudevents.io/sdk-go/datacodec/latency", "The latency in milliseconds for the CloudEvents generic data codec methods.", "ms") -) - -var ( - // LatencyView is an OpenCensus view that shows data codec method latency. - LatencyView = &view.View{ - Name: "datacodec/latency", - Measure: LatencyMs, - Description: "The distribution of latency inside of the generic data codec for CloudEvents.", - Aggregation: view.Distribution(0, .01, .1, 1, 10, 100, 1000, 10000), - TagKeys: observability.LatencyTags(), - } -) - -type observed int32 - -// Adheres to Observable -var _ observability.Observable = observed(0) - -const ( - reportEncode observed = iota - reportDecode -) - -// MethodName implements Observable.MethodName -func (o observed) MethodName() string { - switch o { - case reportEncode: - return "encode" - case reportDecode: - return "decode" - default: - return "unknown" - } -} - -// LatencyMs implements Observable.LatencyMs -func (o observed) LatencyMs() *stats.Float64Measure { - return LatencyMs -} diff --git a/vendor/github.com/cloudevents/sdk-go/v2/event/datacodec/text/data.go b/vendor/github.com/cloudevents/sdk-go/v2/event/datacodec/text/data.go deleted file mode 100644 index 5e6ddc08..00000000 --- a/vendor/github.com/cloudevents/sdk-go/v2/event/datacodec/text/data.go +++ /dev/null @@ -1,25 +0,0 @@ -package text - -import ( - "context" - "fmt" -) - -// Text codec converts []byte or string to string and vice-versa. - -func Decode(_ context.Context, in []byte, out interface{}) error { - p, _ := out.(*string) - if p == nil { - return fmt.Errorf("text.Decode out: want *string, got %T", out) - } - *p = string(in) - return nil -} - -func Encode(_ context.Context, in interface{}) ([]byte, error) { - s, ok := in.(string) - if !ok { - return nil, fmt.Errorf("text.Encode in: want string, got %T", in) - } - return []byte(s), nil -} diff --git a/vendor/github.com/cloudevents/sdk-go/v2/event/datacodec/text/data_observed.go b/vendor/github.com/cloudevents/sdk-go/v2/event/datacodec/text/data_observed.go deleted file mode 100644 index 2897ea6b..00000000 --- a/vendor/github.com/cloudevents/sdk-go/v2/event/datacodec/text/data_observed.go +++ /dev/null @@ -1,30 +0,0 @@ -package text - -import ( - "context" - "github.com/cloudevents/sdk-go/v2/observability" -) - -// DecodeObserved calls Decode and records the results. -func DecodeObserved(ctx context.Context, in []byte, out interface{}) error { - _, r := observability.NewReporter(ctx, reportDecode) - err := Decode(ctx, in, out) - if err != nil { - r.Error() - } else { - r.OK() - } - return err -} - -// EncodeObserved calls Encode and records the results. -func EncodeObserved(ctx context.Context, in interface{}) ([]byte, error) { - _, r := observability.NewReporter(ctx, reportEncode) - b, err := Encode(ctx, in) - if err != nil { - r.Error() - } else { - r.OK() - } - return b, err -} diff --git a/vendor/github.com/cloudevents/sdk-go/v2/event/datacodec/text/doc.go b/vendor/github.com/cloudevents/sdk-go/v2/event/datacodec/text/doc.go deleted file mode 100644 index 13316702..00000000 --- a/vendor/github.com/cloudevents/sdk-go/v2/event/datacodec/text/doc.go +++ /dev/null @@ -1,4 +0,0 @@ -/* -Package text holds the encoder/decoder implementation for `text/plain`. -*/ -package text diff --git a/vendor/github.com/cloudevents/sdk-go/v2/event/datacodec/text/observability.go b/vendor/github.com/cloudevents/sdk-go/v2/event/datacodec/text/observability.go deleted file mode 100644 index ede85a2a..00000000 --- a/vendor/github.com/cloudevents/sdk-go/v2/event/datacodec/text/observability.go +++ /dev/null @@ -1,51 +0,0 @@ -package text - -import ( - "github.com/cloudevents/sdk-go/v2/observability" - "go.opencensus.io/stats" - "go.opencensus.io/stats/view" -) - -var ( - // LatencyMs measures the latency in milliseconds for the CloudEvents xml data - // codec methods. - LatencyMs = stats.Float64("cloudevents.io/sdk-go/datacodec/text/latency", "The latency in milliseconds for the CloudEvents text data codec methods.", "ms") -) - -var ( - // LatencyView is an OpenCensus view that shows data codec xml method latency. - LatencyView = &view.View{ - Name: "datacodec/text/latency", - Measure: LatencyMs, - Description: "The distribution of latency inside of the text data codec for CloudEvents.", - Aggregation: view.Distribution(0, .01, .1, 1, 10, 100, 1000, 10000), - TagKeys: observability.LatencyTags(), - } -) - -type observed int32 - -// Adheres to Observable -var _ observability.Observable = observed(0) - -const ( - reportEncode observed = iota - reportDecode -) - -// MethodName implements Observable.MethodName -func (o observed) MethodName() string { - switch o { - case reportEncode: - return "encode" - case reportDecode: - return "decode" - default: - return "unknown" - } -} - -// LatencyMs implements Observable.LatencyMs -func (o observed) LatencyMs() *stats.Float64Measure { - return LatencyMs -} diff --git a/vendor/github.com/cloudevents/sdk-go/v2/event/datacodec/xml/data.go b/vendor/github.com/cloudevents/sdk-go/v2/event/datacodec/xml/data.go deleted file mode 100644 index 13045e03..00000000 --- a/vendor/github.com/cloudevents/sdk-go/v2/event/datacodec/xml/data.go +++ /dev/null @@ -1,35 +0,0 @@ -package xml - -import ( - "context" - "encoding/xml" - "fmt" -) - -// Decode takes `in` as []byte. -// If Event sent the payload as base64, Decoder assumes that `in` is the -// decoded base64 byte array. -func Decode(ctx context.Context, in []byte, out interface{}) error { - if in == nil { - return nil - } - - if err := xml.Unmarshal(in, out); err != nil { - return fmt.Errorf("[xml] found bytes, but failed to unmarshal: %s %s", err.Error(), string(in)) - } - return nil -} - -// Encode attempts to xml.Marshal `in` into bytes. Encode will inspect `in` -// and returns `in` unmodified if it is detected that `in` is already a []byte; -// Or xml.Marshal errors. -func Encode(ctx context.Context, in interface{}) ([]byte, error) { - if b, ok := in.([]byte); ok { - // check to see if it is a pre-encoded byte string. - if len(b) > 0 && b[0] == byte('"') { - return b, nil - } - } - - return xml.Marshal(in) -} diff --git a/vendor/github.com/cloudevents/sdk-go/v2/event/datacodec/xml/data_observed.go b/vendor/github.com/cloudevents/sdk-go/v2/event/datacodec/xml/data_observed.go deleted file mode 100644 index 14f6c282..00000000 --- a/vendor/github.com/cloudevents/sdk-go/v2/event/datacodec/xml/data_observed.go +++ /dev/null @@ -1,30 +0,0 @@ -package xml - -import ( - "context" - "github.com/cloudevents/sdk-go/v2/observability" -) - -// DecodeObserved calls Decode and records the result. -func DecodeObserved(ctx context.Context, in []byte, out interface{}) error { - _, r := observability.NewReporter(ctx, reportDecode) - err := Decode(ctx, in, out) - if err != nil { - r.Error() - } else { - r.OK() - } - return err -} - -// EncodeObserved calls Encode and records the result. -func EncodeObserved(ctx context.Context, in interface{}) ([]byte, error) { - _, r := observability.NewReporter(ctx, reportEncode) - b, err := Encode(ctx, in) - if err != nil { - r.Error() - } else { - r.OK() - } - return b, err -} diff --git a/vendor/github.com/cloudevents/sdk-go/v2/event/datacodec/xml/doc.go b/vendor/github.com/cloudevents/sdk-go/v2/event/datacodec/xml/doc.go deleted file mode 100644 index d90b7c44..00000000 --- a/vendor/github.com/cloudevents/sdk-go/v2/event/datacodec/xml/doc.go +++ /dev/null @@ -1,4 +0,0 @@ -/* -Package xml holds the encoder/decoder implementation for `application/xml`. -*/ -package xml diff --git a/vendor/github.com/cloudevents/sdk-go/v2/event/datacodec/xml/observability.go b/vendor/github.com/cloudevents/sdk-go/v2/event/datacodec/xml/observability.go deleted file mode 100644 index b0f4c935..00000000 --- a/vendor/github.com/cloudevents/sdk-go/v2/event/datacodec/xml/observability.go +++ /dev/null @@ -1,51 +0,0 @@ -package xml - -import ( - "github.com/cloudevents/sdk-go/v2/observability" - "go.opencensus.io/stats" - "go.opencensus.io/stats/view" -) - -var ( - // LatencyMs measures the latency in milliseconds for the CloudEvents xml data - // codec methods. - LatencyMs = stats.Float64("cloudevents.io/sdk-go/datacodec/xml/latency", "The latency in milliseconds for the CloudEvents xml data codec methods.", "ms") -) - -var ( - // LatencyView is an OpenCensus view that shows data codec xml method latency. - LatencyView = &view.View{ - Name: "datacodec/xml/latency", - Measure: LatencyMs, - Description: "The distribution of latency inside of the xml data codec for CloudEvents.", - Aggregation: view.Distribution(0, .01, .1, 1, 10, 100, 1000, 10000), - TagKeys: observability.LatencyTags(), - } -) - -type observed int32 - -// Adheres to Observable -var _ observability.Observable = observed(0) - -const ( - reportEncode observed = iota - reportDecode -) - -// MethodName implements Observable.MethodName -func (o observed) MethodName() string { - switch o { - case reportEncode: - return "encode" - case reportDecode: - return "decode" - default: - return "unknown" - } -} - -// LatencyMs implements Observable.LatencyMs -func (o observed) LatencyMs() *stats.Float64Measure { - return LatencyMs -} diff --git a/vendor/github.com/cloudevents/sdk-go/v2/event/doc.go b/vendor/github.com/cloudevents/sdk-go/v2/event/doc.go deleted file mode 100644 index eebbeb4e..00000000 --- a/vendor/github.com/cloudevents/sdk-go/v2/event/doc.go +++ /dev/null @@ -1,4 +0,0 @@ -/* -Package event provides primitives to work with CloudEvents specification: https://github.com/cloudevents/spec. -*/ -package event diff --git a/vendor/github.com/cloudevents/sdk-go/v2/event/event.go b/vendor/github.com/cloudevents/sdk-go/v2/event/event.go deleted file mode 100644 index 3f8215a0..00000000 --- a/vendor/github.com/cloudevents/sdk-go/v2/event/event.go +++ /dev/null @@ -1,134 +0,0 @@ -package event - -import ( - "bytes" - "encoding/json" - "fmt" - "strings" -) - -// Event represents the canonical representation of a CloudEvent. -type Event struct { - Context EventContext - DataEncoded []byte - // DataBase64 indicates if the event, when serialized, represents - // the data field using the base64 encoding. - // In v0.3, this field is superseded by DataContentEncoding - DataBase64 bool - FieldErrors map[string]error -} - -const ( - defaultEventVersion = CloudEventsVersionV1 -) - -func (e *Event) fieldError(field string, err error) { - if e.FieldErrors == nil { - e.FieldErrors = make(map[string]error) - } - e.FieldErrors[field] = err -} - -func (e *Event) fieldOK(field string) { - if e.FieldErrors != nil { - delete(e.FieldErrors, field) - } -} - -// New returns a new Event, an optional version can be passed to change the -// default spec version from 1.0 to the provided version. -func New(version ...string) Event { - specVersion := defaultEventVersion - if len(version) >= 1 { - specVersion = version[0] - } - e := &Event{} - e.SetSpecVersion(specVersion) - return *e -} - -// ExtensionAs is deprecated: access extensions directly via the e.Extensions() map. -// Use functions in the types package to convert extension values. -// For example replace this: -// -// var i int -// err := e.ExtensionAs("foo", &i) -// -// With this: -// -// i, err := types.ToInteger(e.Extensions["foo"]) -// -func (e Event) ExtensionAs(name string, obj interface{}) error { - return e.Context.ExtensionAs(name, obj) -} - -// String returns a pretty-printed representation of the Event. -func (e Event) String() string { - b := strings.Builder{} - - b.WriteString("Validation: ") - - valid := e.Validate() - if valid == nil { - b.WriteString("valid\n") - } else { - b.WriteString("invalid\n") - } - if valid != nil { - b.WriteString(fmt.Sprintf("Validation Error: \n%s\n", valid.Error())) - } - - b.WriteString(e.Context.String()) - - if e.DataEncoded != nil { - if e.DataBase64 { - b.WriteString("Data (binary),\n ") - } else { - b.WriteString("Data,\n ") - } - switch e.DataMediaType() { - case ApplicationJSON: - var prettyJSON bytes.Buffer - err := json.Indent(&prettyJSON, e.DataEncoded, " ", " ") - if err != nil { - b.Write(e.DataEncoded) - } else { - b.Write(prettyJSON.Bytes()) - } - default: - b.Write(e.DataEncoded) - } - b.WriteString("\n") - } - - return b.String() -} - -func (e Event) Clone() Event { - out := Event{} - out.Context = e.Context.Clone() - out.DataEncoded = cloneBytes(e.DataEncoded) - out.DataBase64 = e.DataBase64 - out.FieldErrors = e.cloneFieldErrors() - return out -} - -func cloneBytes(in []byte) []byte { - if in == nil { - return nil - } - out := make([]byte, len(in)) - copy(out, in) - return out -} - -func (e Event) cloneFieldErrors() map[string]error { - if e.FieldErrors == nil { - return nil - } - newFE := make(map[string]error, len(e.FieldErrors)) - for k, v := range e.FieldErrors { - newFE[k] = v - } - return newFE -} diff --git a/vendor/github.com/cloudevents/sdk-go/v2/event/event_data.go b/vendor/github.com/cloudevents/sdk-go/v2/event/event_data.go deleted file mode 100644 index c85fe7e5..00000000 --- a/vendor/github.com/cloudevents/sdk-go/v2/event/event_data.go +++ /dev/null @@ -1,113 +0,0 @@ -package event - -import ( - "context" - "encoding/base64" - "fmt" - "strconv" - - "github.com/cloudevents/sdk-go/v2/event/datacodec" -) - -// SetData encodes the given payload with the given content type. -// If the provided payload is a byte array, when marshalled to json it will be encoded as base64. -// If the provided payload is different from byte array, datacodec.Encode is invoked to attempt a -// marshalling to byte array. -func (e *Event) SetData(contentType string, obj interface{}) error { - e.SetDataContentType(contentType) - - if e.SpecVersion() != CloudEventsVersionV1 { - return e.legacySetData(obj) - } - - // Version 1.0 and above. - switch obj := obj.(type) { - case []byte: - e.DataEncoded = obj - e.DataBase64 = true - default: - data, err := datacodec.Encode(context.Background(), e.DataMediaType(), obj) - if err != nil { - return err - } - e.DataEncoded = data - e.DataBase64 = false - } - - return nil -} - -// Deprecated: Delete when we do not have to support Spec v0.3. -func (e *Event) legacySetData(obj interface{}) error { - data, err := datacodec.Encode(context.Background(), e.DataMediaType(), obj) - if err != nil { - return err - } - if e.DeprecatedDataContentEncoding() == Base64 { - buf := make([]byte, base64.StdEncoding.EncodedLen(len(data))) - base64.StdEncoding.Encode(buf, data) - e.DataEncoded = buf - e.DataBase64 = false - } else { - data, err := datacodec.Encode(context.Background(), e.DataMediaType(), obj) - if err != nil { - return err - } - e.DataEncoded = data - e.DataBase64 = false - } - return nil -} - -const ( - quotes = `"'` -) - -func (e Event) Data() []byte { - return e.DataEncoded -} - -// DataAs attempts to populate the provided data object with the event payload. -// data should be a pointer type. -func (e Event) DataAs(obj interface{}) error { - data := e.Data() - - if len(data) == 0 { - // No data. - return nil - } - - if e.SpecVersion() != CloudEventsVersionV1 { - var err error - if data, err = e.legacyConvertData(data); err != nil { - return err - } - } - - return datacodec.Decode(context.Background(), e.DataMediaType(), data, obj) -} - -func (e Event) legacyConvertData(data []byte) ([]byte, error) { - if e.Context.DeprecatedGetDataContentEncoding() == Base64 { - var bs []byte - // test to see if we need to unquote the data. - if data[0] == quotes[0] || data[0] == quotes[1] { - str, err := strconv.Unquote(string(data)) - if err != nil { - return nil, err - } - bs = []byte(str) - } else { - bs = data - } - - buf := make([]byte, base64.StdEncoding.DecodedLen(len(bs))) - n, err := base64.StdEncoding.Decode(buf, bs) - if err != nil { - return nil, fmt.Errorf("failed to decode data from base64: %s", err.Error()) - } - data = buf[:n] - } - - return data, nil -} diff --git a/vendor/github.com/cloudevents/sdk-go/v2/event/event_interface.go b/vendor/github.com/cloudevents/sdk-go/v2/event/event_interface.go deleted file mode 100644 index af87454d..00000000 --- a/vendor/github.com/cloudevents/sdk-go/v2/event/event_interface.go +++ /dev/null @@ -1,97 +0,0 @@ -package event - -import ( - "time" -) - -// EventReader is the interface for reading through an event from attributes. -type EventReader interface { - // SpecVersion returns event.Context.GetSpecVersion(). - SpecVersion() string - // Type returns event.Context.GetType(). - Type() string - // Source returns event.Context.GetSource(). - Source() string - // Subject returns event.Context.GetSubject(). - Subject() string - // ID returns event.Context.GetID(). - ID() string - // Time returns event.Context.GetTime(). - Time() time.Time - // DataSchema returns event.Context.GetDataSchema(). - DataSchema() string - // DataContentType returns event.Context.GetDataContentType(). - DataContentType() string - // DataMediaType returns event.Context.GetDataMediaType(). - DataMediaType() string - // DeprecatedDataContentEncoding returns event.Context.DeprecatedGetDataContentEncoding(). - DeprecatedDataContentEncoding() string - - // Extension Attributes - - // Extensions returns the event.Context.GetExtensions(). - // Extensions use the CloudEvents type system, details in package cloudevents/types. - Extensions() map[string]interface{} - - // ExtensionAs returns event.Context.ExtensionAs(name, obj). - // - // DEPRECATED: Access extensions directly via the e.Extensions() map. - // Use functions in the types package to convert extension values. - // For example replace this: - // - // var i int - // err := e.ExtensionAs("foo", &i) - // - // With this: - // - // i, err := types.ToInteger(e.Extensions["foo"]) - // - ExtensionAs(string, interface{}) error - - // Data Attribute - - // Data returns the raw data buffer - // If the event was encoded with base64 encoding, Data returns the already decoded - // byte array - Data() []byte - - // DataAs attempts to populate the provided data object with the event payload. - DataAs(interface{}) error -} - -// EventWriter is the interface for writing through an event onto attributes. -// If an error is thrown by a sub-component, EventWriter caches the error -// internally and exposes errors with a call to event.Validate(). -type EventWriter interface { - // Context Attributes - - // SetSpecVersion performs event.Context.SetSpecVersion. - SetSpecVersion(string) - // SetType performs event.Context.SetType. - SetType(string) - // SetSource performs event.Context.SetSource. - SetSource(string) - // SetSubject( performs event.Context.SetSubject. - SetSubject(string) - // SetID performs event.Context.SetID. - SetID(string) - // SetTime performs event.Context.SetTime. - SetTime(time.Time) - // SetDataSchema performs event.Context.SetDataSchema. - SetDataSchema(string) - // SetDataContentType performs event.Context.SetDataContentType. - SetDataContentType(string) - // DeprecatedSetDataContentEncoding performs event.Context.DeprecatedSetDataContentEncoding. - SetDataContentEncoding(string) - - // Extension Attributes - - // SetExtension performs event.Context.SetExtension. - SetExtension(string, interface{}) - - // SetData encodes the given payload with the given content type. - // If the provided payload is a byte array, when marshalled to json it will be encoded as base64. - // If the provided payload is different from byte array, datacodec.Encode is invoked to attempt a - // marshalling to byte array. - SetData(string, interface{}) error -} diff --git a/vendor/github.com/cloudevents/sdk-go/v2/event/event_marshal.go b/vendor/github.com/cloudevents/sdk-go/v2/event/event_marshal.go deleted file mode 100644 index 2289774f..00000000 --- a/vendor/github.com/cloudevents/sdk-go/v2/event/event_marshal.go +++ /dev/null @@ -1,324 +0,0 @@ -package event - -import ( - "context" - "encoding/json" - "fmt" - "strings" - - "github.com/cloudevents/sdk-go/v2/observability" -) - -// MarshalJSON implements a custom json marshal method used when this type is -// marshaled using json.Marshal. -func (e Event) MarshalJSON() ([]byte, error) { - _, r := observability.NewReporter(context.Background(), eventJSONObserved{o: reportMarshal, v: e.SpecVersion()}) - - if err := e.Validate(); err != nil { - r.Error() - return nil, err - } - - var b []byte - var err error - - switch e.SpecVersion() { - case CloudEventsVersionV03: - b, err = JsonEncodeLegacy(e) - case CloudEventsVersionV1: - b, err = JsonEncode(e) - default: - return nil, ValidationError{"specversion": fmt.Errorf("unknown : %q", e.SpecVersion())} - } - - // Report the observable - if err != nil { - r.Error() - return nil, err - } else { - r.OK() - } - - return b, nil -} - -// UnmarshalJSON implements the json unmarshal method used when this type is -// unmarshaled using json.Unmarshal. -func (e *Event) UnmarshalJSON(b []byte) error { - raw := make(map[string]json.RawMessage) - if err := json.Unmarshal(b, &raw); err != nil { - return err - } - - version := versionFromRawMessage(raw) - - _, r := observability.NewReporter(context.Background(), eventJSONObserved{o: reportUnmarshal, v: version}) - - var err error - switch version { - case CloudEventsVersionV03: - err = e.JsonDecodeV03(b, raw) - case CloudEventsVersionV1: - err = e.JsonDecodeV1(b, raw) - default: - return ValidationError{"specversion": fmt.Errorf("unknown : %q", version)} - } - - // Report the observable - if err != nil { - r.Error() - return err - } else { - r.OK() - } - return nil -} - -func versionFromRawMessage(raw map[string]json.RawMessage) string { - // v0.2 and after - if v, ok := raw["specversion"]; ok { - var version string - if err := json.Unmarshal(v, &version); err != nil { - return "" - } - return version - } - return "" -} - -// JsonEncode encodes an event to JSON -func JsonEncode(e Event) ([]byte, error) { - return jsonEncode(e.Context, e.DataEncoded, e.DataBase64) -} - -// JsonEncodeLegacy performs legacy JSON encoding -func JsonEncodeLegacy(e Event) ([]byte, error) { - isBase64 := e.Context.DeprecatedGetDataContentEncoding() == Base64 - return jsonEncode(e.Context, e.DataEncoded, isBase64) -} - -func jsonEncode(ctx EventContextReader, data []byte, shouldEncodeToBase64 bool) ([]byte, error) { - var b map[string]json.RawMessage - var err error - - b, err = marshalEvent(ctx, ctx.GetExtensions()) - if err != nil { - return nil, err - } - - if data != nil { - // data here is a serialized version of whatever payload. - // If we need to write the payload as base64, shouldEncodeToBase64 is true. - mediaType, err := ctx.GetDataMediaType() - if err != nil { - return nil, err - } - isJson := mediaType == "" || mediaType == ApplicationJSON || mediaType == TextJSON - // If isJson and no encoding to base64, we don't need to perform additional steps - if isJson && !shouldEncodeToBase64 { - b["data"] = data - } else { - var dataKey = "data" - if ctx.GetSpecVersion() == CloudEventsVersionV1 && shouldEncodeToBase64 { - dataKey = "data_base64" - } - var dataPointer []byte - if shouldEncodeToBase64 { - dataPointer, err = json.Marshal(data) - } else { - dataPointer, err = json.Marshal(string(data)) - } - if err != nil { - return nil, err - } - - b[dataKey] = dataPointer - } - } - - body, err := json.Marshal(b) - if err != nil { - return nil, err - } - - return body, nil -} - -// JsonDecodeV03 takes in the byte representation of a version 0.3 structured json CloudEvent and returns a -// cloudevent.Event or an error if there are parsing errors. -func (e *Event) JsonDecodeV03(body []byte, raw map[string]json.RawMessage) error { - ec := EventContextV03{} - if err := json.Unmarshal(body, &ec); err != nil { - return err - } - - delete(raw, "specversion") - delete(raw, "type") - delete(raw, "source") - delete(raw, "subject") - delete(raw, "id") - delete(raw, "time") - delete(raw, "schemaurl") - delete(raw, "datacontenttype") - delete(raw, "datacontentencoding") - - var data []byte - if d, ok := raw["data"]; ok { - data = d - - // Decode the Base64 if we have a base64 payload - if ec.DeprecatedGetDataContentEncoding() == Base64 { - var tmp []byte - if err := json.Unmarshal(d, &tmp); err != nil { - return err - } - e.DataBase64 = true - e.DataEncoded = tmp - } else { - if ec.DataContentType != nil { - ct := *ec.DataContentType - if ct != ApplicationJSON && ct != TextJSON { - var dataStr string - err := json.Unmarshal(d, &dataStr) - if err != nil { - return err - } - - data = []byte(dataStr) - } - } - e.DataEncoded = data - e.DataBase64 = false - } - } - delete(raw, "data") - - if len(raw) > 0 { - extensions := make(map[string]interface{}, len(raw)) - ec.Extensions = extensions - for k, v := range raw { - k = strings.ToLower(k) - var tmp interface{} - if err := json.Unmarshal(v, &tmp); err != nil { - return err - } - if err := ec.SetExtension(k, tmp); err != nil { - return fmt.Errorf("%w: Cannot set extension with key %s", err, k) - } - } - } - - e.Context = &ec - - return nil -} - -// JsonDecodeV1 takes in the byte representation of a version 1.0 structured json CloudEvent and returns a -// cloudevent.Event or an error if there are parsing errors. -func (e *Event) JsonDecodeV1(body []byte, raw map[string]json.RawMessage) error { - ec := EventContextV1{} - if err := json.Unmarshal(body, &ec); err != nil { - return err - } - - delete(raw, "specversion") - delete(raw, "type") - delete(raw, "source") - delete(raw, "subject") - delete(raw, "id") - delete(raw, "time") - delete(raw, "dataschema") - delete(raw, "datacontenttype") - - var data []byte - if d, ok := raw["data"]; ok { - data = d - if ec.DataContentType != nil { - ct := *ec.DataContentType - if ct != ApplicationJSON && ct != TextJSON { - var dataStr string - err := json.Unmarshal(d, &dataStr) - if err != nil { - return err - } - - data = []byte(dataStr) - } - } - } - delete(raw, "data") - - var dataBase64 []byte - if d, ok := raw["data_base64"]; ok { - var tmp []byte - if err := json.Unmarshal(d, &tmp); err != nil { - return err - } - dataBase64 = tmp - - } - delete(raw, "data_base64") - - if data != nil && dataBase64 != nil { - return ValidationError{"data": fmt.Errorf("found both 'data', and 'data_base64' in JSON payload")} - } - if data != nil { - e.DataEncoded = data - e.DataBase64 = false - } else if dataBase64 != nil { - e.DataEncoded = dataBase64 - e.DataBase64 = true - } - - if len(raw) > 0 { - extensions := make(map[string]interface{}, len(raw)) - ec.Extensions = extensions - for k, v := range raw { - k = strings.ToLower(k) - var tmp interface{} - if err := json.Unmarshal(v, &tmp); err != nil { - return err - } - if err := ec.SetExtension(k, tmp); err != nil { - return fmt.Errorf("%w: Cannot set extension with key %s", err, k) - } - } - } - - e.Context = &ec - - return nil -} - -func marshalEvent(eventCtx EventContextReader, extensions map[string]interface{}) (map[string]json.RawMessage, error) { - b, err := json.Marshal(eventCtx) - if err != nil { - return nil, err - } - - brm := map[string]json.RawMessage{} - if err := json.Unmarshal(b, &brm); err != nil { - return nil, err - } - - sv, err := json.Marshal(eventCtx.GetSpecVersion()) - if err != nil { - return nil, err - } - - brm["specversion"] = sv - - for k, v := range extensions { - k = strings.ToLower(k) - vb, err := json.Marshal(v) - if err != nil { - return nil, err - } - // Don't overwrite spec keys. - if _, ok := brm[k]; !ok { - brm[k] = vb - } - } - - return brm, nil -} diff --git a/vendor/github.com/cloudevents/sdk-go/v2/event/event_observability.go b/vendor/github.com/cloudevents/sdk-go/v2/event/event_observability.go deleted file mode 100644 index e21a845f..00000000 --- a/vendor/github.com/cloudevents/sdk-go/v2/event/event_observability.go +++ /dev/null @@ -1,77 +0,0 @@ -package event - -import ( - "fmt" - - "github.com/cloudevents/sdk-go/v2/observability" - "go.opencensus.io/stats" - "go.opencensus.io/stats/view" -) - -var ( - // EventMarshalLatencyMs measures the latency in milliseconds for the - // CloudEvents.Event marshal/unmarshalJSON methods. - EventMarshalLatencyMs = stats.Float64( - "cloudevents.io/sdk-go/event/json/latency", - "The latency in milliseconds of (un)marshalJSON methods for CloudEvents.Event.", - "ms") -) - -var ( - // LatencyView is an OpenCensus view that shows CloudEvents.Event (un)marshalJSON method latency. - EventMarshalLatencyView = &view.View{ - Name: "event/json/latency", - Measure: EventMarshalLatencyMs, - Description: "The distribution of latency inside of (un)marshalJSON methods for CloudEvents.Event.", - Aggregation: view.Distribution(0, .01, .1, 1, 10, 100, 1000, 10000), - TagKeys: observability.LatencyTags(), - } -) - -type observed int32 - -// Adheres to Observable -var _ observability.Observable = observed(0) - -const ( - reportMarshal observed = iota - reportUnmarshal -) - -// MethodName implements Observable.MethodName -func (o observed) MethodName() string { - switch o { - case reportMarshal: - return "marshaljson" - case reportUnmarshal: - return "unmarshaljson" - default: - return "unknown" - } -} - -// LatencyMs implements Observable.LatencyMs -func (o observed) LatencyMs() *stats.Float64Measure { - return EventMarshalLatencyMs -} - -// eventJSONObserved is a wrapper to append version to observed. -type eventJSONObserved struct { - // Method - o observed - // Version - v string -} - -// Adheres to Observable -var _ observability.Observable = (*eventJSONObserved)(nil) - -// MethodName implements Observable.MethodName -func (c eventJSONObserved) MethodName() string { - return fmt.Sprintf("%s/%s", c.o.MethodName(), c.v) -} - -// LatencyMs implements Observable.LatencyMs -func (c eventJSONObserved) LatencyMs() *stats.Float64Measure { - return c.o.LatencyMs() -} diff --git a/vendor/github.com/cloudevents/sdk-go/v2/event/event_reader.go b/vendor/github.com/cloudevents/sdk-go/v2/event/event_reader.go deleted file mode 100644 index 86ca609b..00000000 --- a/vendor/github.com/cloudevents/sdk-go/v2/event/event_reader.go +++ /dev/null @@ -1,98 +0,0 @@ -package event - -import ( - "time" -) - -var _ EventReader = (*Event)(nil) - -// SpecVersion implements EventReader.SpecVersion -func (e Event) SpecVersion() string { - if e.Context != nil { - return e.Context.GetSpecVersion() - } - return "" -} - -// Type implements EventReader.Type -func (e Event) Type() string { - if e.Context != nil { - return e.Context.GetType() - } - return "" -} - -// Source implements EventReader.Source -func (e Event) Source() string { - if e.Context != nil { - return e.Context.GetSource() - } - return "" -} - -// Subject implements EventReader.Subject -func (e Event) Subject() string { - if e.Context != nil { - return e.Context.GetSubject() - } - return "" -} - -// ID implements EventReader.ID -func (e Event) ID() string { - if e.Context != nil { - return e.Context.GetID() - } - return "" -} - -// Time implements EventReader.Time -func (e Event) Time() time.Time { - if e.Context != nil { - return e.Context.GetTime() - } - return time.Time{} -} - -// DataSchema implements EventReader.DataSchema -func (e Event) DataSchema() string { - if e.Context != nil { - return e.Context.GetDataSchema() - } - return "" -} - -// DataContentType implements EventReader.DataContentType -func (e Event) DataContentType() string { - if e.Context != nil { - return e.Context.GetDataContentType() - } - return "" -} - -// DataMediaType returns the parsed DataMediaType of the event. If parsing -// fails, the empty string is returned. To retrieve the parsing error, use -// `Context.GetDataMediaType` instead. -func (e Event) DataMediaType() string { - if e.Context != nil { - mediaType, _ := e.Context.GetDataMediaType() - return mediaType - } - return "" -} - -// DeprecatedDataContentEncoding implements EventReader.DeprecatedDataContentEncoding -func (e Event) DeprecatedDataContentEncoding() string { - if e.Context != nil { - return e.Context.DeprecatedGetDataContentEncoding() - } - return "" -} - -// Extensions implements EventReader.Extensions -func (e Event) Extensions() map[string]interface{} { - if e.Context != nil { - return e.Context.GetExtensions() - } - return map[string]interface{}(nil) -} diff --git a/vendor/github.com/cloudevents/sdk-go/v2/event/event_validation.go b/vendor/github.com/cloudevents/sdk-go/v2/event/event_validation.go deleted file mode 100644 index b5759fa4..00000000 --- a/vendor/github.com/cloudevents/sdk-go/v2/event/event_validation.go +++ /dev/null @@ -1,45 +0,0 @@ -package event - -import ( - "fmt" - "strings" -) - -type ValidationError map[string]error - -func (e ValidationError) Error() string { - b := strings.Builder{} - for k, v := range e { - b.WriteString(k) - b.WriteString(": ") - b.WriteString(v.Error()) - b.WriteRune('\n') - } - return b.String() -} - -// Validate performs a spec based validation on this event. -// Validation is dependent on the spec version specified in the event context. -func (e Event) Validate() error { - if e.Context == nil { - return ValidationError{"specversion": fmt.Errorf("missing Event.Context")} - } - - errs := map[string]error{} - if e.FieldErrors != nil { - for k, v := range e.FieldErrors { - errs[k] = v - } - } - - if fieldErrors := e.Context.Validate(); fieldErrors != nil { - for k, v := range fieldErrors { - errs[k] = v - } - } - - if len(errs) > 0 { - return ValidationError(errs) - } - return nil -} diff --git a/vendor/github.com/cloudevents/sdk-go/v2/event/event_writer.go b/vendor/github.com/cloudevents/sdk-go/v2/event/event_writer.go deleted file mode 100644 index 00018cbd..00000000 --- a/vendor/github.com/cloudevents/sdk-go/v2/event/event_writer.go +++ /dev/null @@ -1,112 +0,0 @@ -package event - -import ( - "fmt" - "time" -) - -var _ EventWriter = (*Event)(nil) - -// SetSpecVersion implements EventWriter.SetSpecVersion -func (e *Event) SetSpecVersion(v string) { - switch v { - case CloudEventsVersionV03: - if e.Context == nil { - e.Context = &EventContextV03{} - } else { - e.Context = e.Context.AsV03() - } - case CloudEventsVersionV1: - if e.Context == nil { - e.Context = &EventContextV1{} - } else { - e.Context = e.Context.AsV1() - } - default: - e.fieldError("specversion", fmt.Errorf("a valid spec version is required: [%s, %s]", - CloudEventsVersionV03, CloudEventsVersionV1)) - return - } - e.fieldOK("specversion") -} - -// SetType implements EventWriter.SetType -func (e *Event) SetType(t string) { - if err := e.Context.SetType(t); err != nil { - e.fieldError("type", err) - } else { - e.fieldOK("type") - } -} - -// SetSource implements EventWriter.SetSource -func (e *Event) SetSource(s string) { - if err := e.Context.SetSource(s); err != nil { - e.fieldError("source", err) - } else { - e.fieldOK("source") - } -} - -// SetSubject implements EventWriter.SetSubject -func (e *Event) SetSubject(s string) { - if err := e.Context.SetSubject(s); err != nil { - e.fieldError("subject", err) - } else { - e.fieldOK("subject") - } -} - -// SetID implements EventWriter.SetID -func (e *Event) SetID(id string) { - if err := e.Context.SetID(id); err != nil { - e.fieldError("id", err) - } else { - e.fieldOK("id") - } -} - -// SetTime implements EventWriter.SetTime -func (e *Event) SetTime(t time.Time) { - if err := e.Context.SetTime(t); err != nil { - e.fieldError("time", err) - } else { - e.fieldOK("time") - } -} - -// SetDataSchema implements EventWriter.SetDataSchema -func (e *Event) SetDataSchema(s string) { - if err := e.Context.SetDataSchema(s); err != nil { - e.fieldError("dataschema", err) - } else { - e.fieldOK("dataschema") - } -} - -// SetDataContentType implements EventWriter.SetDataContentType -func (e *Event) SetDataContentType(ct string) { - if err := e.Context.SetDataContentType(ct); err != nil { - e.fieldError("datacontenttype", err) - } else { - e.fieldOK("datacontenttype") - } -} - -// SetDataContentEncoding is deprecated. Implements EventWriter.SetDataContentEncoding. -func (e *Event) SetDataContentEncoding(enc string) { - if err := e.Context.DeprecatedSetDataContentEncoding(enc); err != nil { - e.fieldError("datacontentencoding", err) - } else { - e.fieldOK("datacontentencoding") - } -} - -// SetExtension implements EventWriter.SetExtension -func (e *Event) SetExtension(name string, obj interface{}) { - if err := e.Context.SetExtension(name, obj); err != nil { - e.fieldError("extension:"+name, err) - } else { - e.fieldOK("extension:" + name) - } -} diff --git a/vendor/github.com/cloudevents/sdk-go/v2/event/eventcontext.go b/vendor/github.com/cloudevents/sdk-go/v2/event/eventcontext.go deleted file mode 100644 index 2d061121..00000000 --- a/vendor/github.com/cloudevents/sdk-go/v2/event/eventcontext.go +++ /dev/null @@ -1,120 +0,0 @@ -package event - -import "time" - -// EventContextReader are the methods required to be a reader of context -// attributes. -type EventContextReader interface { - // GetSpecVersion returns the native CloudEvents Spec version of the event - // context. - GetSpecVersion() string - // GetType returns the CloudEvents type from the context. - GetType() string - // GetSource returns the CloudEvents source from the context. - GetSource() string - // GetSubject returns the CloudEvents subject from the context. - GetSubject() string - // GetID returns the CloudEvents ID from the context. - GetID() string - // GetTime returns the CloudEvents creation time from the context. - GetTime() time.Time - // GetDataSchema returns the CloudEvents schema URL (if any) from the - // context. - GetDataSchema() string - // GetDataContentType returns content type on the context. - GetDataContentType() string - // DeprecatedGetDataContentEncoding returns content encoding on the context. - DeprecatedGetDataContentEncoding() string - - // GetDataMediaType returns the MIME media type for encoded data, which is - // needed by both encoding and decoding. This is a processed form of - // GetDataContentType and it may return an error. - GetDataMediaType() (string, error) - - // DEPRECATED: Access extensions directly via the GetExtensions() - // For example replace this: - // - // var i int - // err := ec.ExtensionAs("foo", &i) - // - // With this: - // - // i, err := types.ToInteger(ec.GetExtensions["foo"]) - // - ExtensionAs(string, interface{}) error - - // GetExtensions returns the full extensions map. - // - // Extensions use the CloudEvents type system, details in package cloudevents/types. - GetExtensions() map[string]interface{} - - // GetExtension returns the extension associated with with the given key. - // The given key is case insensitive. If the extension can not be found, - // an error will be returned. - GetExtension(string) (interface{}, error) -} - -// EventContextWriter are the methods required to be a writer of context -// attributes. -type EventContextWriter interface { - // SetType sets the type of the context. - SetType(string) error - // SetSource sets the source of the context. - SetSource(string) error - // SetSubject sets the subject of the context. - SetSubject(string) error - // SetID sets the ID of the context. - SetID(string) error - // SetTime sets the time of the context. - SetTime(time time.Time) error - // SetDataSchema sets the schema url of the context. - SetDataSchema(string) error - // SetDataContentType sets the data content type of the context. - SetDataContentType(string) error - // DeprecatedSetDataContentEncoding sets the data context encoding of the context. - DeprecatedSetDataContentEncoding(string) error - - // SetExtension sets the given interface onto the extension attributes - // determined by the provided name. - // - // This function fails in V1 if the name doesn't respect the regex ^[a-zA-Z0-9]+$ - // - // Package ./types documents the types that are allowed as extension values. - SetExtension(string, interface{}) error -} - -// EventContextConverter are the methods that allow for event version -// conversion. -type EventContextConverter interface { - // AsV03 provides a translation from whatever the "native" encoding of the - // CloudEvent was to the equivalent in v0.3 field names, moving fields to or - // from extensions as necessary. - AsV03() *EventContextV03 - - // AsV1 provides a translation from whatever the "native" encoding of the - // CloudEvent was to the equivalent in v1.0 field names, moving fields to or - // from extensions as necessary. - AsV1() *EventContextV1 -} - -// EventContext is conical interface for a CloudEvents Context. -type EventContext interface { - // EventContextConverter allows for conversion between versions. - EventContextConverter - - // EventContextReader adds methods for reading context. - EventContextReader - - // EventContextWriter adds methods for writing to context. - EventContextWriter - - // Validate the event based on the specifics of the CloudEvents spec version - // represented by this event context. - Validate() ValidationError - - // Clone clones the event context. - Clone() EventContext - - // String returns a pretty-printed representation of the EventContext. - String() string -} diff --git a/vendor/github.com/cloudevents/sdk-go/v2/event/eventcontext_v03.go b/vendor/github.com/cloudevents/sdk-go/v2/event/eventcontext_v03.go deleted file mode 100644 index c626311d..00000000 --- a/vendor/github.com/cloudevents/sdk-go/v2/event/eventcontext_v03.go +++ /dev/null @@ -1,307 +0,0 @@ -package event - -import ( - "encoding/json" - "fmt" - "mime" - "sort" - "strings" - - "github.com/cloudevents/sdk-go/v2/types" -) - -const ( - // CloudEventsVersionV03 represents the version 0.3 of the CloudEvents spec. - CloudEventsVersionV03 = "0.3" -) - -// EventContextV03 represents the non-data attributes of a CloudEvents v0.3 -// event. -type EventContextV03 struct { - // Type - The type of the occurrence which has happened. - Type string `json:"type"` - // Source - A URI describing the event producer. - Source types.URIRef `json:"source"` - // Subject - The subject of the event in the context of the event producer - // (identified by `source`). - Subject *string `json:"subject,omitempty"` - // ID of the event; must be non-empty and unique within the scope of the producer. - ID string `json:"id"` - // Time - A Timestamp when the event happened. - Time *types.Timestamp `json:"time,omitempty"` - // DataSchema - A link to the schema that the `data` attribute adheres to. - SchemaURL *types.URIRef `json:"schemaurl,omitempty"` - // GetDataMediaType - A MIME (RFC2046) string describing the media type of `data`. - DataContentType *string `json:"datacontenttype,omitempty"` - // DeprecatedDataContentEncoding describes the content encoding for the `data` attribute. Valid: nil, `Base64`. - DataContentEncoding *string `json:"datacontentencoding,omitempty"` - // Extensions - Additional extension metadata beyond the base spec. - Extensions map[string]interface{} `json:"-"` -} - -// Adhere to EventContext -var _ EventContext = (*EventContextV03)(nil) - -// ExtensionAs implements EventContext.ExtensionAs -func (ec EventContextV03) ExtensionAs(name string, obj interface{}) error { - value, ok := ec.Extensions[name] - if !ok { - return fmt.Errorf("extension %q does not exist", name) - } - - // Try to unmarshal extension if we find it as a RawMessage. - switch v := value.(type) { - case json.RawMessage: - if err := json.Unmarshal(v, obj); err == nil { - // if that worked, return with obj set. - return nil - } - } - // else try as a string ptr. - - // Only support *string for now. - switch v := obj.(type) { - case *string: - if valueAsString, ok := value.(string); ok { - *v = valueAsString - return nil - } else { - return fmt.Errorf("invalid type for extension %q", name) - } - default: - return fmt.Errorf("unknown extension type %T", obj) - } -} - -// SetExtension adds the extension 'name' with value 'value' to the CloudEvents context. -func (ec *EventContextV03) SetExtension(name string, value interface{}) error { - if ec.Extensions == nil { - ec.Extensions = make(map[string]interface{}) - } - if value == nil { - delete(ec.Extensions, name) - if len(ec.Extensions) == 0 { - ec.Extensions = nil - } - return nil - } else { - v, err := types.Validate(value) - if err == nil { - ec.Extensions[name] = v - } - return err - } -} - -// Clone implements EventContextConverter.Clone -func (ec EventContextV03) Clone() EventContext { - ec03 := ec.AsV03() - ec03.Source = types.Clone(ec.Source).(types.URIRef) - if ec.Time != nil { - ec03.Time = types.Clone(ec.Time).(*types.Timestamp) - } - if ec.SchemaURL != nil { - ec03.SchemaURL = types.Clone(ec.SchemaURL).(*types.URIRef) - } - ec03.Extensions = ec.cloneExtensions() - return ec03 -} - -func (ec *EventContextV03) cloneExtensions() map[string]interface{} { - old := ec.Extensions - if old == nil { - return nil - } - new := make(map[string]interface{}, len(ec.Extensions)) - for k, v := range old { - new[k] = types.Clone(v) - } - return new -} - -// AsV03 implements EventContextConverter.AsV03 -func (ec EventContextV03) AsV03() *EventContextV03 { - return &ec -} - -// AsV1 implements EventContextConverter.AsV1 -func (ec EventContextV03) AsV1() *EventContextV1 { - ret := EventContextV1{ - ID: ec.ID, - Time: ec.Time, - Type: ec.Type, - DataContentType: ec.DataContentType, - Source: types.URIRef{URL: ec.Source.URL}, - Subject: ec.Subject, - Extensions: make(map[string]interface{}), - } - if ec.SchemaURL != nil { - ret.DataSchema = &types.URI{URL: ec.SchemaURL.URL} - } - - // DataContentEncoding was removed in 1.0, so put it in an extension for 1.0. - if ec.DataContentEncoding != nil { - _ = ret.SetExtension(DataContentEncodingKey, *ec.DataContentEncoding) - } - - if ec.Extensions != nil { - for k, v := range ec.Extensions { - k = strings.ToLower(k) - ret.Extensions[k] = v - } - } - if len(ret.Extensions) == 0 { - ret.Extensions = nil - } - return &ret -} - -// Validate returns errors based on requirements from the CloudEvents spec. -// For more details, see https://github.com/cloudevents/spec/blob/master/spec.md -// As of Feb 26, 2019, commit 17c32ea26baf7714ad027d9917d03d2fff79fc7e -// + https://github.com/cloudevents/spec/pull/387 -> datacontentencoding -// + https://github.com/cloudevents/spec/pull/406 -> subject -func (ec EventContextV03) Validate() ValidationError { - errors := map[string]error{} - - // type - // Type: String - // Constraints: - // REQUIRED - // MUST be a non-empty string - // SHOULD be prefixed with a reverse-DNS name. The prefixed domain dictates the organization which defines the semantics of this event type. - eventType := strings.TrimSpace(ec.Type) - if eventType == "" { - errors["type"] = fmt.Errorf("MUST be a non-empty string") - } - - // source - // Type: URI-reference - // Constraints: - // REQUIRED - source := strings.TrimSpace(ec.Source.String()) - if source == "" { - errors["source"] = fmt.Errorf("REQUIRED") - } - - // subject - // Type: String - // Constraints: - // OPTIONAL - // MUST be a non-empty string - if ec.Subject != nil { - subject := strings.TrimSpace(*ec.Subject) - if subject == "" { - errors["subject"] = fmt.Errorf("if present, MUST be a non-empty string") - } - } - - // id - // Type: String - // Constraints: - // REQUIRED - // MUST be a non-empty string - // MUST be unique within the scope of the producer - id := strings.TrimSpace(ec.ID) - if id == "" { - errors["id"] = fmt.Errorf("MUST be a non-empty string") - - // no way to test "MUST be unique within the scope of the producer" - } - - // time - // Type: Timestamp - // Constraints: - // OPTIONAL - // If present, MUST adhere to the format specified in RFC 3339 - // --> no need to test this, no way to set the time without it being valid. - - // schemaurl - // Type: URI - // Constraints: - // OPTIONAL - // If present, MUST adhere to the format specified in RFC 3986 - if ec.SchemaURL != nil { - schemaURL := strings.TrimSpace(ec.SchemaURL.String()) - // empty string is not RFC 3986 compatible. - if schemaURL == "" { - errors["schemaurl"] = fmt.Errorf("if present, MUST adhere to the format specified in RFC 3986") - } - } - - // datacontenttype - // Type: String per RFC 2046 - // Constraints: - // OPTIONAL - // If present, MUST adhere to the format specified in RFC 2046 - if ec.DataContentType != nil { - dataContentType := strings.TrimSpace(*ec.DataContentType) - if dataContentType == "" { - errors["datacontenttype"] = fmt.Errorf("if present, MUST adhere to the format specified in RFC 2046") - } else { - _, _, err := mime.ParseMediaType(dataContentType) - if err != nil { - errors["datacontenttype"] = fmt.Errorf("if present, MUST adhere to the format specified in RFC 2046") - } - } - } - - // datacontentencoding - // Type: String per RFC 2045 Section 6.1 - // Constraints: - // The attribute MUST be set if the data attribute contains string-encoded binary data. - // Otherwise the attribute MUST NOT be set. - // If present, MUST adhere to RFC 2045 Section 6.1 - if ec.DataContentEncoding != nil { - dataContentEncoding := strings.ToLower(strings.TrimSpace(*ec.DataContentEncoding)) - if dataContentEncoding != Base64 { - errors["datacontentencoding"] = fmt.Errorf("if present, MUST adhere to RFC 2045 Section 6.1") - } - } - - if len(errors) > 0 { - return errors - } - return nil -} - -// String returns a pretty-printed representation of the EventContext. -func (ec EventContextV03) String() string { - b := strings.Builder{} - - b.WriteString("Context Attributes,\n") - - b.WriteString(" specversion: " + CloudEventsVersionV03 + "\n") - b.WriteString(" type: " + ec.Type + "\n") - b.WriteString(" source: " + ec.Source.String() + "\n") - if ec.Subject != nil { - b.WriteString(" subject: " + *ec.Subject + "\n") - } - b.WriteString(" id: " + ec.ID + "\n") - if ec.Time != nil { - b.WriteString(" time: " + ec.Time.String() + "\n") - } - if ec.SchemaURL != nil { - b.WriteString(" schemaurl: " + ec.SchemaURL.String() + "\n") - } - if ec.DataContentType != nil { - b.WriteString(" datacontenttype: " + *ec.DataContentType + "\n") - } - if ec.DataContentEncoding != nil { - b.WriteString(" datacontentencoding: " + *ec.DataContentEncoding + "\n") - } - - if ec.Extensions != nil && len(ec.Extensions) > 0 { - b.WriteString("Extensions,\n") - keys := make([]string, 0, len(ec.Extensions)) - for k := range ec.Extensions { - keys = append(keys, k) - } - sort.Strings(keys) - for _, key := range keys { - b.WriteString(fmt.Sprintf(" %s: %v\n", key, ec.Extensions[key])) - } - } - - return b.String() -} diff --git a/vendor/github.com/cloudevents/sdk-go/v2/event/eventcontext_v03_reader.go b/vendor/github.com/cloudevents/sdk-go/v2/event/eventcontext_v03_reader.go deleted file mode 100644 index 8e6eec5c..00000000 --- a/vendor/github.com/cloudevents/sdk-go/v2/event/eventcontext_v03_reader.go +++ /dev/null @@ -1,93 +0,0 @@ -package event - -import ( - "fmt" - "mime" - "time" -) - -// GetSpecVersion implements EventContextReader.GetSpecVersion -func (ec EventContextV03) GetSpecVersion() string { - return CloudEventsVersionV03 -} - -// GetDataContentType implements EventContextReader.GetDataContentType -func (ec EventContextV03) GetDataContentType() string { - if ec.DataContentType != nil { - return *ec.DataContentType - } - return "" -} - -// GetDataMediaType implements EventContextReader.GetDataMediaType -func (ec EventContextV03) GetDataMediaType() (string, error) { - if ec.DataContentType != nil { - mediaType, _, err := mime.ParseMediaType(*ec.DataContentType) - if err != nil { - return "", err - } - return mediaType, nil - } - return "", nil -} - -// GetType implements EventContextReader.GetType -func (ec EventContextV03) GetType() string { - return ec.Type -} - -// GetSource implements EventContextReader.GetSource -func (ec EventContextV03) GetSource() string { - return ec.Source.String() -} - -// GetSubject implements EventContextReader.GetSubject -func (ec EventContextV03) GetSubject() string { - if ec.Subject != nil { - return *ec.Subject - } - return "" -} - -// GetTime implements EventContextReader.GetTime -func (ec EventContextV03) GetTime() time.Time { - if ec.Time != nil { - return ec.Time.Time - } - return time.Time{} -} - -// GetID implements EventContextReader.GetID -func (ec EventContextV03) GetID() string { - return ec.ID -} - -// GetDataSchema implements EventContextReader.GetDataSchema -func (ec EventContextV03) GetDataSchema() string { - if ec.SchemaURL != nil { - return ec.SchemaURL.String() - } - return "" -} - -// DeprecatedGetDataContentEncoding implements EventContextReader.DeprecatedGetDataContentEncoding -func (ec EventContextV03) DeprecatedGetDataContentEncoding() string { - if ec.DataContentEncoding != nil { - return *ec.DataContentEncoding - } - return "" -} - -// GetExtensions implements EventContextReader.GetExtensions -func (ec EventContextV03) GetExtensions() map[string]interface{} { - return ec.Extensions -} - -// GetExtension implements EventContextReader.GetExtension -func (ec EventContextV03) GetExtension(key string) (interface{}, error) { - v, ok := caseInsensitiveSearch(key, ec.Extensions) - if !ok { - return "", fmt.Errorf("%q not found", key) - } - return v, nil -} diff --git a/vendor/github.com/cloudevents/sdk-go/v2/event/eventcontext_v03_writer.go b/vendor/github.com/cloudevents/sdk-go/v2/event/eventcontext_v03_writer.go deleted file mode 100644 index 94748c67..00000000 --- a/vendor/github.com/cloudevents/sdk-go/v2/event/eventcontext_v03_writer.go +++ /dev/null @@ -1,98 +0,0 @@ -package event - -import ( - "errors" - "net/url" - "strings" - "time" - - "github.com/cloudevents/sdk-go/v2/types" -) - -// Adhere to EventContextWriter -var _ EventContextWriter = (*EventContextV03)(nil) - -// SetDataContentType implements EventContextWriter.SetDataContentType -func (ec *EventContextV03) SetDataContentType(ct string) error { - ct = strings.TrimSpace(ct) - if ct == "" { - ec.DataContentType = nil - } else { - ec.DataContentType = &ct - } - return nil -} - -// SetType implements EventContextWriter.SetType -func (ec *EventContextV03) SetType(t string) error { - t = strings.TrimSpace(t) - ec.Type = t - return nil -} - -// SetSource implements EventContextWriter.SetSource -func (ec *EventContextV03) SetSource(u string) error { - pu, err := url.Parse(u) - if err != nil { - return err - } - ec.Source = types.URIRef{URL: *pu} - return nil -} - -// SetSubject implements EventContextWriter.SetSubject -func (ec *EventContextV03) SetSubject(s string) error { - s = strings.TrimSpace(s) - if s == "" { - ec.Subject = nil - } else { - ec.Subject = &s - } - return nil -} - -// SetID implements EventContextWriter.SetID -func (ec *EventContextV03) SetID(id string) error { - id = strings.TrimSpace(id) - if id == "" { - return errors.New("id is required to be a non-empty string") - } - ec.ID = id - return nil -} - -// SetTime implements EventContextWriter.SetTime -func (ec *EventContextV03) SetTime(t time.Time) error { - if t.IsZero() { - ec.Time = nil - } else { - ec.Time = &types.Timestamp{Time: t} - } - return nil -} - -// SetDataSchema implements EventContextWriter.SetDataSchema -func (ec *EventContextV03) SetDataSchema(u string) error { - u = strings.TrimSpace(u) - if u == "" { - ec.SchemaURL = nil - return nil - } - pu, err := url.Parse(u) - if err != nil { - return err - } - ec.SchemaURL = &types.URIRef{URL: *pu} - return nil -} - -// DeprecatedSetDataContentEncoding implements EventContextWriter.DeprecatedSetDataContentEncoding -func (ec *EventContextV03) DeprecatedSetDataContentEncoding(e string) error { - e = strings.ToLower(strings.TrimSpace(e)) - if e == "" { - ec.DataContentEncoding = nil - } else { - ec.DataContentEncoding = &e - } - return nil -} diff --git a/vendor/github.com/cloudevents/sdk-go/v2/event/eventcontext_v1.go b/vendor/github.com/cloudevents/sdk-go/v2/event/eventcontext_v1.go deleted file mode 100644 index f7e09ed6..00000000 --- a/vendor/github.com/cloudevents/sdk-go/v2/event/eventcontext_v1.go +++ /dev/null @@ -1,297 +0,0 @@ -package event - -import ( - "errors" - "fmt" - "mime" - "sort" - "strings" - - "github.com/cloudevents/sdk-go/v2/types" -) - -// WIP: AS OF SEP 20, 2019 - -const ( - // CloudEventsVersionV1 represents the version 1.0 of the CloudEvents spec. - CloudEventsVersionV1 = "1.0" -) - -// EventContextV1 represents the non-data attributes of a CloudEvents v1.0 -// event. -type EventContextV1 struct { - // ID of the event; must be non-empty and unique within the scope of the producer. - // +required - ID string `json:"id"` - // Source - A URI describing the event producer. - // +required - Source types.URIRef `json:"source"` - // Type - The type of the occurrence which has happened. - // +required - Type string `json:"type"` - - // DataContentType - A MIME (RFC2046) string describing the media type of `data`. - // +optional - DataContentType *string `json:"datacontenttype,omitempty"` - // Subject - The subject of the event in the context of the event producer - // (identified by `source`). - // +optional - Subject *string `json:"subject,omitempty"` - // Time - A Timestamp when the event happened. - // +optional - Time *types.Timestamp `json:"time,omitempty"` - // DataSchema - A link to the schema that the `data` attribute adheres to. - // +optional - DataSchema *types.URI `json:"dataschema,omitempty"` - - // Extensions - Additional extension metadata beyond the base spec. - // +optional - Extensions map[string]interface{} `json:"-"` -} - -// Adhere to EventContext -var _ EventContext = (*EventContextV1)(nil) - -// ExtensionAs implements EventContext.ExtensionAs -func (ec EventContextV1) ExtensionAs(name string, obj interface{}) error { - name = strings.ToLower(name) - value, ok := ec.Extensions[name] - if !ok { - return fmt.Errorf("extension %q does not exist", name) - } - - // Only support *string for now. - if v, ok := obj.(*string); ok { - if *v, ok = value.(string); ok { - return nil - } - } - return fmt.Errorf("unknown extension type %T", obj) -} - -// SetExtension adds the extension 'name' with value 'value' to the CloudEvents context. -// This function fails if the name doesn't respect the regex ^[a-zA-Z0-9]+$ -func (ec *EventContextV1) SetExtension(name string, value interface{}) error { - if !IsAlphaNumeric(name) { - return errors.New("bad key, CloudEvents attribute names MUST consist of lower-case letters ('a' to 'z') or digits ('0' to '9') from the ASCII character set") - } - - name = strings.ToLower(name) - if ec.Extensions == nil { - ec.Extensions = make(map[string]interface{}) - } - if value == nil { - delete(ec.Extensions, name) - if len(ec.Extensions) == 0 { - ec.Extensions = nil - } - return nil - } else { - v, err := types.Validate(value) // Ensure it's a legal CE attribute value - if err == nil { - ec.Extensions[name] = v - } - return err - } -} - -// Clone implements EventContextConverter.Clone -func (ec EventContextV1) Clone() EventContext { - ec1 := ec.AsV1() - ec1.Source = types.Clone(ec.Source).(types.URIRef) - if ec.Time != nil { - ec1.Time = types.Clone(ec.Time).(*types.Timestamp) - } - if ec.DataSchema != nil { - ec1.DataSchema = types.Clone(ec.DataSchema).(*types.URI) - } - ec1.Extensions = ec.cloneExtensions() - return ec1 -} - -func (ec *EventContextV1) cloneExtensions() map[string]interface{} { - old := ec.Extensions - if old == nil { - return nil - } - new := make(map[string]interface{}, len(ec.Extensions)) - for k, v := range old { - new[k] = types.Clone(v) - } - return new -} - -// AsV03 implements EventContextConverter.AsV03 -func (ec EventContextV1) AsV03() *EventContextV03 { - ret := EventContextV03{ - ID: ec.ID, - Time: ec.Time, - Type: ec.Type, - DataContentType: ec.DataContentType, - Source: types.URIRef{URL: ec.Source.URL}, - Subject: ec.Subject, - Extensions: make(map[string]interface{}), - } - - if ec.DataSchema != nil { - ret.SchemaURL = &types.URIRef{URL: ec.DataSchema.URL} - } - - if ec.Extensions != nil { - for k, v := range ec.Extensions { - k = strings.ToLower(k) - // DeprecatedDataContentEncoding was introduced in 0.3, removed in 1.0 - if strings.EqualFold(k, DataContentEncodingKey) { - etv, ok := v.(string) - if ok && etv != "" { - ret.DataContentEncoding = &etv - } - continue - } - ret.Extensions[k] = v - } - } - if len(ret.Extensions) == 0 { - ret.Extensions = nil - } - return &ret -} - -// AsV1 implements EventContextConverter.AsV1 -func (ec EventContextV1) AsV1() *EventContextV1 { - return &ec -} - -// Validate returns errors based on requirements from the CloudEvents spec. -// For more details, see https://github.com/cloudevents/spec/blob/v1.0/spec.md. -func (ec EventContextV1) Validate() ValidationError { - errors := map[string]error{} - - // id - // Type: String - // Constraints: - // REQUIRED - // MUST be a non-empty string - // MUST be unique within the scope of the producer - id := strings.TrimSpace(ec.ID) - if id == "" { - errors["id"] = fmt.Errorf("MUST be a non-empty string") - // no way to test "MUST be unique within the scope of the producer" - } - - // source - // Type: URI-reference - // Constraints: - // REQUIRED - // MUST be a non-empty URI-reference - // An absolute URI is RECOMMENDED - source := strings.TrimSpace(ec.Source.String()) - if source == "" { - errors["source"] = fmt.Errorf("REQUIRED") - } - - // type - // Type: String - // Constraints: - // REQUIRED - // MUST be a non-empty string - // SHOULD be prefixed with a reverse-DNS name. The prefixed domain dictates the organization which defines the semantics of this event type. - eventType := strings.TrimSpace(ec.Type) - if eventType == "" { - errors["type"] = fmt.Errorf("MUST be a non-empty string") - } - - // The following attributes are optional but still have validation. - - // datacontenttype - // Type: String per RFC 2046 - // Constraints: - // OPTIONAL - // If present, MUST adhere to the format specified in RFC 2046 - if ec.DataContentType != nil { - dataContentType := strings.TrimSpace(*ec.DataContentType) - if dataContentType == "" { - errors["datacontenttype"] = fmt.Errorf("if present, MUST adhere to the format specified in RFC 2046") - } else { - _, _, err := mime.ParseMediaType(dataContentType) - if err != nil { - errors["datacontenttype"] = fmt.Errorf("failed to parse RFC 2046 media type %w", err) - } - } - } - - // dataschema - // Type: URI - // Constraints: - // OPTIONAL - // If present, MUST adhere to the format specified in RFC 3986 - if ec.DataSchema != nil { - dataSchema := strings.TrimSpace(ec.DataSchema.String()) - // empty string is not RFC 3986 compatible. - if dataSchema == "" { - errors["dataschema"] = fmt.Errorf("if present, MUST adhere to the format specified in RFC 3986") - } - } - - // subject - // Type: String - // Constraints: - // OPTIONAL - // MUST be a non-empty string - if ec.Subject != nil { - subject := strings.TrimSpace(*ec.Subject) - if subject == "" { - errors["subject"] = fmt.Errorf("if present, MUST be a non-empty string") - } - } - - // time - // Type: Timestamp - // Constraints: - // OPTIONAL - // If present, MUST adhere to the format specified in RFC 3339 - // --> no need to test this, no way to set the time without it being valid. - - if len(errors) > 0 { - return errors - } - return nil -} - -// String returns a pretty-printed representation of the EventContext. -func (ec EventContextV1) String() string { - b := strings.Builder{} - - b.WriteString("Context Attributes,\n") - - b.WriteString(" specversion: " + CloudEventsVersionV1 + "\n") - b.WriteString(" type: " + ec.Type + "\n") - b.WriteString(" source: " + ec.Source.String() + "\n") - if ec.Subject != nil { - b.WriteString(" subject: " + *ec.Subject + "\n") - } - b.WriteString(" id: " + ec.ID + "\n") - if ec.Time != nil { - b.WriteString(" time: " + ec.Time.String() + "\n") - } - if ec.DataSchema != nil { - b.WriteString(" dataschema: " + ec.DataSchema.String() + "\n") - } - if ec.DataContentType != nil { - b.WriteString(" datacontenttype: " + *ec.DataContentType + "\n") - } - - if ec.Extensions != nil && len(ec.Extensions) > 0 { - b.WriteString("Extensions,\n") - keys := make([]string, 0, len(ec.Extensions)) - for k := range ec.Extensions { - keys = append(keys, k) - } - sort.Strings(keys) - for _, key := range keys { - b.WriteString(fmt.Sprintf(" %s: %v\n", key, ec.Extensions[key])) - } - } - - return b.String() -} diff --git a/vendor/github.com/cloudevents/sdk-go/v2/event/eventcontext_v1_reader.go b/vendor/github.com/cloudevents/sdk-go/v2/event/eventcontext_v1_reader.go deleted file mode 100644 index 64f1a919..00000000 --- a/vendor/github.com/cloudevents/sdk-go/v2/event/eventcontext_v1_reader.go +++ /dev/null @@ -1,98 +0,0 @@ -package event - -import ( - "fmt" - "mime" - "time" -) - -// GetSpecVersion implements EventContextReader.GetSpecVersion -func (ec EventContextV1) GetSpecVersion() string { - return CloudEventsVersionV1 -} - -// GetDataContentType implements EventContextReader.GetDataContentType -func (ec EventContextV1) GetDataContentType() string { - if ec.DataContentType != nil { - return *ec.DataContentType - } - return "" -} - -// GetDataMediaType implements EventContextReader.GetDataMediaType -func (ec EventContextV1) GetDataMediaType() (string, error) { - if ec.DataContentType != nil { - mediaType, _, err := mime.ParseMediaType(*ec.DataContentType) - if err != nil { - return "", err - } - return mediaType, nil - } - return "", nil -} - -// GetType implements EventContextReader.GetType -func (ec EventContextV1) GetType() string { - return ec.Type -} - -// GetSource implements EventContextReader.GetSource -func (ec EventContextV1) GetSource() string { - return ec.Source.String() -} - -// GetSubject implements EventContextReader.GetSubject -func (ec EventContextV1) GetSubject() string { - if ec.Subject != nil { - return *ec.Subject - } - return "" -} - -// GetTime implements EventContextReader.GetTime -func (ec EventContextV1) GetTime() time.Time { - if ec.Time != nil { - return ec.Time.Time - } - return time.Time{} -} - -// GetID implements EventContextReader.GetID -func (ec EventContextV1) GetID() string { - return ec.ID -} - -// GetDataSchema implements EventContextReader.GetDataSchema -func (ec EventContextV1) GetDataSchema() string { - if ec.DataSchema != nil { - return ec.DataSchema.String() - } - return "" -} - -// DeprecatedGetDataContentEncoding implements EventContextReader.DeprecatedGetDataContentEncoding -func (ec EventContextV1) DeprecatedGetDataContentEncoding() string { - return "" -} - -// GetExtensions implements EventContextReader.GetExtensions -func (ec EventContextV1) GetExtensions() map[string]interface{} { - if len(ec.Extensions) == 0 { - return nil - } - // For now, convert the extensions of v1.0 to the pre-v1.0 style. - ext := make(map[string]interface{}, len(ec.Extensions)) - for k, v := range ec.Extensions { - ext[k] = v - } - return ext -} - -// GetExtension implements EventContextReader.GetExtension -func (ec EventContextV1) GetExtension(key string) (interface{}, error) { - v, ok := caseInsensitiveSearch(key, ec.Extensions) - if !ok { - return "", fmt.Errorf("%q not found", key) - } - return v, nil -} diff --git a/vendor/github.com/cloudevents/sdk-go/v2/event/eventcontext_v1_writer.go b/vendor/github.com/cloudevents/sdk-go/v2/event/eventcontext_v1_writer.go deleted file mode 100644 index 1ec29e65..00000000 --- a/vendor/github.com/cloudevents/sdk-go/v2/event/eventcontext_v1_writer.go +++ /dev/null @@ -1,92 +0,0 @@ -package event - -import ( - "errors" - "net/url" - "strings" - "time" - - "github.com/cloudevents/sdk-go/v2/types" -) - -// Adhere to EventContextWriter -var _ EventContextWriter = (*EventContextV1)(nil) - -// SetDataContentType implements EventContextWriter.SetDataContentType -func (ec *EventContextV1) SetDataContentType(ct string) error { - ct = strings.TrimSpace(ct) - if ct == "" { - ec.DataContentType = nil - } else { - ec.DataContentType = &ct - } - return nil -} - -// SetType implements EventContextWriter.SetType -func (ec *EventContextV1) SetType(t string) error { - t = strings.TrimSpace(t) - ec.Type = t - return nil -} - -// SetSource implements EventContextWriter.SetSource -func (ec *EventContextV1) SetSource(u string) error { - pu, err := url.Parse(u) - if err != nil { - return err - } - ec.Source = types.URIRef{URL: *pu} - return nil -} - -// SetSubject implements EventContextWriter.SetSubject -func (ec *EventContextV1) SetSubject(s string) error { - s = strings.TrimSpace(s) - if s == "" { - ec.Subject = nil - } else { - ec.Subject = &s - } - return nil -} - -// SetID implements EventContextWriter.SetID -func (ec *EventContextV1) SetID(id string) error { - id = strings.TrimSpace(id) - if id == "" { - return errors.New("id is required to be a non-empty string") - } - ec.ID = id - return nil -} - -// SetTime implements EventContextWriter.SetTime -func (ec *EventContextV1) SetTime(t time.Time) error { - if t.IsZero() { - ec.Time = nil - } else { - ec.Time = &types.Timestamp{Time: t} - } - return nil -} - -// SetDataSchema implements EventContextWriter.SetDataSchema -func (ec *EventContextV1) SetDataSchema(u string) error { - u = strings.TrimSpace(u) - if u == "" { - ec.DataSchema = nil - return nil - } - pu, err := url.Parse(u) - if err != nil { - return err - } - ec.DataSchema = &types.URI{URL: *pu} - return nil -} - -// DeprecatedSetDataContentEncoding implements EventContextWriter.DeprecatedSetDataContentEncoding -func (ec *EventContextV1) DeprecatedSetDataContentEncoding(e string) error { - return errors.New("deprecated: SetDataContentEncoding is not supported in v1.0 of CloudEvents") -} diff --git a/vendor/github.com/cloudevents/sdk-go/v2/event/extensions.go b/vendor/github.com/cloudevents/sdk-go/v2/event/extensions.go deleted file mode 100644 index 4a202e5e..00000000 --- a/vendor/github.com/cloudevents/sdk-go/v2/event/extensions.go +++ /dev/null @@ -1,24 +0,0 @@ -package event - -import ( - "regexp" - "strings" -) - -const ( - // DataContentEncodingKey is the key to DeprecatedDataContentEncoding for versions that do not support data content encoding - // directly. - DataContentEncodingKey = "datacontentencoding" -) - -func caseInsensitiveSearch(key string, space map[string]interface{}) (interface{}, bool) { - lkey := strings.ToLower(key) - for k, v := range space { - if strings.EqualFold(lkey, strings.ToLower(k)) { - return v, true - } - } - return nil, false -} - -var IsAlphaNumeric = regexp.MustCompile(`^[a-zA-Z0-9]+$`).MatchString diff --git a/vendor/github.com/cloudevents/sdk-go/v2/extensions/distributed_tracing_extension.go b/vendor/github.com/cloudevents/sdk-go/v2/extensions/distributed_tracing_extension.go deleted file mode 100644 index 7988b65f..00000000 --- a/vendor/github.com/cloudevents/sdk-go/v2/extensions/distributed_tracing_extension.go +++ /dev/null @@ -1,164 +0,0 @@ -package extensions - -import ( - "context" - "reflect" - "strings" - - "github.com/cloudevents/sdk-go/v2/binding" - "github.com/cloudevents/sdk-go/v2/event" - - "github.com/cloudevents/sdk-go/v2/types" - "github.com/lightstep/tracecontext.go/traceparent" - "github.com/lightstep/tracecontext.go/tracestate" - "go.opencensus.io/trace" - octs "go.opencensus.io/trace/tracestate" -) - -const ( - TraceParentExtension = "traceparent" - TraceStateExtension = "tracestate" -) - -// DistributedTracingExtension represents the extension for cloudevents context -type DistributedTracingExtension struct { - TraceParent string `json:"traceparent"` - TraceState string `json:"tracestate"` -} - -// AddTracingAttributes adds the tracing attributes traceparent and tracestate to the cloudevents context -func (d DistributedTracingExtension) AddTracingAttributes(e event.EventWriter) { - if d.TraceParent != "" { - value := reflect.ValueOf(d) - typeOf := value.Type() - - for i := 0; i < value.NumField(); i++ { - k := strings.ToLower(typeOf.Field(i).Name) - v := value.Field(i).Interface() - if k == TraceStateExtension && v == "" { - continue - } - e.SetExtension(k, v) - } - } -} - -func GetDistributedTracingExtension(event event.Event) (DistributedTracingExtension, bool) { - if tp, ok := event.Extensions()[TraceParentExtension]; ok { - if tpStr, err := types.ToString(tp); err == nil { - var tsStr string - if ts, ok := event.Extensions()[TraceStateExtension]; ok { - tsStr, _ = types.ToString(ts) - } - return DistributedTracingExtension{TraceParent: tpStr, TraceState: tsStr}, true - } - } - return DistributedTracingExtension{}, false -} - -func (d *DistributedTracingExtension) ReadTransformer() binding.TransformerFunc { - return func(reader binding.MessageMetadataReader, writer binding.MessageMetadataWriter) error { - tp := reader.GetExtension(TraceParentExtension) - if tp != nil { - tpFormatted, err := types.Format(tp) - if err != nil { - return err - } - d.TraceParent = tpFormatted - } - ts := reader.GetExtension(TraceStateExtension) - if ts != nil { - tsFormatted, err := types.Format(ts) - if err != nil { - return err - } - d.TraceState = tsFormatted - } - return nil - } -} - -func (d *DistributedTracingExtension) WriteTransformer() binding.TransformerFunc { - return func(reader binding.MessageMetadataReader, writer binding.MessageMetadataWriter) error { - err := writer.SetExtension(TraceParentExtension, d.TraceParent) - if err != nil { - return nil - } - if d.TraceState != "" { - return writer.SetExtension(TraceStateExtension, d.TraceState) - } - return nil - } -} - -// FromSpanContext populates DistributedTracingExtension from a SpanContext. -func FromSpanContext(sc trace.SpanContext) DistributedTracingExtension { - tp := traceparent.TraceParent{ - TraceID: sc.TraceID, - SpanID: sc.SpanID, - Flags: traceparent.Flags{ - Recorded: sc.IsSampled(), - }, - } - - entries := make([]string, 0, len(sc.Tracestate.Entries())) - for _, entry := range sc.Tracestate.Entries() { - entries = append(entries, strings.Join([]string{entry.Key, entry.Value}, "=")) - } - - return DistributedTracingExtension{ - TraceParent: tp.String(), - TraceState: strings.Join(entries, ","), - } -} - -// ToSpanContext creates a SpanContext from a DistributedTracingExtension instance. -func (d DistributedTracingExtension) ToSpanContext() (trace.SpanContext, error) { - tp, err := traceparent.ParseString(d.TraceParent) - if err != nil { - return trace.SpanContext{}, err - } - sc := trace.SpanContext{ - TraceID: tp.TraceID, - SpanID: tp.SpanID, - } - if tp.Flags.Recorded { - sc.TraceOptions |= 1 - } - - if ts, err := tracestate.ParseString(d.TraceState); err == nil { - entries := make([]octs.Entry, 0, len(ts)) - for _, member := range ts { - var key string - if member.Tenant != "" { - // Due to github.com/lightstep/tracecontext.go/issues/6, - // the meaning of Vendor and Tenant are swapped here. - key = member.Vendor + "@" + member.Tenant - } else { - key = member.Vendor - } - entries = append(entries, octs.Entry{Key: key, Value: member.Value}) - } - sc.Tracestate, _ = octs.New(nil, entries...) - } - - return sc, nil -} - -func (d DistributedTracingExtension) StartChildSpan(ctx context.Context, name string, opts ...trace.StartOption) (context.Context, *trace.Span) { - if sc, err := d.ToSpanContext(); err == nil { - tSpan := trace.FromContext(ctx) - ctx, span := trace.StartSpanWithRemoteParent(ctx, name, sc, opts...) - if tSpan != nil { - // Add link to the previous in-process trace. - tsc := tSpan.SpanContext() - span.AddLink(trace.Link{ - TraceID: tsc.TraceID, - SpanID: tsc.SpanID, - Type: trace.LinkTypeParent, - }) - } - return ctx, span - } - return ctx, nil -} diff --git a/vendor/github.com/cloudevents/sdk-go/v2/extensions/doc.go b/vendor/github.com/cloudevents/sdk-go/v2/extensions/doc.go deleted file mode 100644 index d7126904..00000000 --- a/vendor/github.com/cloudevents/sdk-go/v2/extensions/doc.go +++ /dev/null @@ -1,2 +0,0 @@ -// Package extensions provides implementations of common event extensions. -package extensions diff --git a/vendor/github.com/cloudevents/sdk-go/v2/go.mod b/vendor/github.com/cloudevents/sdk-go/v2/go.mod deleted file mode 100644 index aff7bd6a..00000000 --- a/vendor/github.com/cloudevents/sdk-go/v2/go.mod +++ /dev/null @@ -1,28 +0,0 @@ -module github.com/cloudevents/sdk-go/v2 - -require ( - github.com/davecgh/go-spew v1.1.1 // indirect - github.com/golang/protobuf v1.3.2 // indirect - github.com/google/go-cmp v0.4.0 - github.com/google/uuid v1.1.1 - github.com/hashicorp/golang-lru v0.5.3 // indirect - github.com/kr/text v0.2.0 // indirect - github.com/lightstep/tracecontext.go v0.0.0-20181129014701-1757c391b1ac - github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect - github.com/onsi/ginkgo v1.10.2 // indirect - github.com/onsi/gomega v1.7.0 // indirect - github.com/pkg/errors v0.9.1 // indirect - github.com/stretchr/testify v1.5.1 - github.com/valyala/bytebufferpool v1.0.0 - go.opencensus.io v0.22.0 - go.uber.org/atomic v1.4.0 // indirect - go.uber.org/multierr v1.1.0 // indirect - go.uber.org/zap v1.10.0 - golang.org/x/net v0.0.0-20200202094626-16171245cfb2 // indirect - golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e - golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5 // indirect - gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect - gopkg.in/yaml.v2 v2.2.8 // indirect -) - -go 1.13 diff --git a/vendor/github.com/cloudevents/sdk-go/v2/go.sum b/vendor/github.com/cloudevents/sdk-go/v2/go.sum deleted file mode 100644 index bfdfa6b6..00000000 --- a/vendor/github.com/cloudevents/sdk-go/v2/go.sum +++ /dev/null @@ -1,115 +0,0 @@ -cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= -github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= -github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= -github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= -github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= -github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= -github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU= -github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.3 h1:YPkqC67at8FYaadspW/6uE0COsBxS2656RLEr8Bppgk= -github.com/hashicorp/golang-lru v0.5.3/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= -github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= -github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= -github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/lightstep/tracecontext.go v0.0.0-20181129014701-1757c391b1ac h1:+2b6iGRJe3hvV/yVXrd41yVEjxuFHxasJqDhkIjS4gk= -github.com/lightstep/tracecontext.go v0.0.0-20181129014701-1757c391b1ac/go.mod h1:Frd2bnT3w5FB5q49ENTfVlztJES+1k/7lyWX2+9gq/M= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= -github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.10.2 h1:uqH7bpe+ERSiDa34FDOF7RikN6RzXgduUF8yarlZp94= -github.com/onsi/ginkgo v1.10.2/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME= -github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= -github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= -github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= -go.opencensus.io v0.22.0 h1:C9hSCOW830chIVkdja34wa6Ky+IzWllkUinR+BtRZd4= -go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= -go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU= -go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI= -go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= -go.uber.org/zap v1.10.0 h1:ORx85nbTijNz8ljznvCMR1ZBIPKFn3jQrag10X2AsuM= -go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjutRKlBEZQ6wTn8ozI/nI= -golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5 h1:LfCXLvNmTYH9kEmVgqbnsWfruoXZIrh4YBgqVHtDvw0= -golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= -golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= -golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.20.1 h1:Hz2g2wirWK7H0qIIhGIqRGTuMwTE8HEKFnDZZ7lm9NU= -google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= -gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= -gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= -gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE= -gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= -gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/vendor/github.com/cloudevents/sdk-go/v2/observability/doc.go b/vendor/github.com/cloudevents/sdk-go/v2/observability/doc.go deleted file mode 100644 index 3067ebe7..00000000 --- a/vendor/github.com/cloudevents/sdk-go/v2/observability/doc.go +++ /dev/null @@ -1,4 +0,0 @@ -/* -Package observability holds metrics and tracing recording implementations. -*/ -package observability diff --git a/vendor/github.com/cloudevents/sdk-go/v2/observability/keys.go b/vendor/github.com/cloudevents/sdk-go/v2/observability/keys.go deleted file mode 100644 index afadddcf..00000000 --- a/vendor/github.com/cloudevents/sdk-go/v2/observability/keys.go +++ /dev/null @@ -1,22 +0,0 @@ -package observability - -import ( - "go.opencensus.io/tag" -) - -var ( - // KeyMethod is the tag used for marking method on a metric. - KeyMethod, _ = tag.NewKey("method") - // KeyResult is the tag used for marking result on a metric. - KeyResult, _ = tag.NewKey("result") -) - -const ( - // ClientSpanName is the key used to start spans from the client. - ClientSpanName = "cloudevents.client" - - // ResultError is a shared result tag value for error. - ResultError = "error" - // ResultOK is a shared result tag value for success. - ResultOK = "success" -) diff --git a/vendor/github.com/cloudevents/sdk-go/v2/observability/observer.go b/vendor/github.com/cloudevents/sdk-go/v2/observability/observer.go deleted file mode 100644 index bedf3e44..00000000 --- a/vendor/github.com/cloudevents/sdk-go/v2/observability/observer.go +++ /dev/null @@ -1,87 +0,0 @@ -package observability - -import ( - "context" - "sync" - "time" - - "go.opencensus.io/stats" - "go.opencensus.io/tag" -) - -// Observable represents the the customization used by the Reporter for a given -// measurement and trace for a single method. -type Observable interface { - MethodName() string - LatencyMs() *stats.Float64Measure -} - -// Reporter represents a running latency counter. When Error or OK are -// called, the latency is calculated. Error or OK are only allowed to -// be called once. -type Reporter interface { - Error() - OK() -} - -type reporter struct { - ctx context.Context - on Observable - start time.Time - once sync.Once -} - -// LatencyTags returns all tags used for Latency measurements. -func LatencyTags() []tag.Key { - return []tag.Key{KeyMethod, KeyResult} -} - -// EnableTracing is deprecated. Tracing is always enabled. -func EnableTracing(enabled bool) {} - -// NewReporter creates and returns a reporter wrapping the provided Observable. -func NewReporter(ctx context.Context, on Observable) (context.Context, Reporter) { - r := &reporter{ - ctx: ctx, - on: on, - start: time.Now(), - } - r.tagMethod() - return ctx, r -} - -func (r *reporter) tagMethod() { - var err error - r.ctx, err = tag.New(r.ctx, tag.Insert(KeyMethod, r.on.MethodName())) - if err != nil { - panic(err) // or ignore? - } -} - -func (r *reporter) record() { - ms := float64(time.Since(r.start) / time.Millisecond) - stats.Record(r.ctx, r.on.LatencyMs().M(ms)) -} - -// Error records the result as an error. -func (r *reporter) Error() { - r.once.Do(func() { - r.result(ResultError) - }) -} - -// OK records the result as a success. -func (r *reporter) OK() { - r.once.Do(func() { - r.result(ResultOK) - }) -} - -func (r *reporter) result(v string) { - var err error - r.ctx, err = tag.New(r.ctx, tag.Insert(KeyResult, v)) - if err != nil { - panic(err) // or ignore? - } - r.record() -} diff --git a/vendor/github.com/cloudevents/sdk-go/v2/protocol/doc.go b/vendor/github.com/cloudevents/sdk-go/v2/protocol/doc.go deleted file mode 100644 index d14bf7f9..00000000 --- a/vendor/github.com/cloudevents/sdk-go/v2/protocol/doc.go +++ /dev/null @@ -1,21 +0,0 @@ -/* -Package protocol defines interfaces to decouple the client package -from protocol implementations. - -Most event sender and receiver applications should not use this -package, they should use the client package. This package is for -infrastructure developers implementing new transports, or intermediary -components like importers, channels or brokers. - -Available protocols: - -* HTTP (using net/http) -* Kafka (using github.com/Shopify/sarama) -* AMQP (using pack.ag/amqp) -* Go Channels -* Nats -* Nats Streaming (stan) -* Google PubSub - -*/ -package protocol diff --git a/vendor/github.com/cloudevents/sdk-go/v2/protocol/error.go b/vendor/github.com/cloudevents/sdk-go/v2/protocol/error.go deleted file mode 100644 index 0c9530d1..00000000 --- a/vendor/github.com/cloudevents/sdk-go/v2/protocol/error.go +++ /dev/null @@ -1,37 +0,0 @@ -package protocol - -import "fmt" - -// ErrTransportMessageConversion is an error produced when the transport -// message can not be converted. -type ErrTransportMessageConversion struct { - fatal bool - handled bool - transport string - message string -} - -// NewErrTransportMessageConversion makes a new ErrTransportMessageConversion. -func NewErrTransportMessageConversion(transport, message string, handled, fatal bool) *ErrTransportMessageConversion { - return &ErrTransportMessageConversion{ - transport: transport, - message: message, - handled: handled, - fatal: fatal, - } -} - -// IsFatal reports if this error should be considered fatal. -func (e *ErrTransportMessageConversion) IsFatal() bool { - return e.fatal -} - -// Handled reports if this error should be considered accepted and no further action. -func (e *ErrTransportMessageConversion) Handled() bool { - return e.handled -} - -// Error implements error.Error -func (e *ErrTransportMessageConversion) Error() string { - return fmt.Sprintf("transport %s failed to convert message: %s", e.transport, e.message) -} diff --git a/vendor/github.com/cloudevents/sdk-go/v2/protocol/http/abuse_protection.go b/vendor/github.com/cloudevents/sdk-go/v2/protocol/http/abuse_protection.go deleted file mode 100644 index eb004101..00000000 --- a/vendor/github.com/cloudevents/sdk-go/v2/protocol/http/abuse_protection.go +++ /dev/null @@ -1,121 +0,0 @@ -package http - -import ( - "context" - cecontext "github.com/cloudevents/sdk-go/v2/context" - "go.uber.org/zap" - "net/http" - "strconv" - "strings" -) - -type WebhookConfig struct { - AllowedMethods []string // defaults to POST - AllowedRate *int - AutoACKCallback bool - AllowedOrigins []string -} - -const ( - DefaultAllowedRate = 1000 -) - -// TODO: implement rate limiting. -// Throttling is indicated by requests being rejected using HTTP status code 429 Too Many Requests. -// TODO: use this if Webhook Request Origin has been turned on. -// Inbound requests should be rejected if Allowed Origins is required by SDK. - -func (p *Protocol) OptionsHandler(rw http.ResponseWriter, req *http.Request) { - if req.Method != http.MethodOptions || p.WebhookConfig == nil { - rw.WriteHeader(http.StatusMethodNotAllowed) - return - } - - headers := make(http.Header) - - // The spec does not say we need to validate the origin, just the request origin. - // After the handshake, we will validate the origin. - if origin, ok := p.ValidateRequestOrigin(req); !ok { - rw.WriteHeader(http.StatusBadRequest) - return - } else { - headers.Set("WebHook-Allowed-Origin", origin) - } - - allowedRateRequired := false - if _, ok := req.Header[http.CanonicalHeaderKey("WebHook-Request-Rate")]; ok { - // must send WebHook-Allowed-Rate - allowedRateRequired = true - } - - if p.WebhookConfig.AllowedRate != nil { - headers.Set("WebHook-Allowed-Rate", strconv.Itoa(*p.WebhookConfig.AllowedRate)) - } else if allowedRateRequired { - headers.Set("WebHook-Allowed-Rate", strconv.Itoa(DefaultAllowedRate)) - } - - if len(p.WebhookConfig.AllowedMethods) > 0 { - headers.Set("Allow", strings.Join(p.WebhookConfig.AllowedMethods, ", ")) - } else { - headers.Set("Allow", http.MethodPost) - } - - cb := req.Header.Get("WebHook-Request-Callback") - if cb != "" { - if p.WebhookConfig.AutoACKCallback { - go func() { - reqAck, err := http.NewRequest(http.MethodPost, cb, nil) - if err != nil { - cecontext.LoggerFrom(req.Context()).Errorw("OPTIONS handler failed to create http request attempting to ack callback.", zap.Error(err), zap.String("callback", cb)) - return - } - - // Write out the headers. - for k := range headers { - reqAck.Header.Set(k, headers.Get(k)) - } - - _, err = http.DefaultClient.Do(reqAck) - if err != nil { - cecontext.LoggerFrom(req.Context()).Errorw("OPTIONS handler failed to ack callback.", zap.Error(err), zap.String("callback", cb)) - return - } - }() - return - } else { - cecontext.LoggerFrom(req.Context()).Infof("ACTION REQUIRED: Please validate web hook request callback: %q", cb) - // TODO: what to do pending https://github.com/cloudevents/spec/issues/617 - return - } - } - - // Write out the headers. - for k := range headers { - rw.Header().Set(k, headers.Get(k)) - } -} - -func (p *Protocol) ValidateRequestOrigin(req *http.Request) (string, bool) { - return p.validateOrigin(req.Header.Get("WebHook-Request-Origin")) -} - -func (p *Protocol) ValidateOrigin(req *http.Request) (string, bool) { - return p.validateOrigin(req.Header.Get("Origin")) -} - -func (p *Protocol) validateOrigin(ro string) (string, bool) { - cecontext.LoggerFrom(context.TODO()).Infow("Validating origin.", zap.String("origin", ro)) - - for _, ao := range p.WebhookConfig.AllowedOrigins { - if ao == "*" { - return ao, true - } - // TODO: it is not clear what the rules for allowed hosts are. - // Need to find docs for this. For now, test for prefix. - if strings.HasPrefix(ro, ao) { - return ao, true - } - } - - return ro, false -} diff --git a/vendor/github.com/cloudevents/sdk-go/v2/protocol/http/doc.go b/vendor/github.com/cloudevents/sdk-go/v2/protocol/http/doc.go deleted file mode 100644 index 5c04b88a..00000000 --- a/vendor/github.com/cloudevents/sdk-go/v2/protocol/http/doc.go +++ /dev/null @@ -1,4 +0,0 @@ -/* -Package http implements an HTTP binding using net/http module -*/ -package http diff --git a/vendor/github.com/cloudevents/sdk-go/v2/protocol/http/headers.go b/vendor/github.com/cloudevents/sdk-go/v2/protocol/http/headers.go deleted file mode 100644 index f60c5044..00000000 --- a/vendor/github.com/cloudevents/sdk-go/v2/protocol/http/headers.go +++ /dev/null @@ -1,33 +0,0 @@ -package http - -import ( - "net/textproto" - "strings" - "unicode" - - "github.com/cloudevents/sdk-go/v2/binding/spec" -) - -var attributeHeadersMapping map[string]string - -func init() { - attributeHeadersMapping = make(map[string]string) - for _, v := range specs.Versions() { - for _, a := range v.Attributes() { - if a.Kind() == spec.DataContentType { - attributeHeadersMapping[a.Name()] = ContentType - } else { - attributeHeadersMapping[a.Name()] = textproto.CanonicalMIMEHeaderKey(prefix + a.Name()) - } - } - } -} - -func extNameToHeaderName(name string) string { - var b strings.Builder - b.Grow(len(name) + len(prefix)) - b.WriteString(prefix) - b.WriteRune(unicode.ToUpper(rune(name[0]))) - b.WriteString(name[1:]) - return b.String() -} diff --git a/vendor/github.com/cloudevents/sdk-go/v2/protocol/http/message.go b/vendor/github.com/cloudevents/sdk-go/v2/protocol/http/message.go deleted file mode 100644 index 451506ce..00000000 --- a/vendor/github.com/cloudevents/sdk-go/v2/protocol/http/message.go +++ /dev/null @@ -1,158 +0,0 @@ -package http - -import ( - "context" - "io" - nethttp "net/http" - "net/textproto" - "strings" - "unicode" - - "github.com/cloudevents/sdk-go/v2/binding" - "github.com/cloudevents/sdk-go/v2/binding/format" - "github.com/cloudevents/sdk-go/v2/binding/spec" -) - -const prefix = "Ce-" - -var specs = spec.WithPrefixMatchExact( - func(s string) string { - if s == "datacontenttype" { - return "Content-Type" - } else { - return textproto.CanonicalMIMEHeaderKey("Ce-" + s) - } - }, - "Ce-", -) - -const ContentType = "Content-Type" -const ContentLength = "Content-Length" - -// Message holds the Header and Body of a HTTP Request or Response. -// The Message instance *must* be constructed from NewMessage function. -// This message *cannot* be read several times. In order to read it more times, buffer it using binding/buffering methods -type Message struct { - Header nethttp.Header - BodyReader io.ReadCloser - OnFinish func(error) error - - format format.Format - version spec.Version -} - -// Check if http.Message implements binding.Message -var _ binding.Message = (*Message)(nil) -var _ binding.MessageMetadataReader = (*Message)(nil) - -// NewMessage returns a binding.Message with header and data. -// The returned binding.Message *cannot* be read several times. In order to read it more times, buffer it using binding/buffering methods -func NewMessage(header nethttp.Header, body io.ReadCloser) *Message { - m := Message{Header: header} - if body != nil { - m.BodyReader = body - } - if m.format = format.Lookup(header.Get(ContentType)); m.format == nil { - m.version = specs.Version(m.Header.Get(specs.PrefixedSpecVersionName())) - } - return &m -} - -// NewMessageFromHttpRequest returns a binding.Message with header and data. -// The returned binding.Message *cannot* be read several times. In order to read it more times, buffer it using binding/buffering methods -func NewMessageFromHttpRequest(req *nethttp.Request) *Message { - if req == nil { - return nil - } - return NewMessage(req.Header, req.Body) -} - -// NewMessageFromHttpResponse returns a binding.Message with header and data. -// The returned binding.Message *cannot* be read several times. In order to read it more times, buffer it using binding/buffering methods -func NewMessageFromHttpResponse(resp *nethttp.Response) *Message { - if resp == nil { - return nil - } - msg := NewMessage(resp.Header, resp.Body) - return msg -} - -func (m *Message) ReadEncoding() binding.Encoding { - if m.version != nil { - return binding.EncodingBinary - } - if m.format != nil { - return binding.EncodingStructured - } - return binding.EncodingUnknown -} - -func (m *Message) ReadStructured(ctx context.Context, encoder binding.StructuredWriter) error { - if m.format == nil { - return binding.ErrNotStructured - } else { - return encoder.SetStructuredEvent(ctx, m.format, m.BodyReader) - } -} - -func (m *Message) ReadBinary(ctx context.Context, encoder binding.BinaryWriter) (err error) { - if m.version == nil { - return binding.ErrNotBinary - } - - for k, v := range m.Header { - attr := m.version.Attribute(k) - if attr != nil { - err = encoder.SetAttribute(attr, v[0]) - } else if strings.HasPrefix(k, prefix) { - // Trim Prefix + To lower - var b strings.Builder - b.Grow(len(k) - len(prefix)) - b.WriteRune(unicode.ToLower(rune(k[len(prefix)]))) - b.WriteString(k[len(prefix)+1:]) - err = encoder.SetExtension(b.String(), v[0]) - } - if err != nil { - return err - } - } - - if m.BodyReader != nil { - err = encoder.SetData(m.BodyReader) - if err != nil { - return err - } - } - - return -} - -func (m *Message) GetAttribute(k spec.Kind) (spec.Attribute, interface{}) { - attr := m.version.AttributeFromKind(k) - if attr != nil { - h := m.Header[attributeHeadersMapping[attr.Name()]] - if h != nil { - return attr, h[0] - } - return attr, nil - } - return nil, nil -} - -func (m *Message) GetExtension(name string) interface{} { - h := m.Header[extNameToHeaderName(name)] - if h != nil { - return h[0] - } - return nil -} - -func (m *Message) Finish(err error) error { - if m.BodyReader != nil { - _ = m.BodyReader.Close() - } - if m.OnFinish != nil { - return m.OnFinish(err) - } - return nil -} diff --git a/vendor/github.com/cloudevents/sdk-go/v2/protocol/http/options.go b/vendor/github.com/cloudevents/sdk-go/v2/protocol/http/options.go deleted file mode 100644 index 71f88fcb..00000000 --- a/vendor/github.com/cloudevents/sdk-go/v2/protocol/http/options.go +++ /dev/null @@ -1,237 +0,0 @@ -package http - -import ( - "fmt" - "net" - nethttp "net/http" - "net/url" - "strings" - "time" -) - -// Option is the function signature required to be considered an http.Option. -type Option func(*Protocol) error - -// WithTarget sets the outbound recipient of cloudevents when using an HTTP -// request. -func WithTarget(targetUrl string) Option { - return func(p *Protocol) error { - if p == nil { - return fmt.Errorf("http target option can not set nil protocol") - } - targetUrl = strings.TrimSpace(targetUrl) - if targetUrl != "" { - var err error - var target *url.URL - target, err = url.Parse(targetUrl) - if err != nil { - return fmt.Errorf("http target option failed to parse target url: %s", err.Error()) - } - - p.Target = target - - if p.RequestTemplate == nil { - p.RequestTemplate = &nethttp.Request{ - Method: nethttp.MethodPost, - } - } - p.RequestTemplate.URL = target - - return nil - } - return fmt.Errorf("http target option was empty string") - } -} - -// WithHeader sets an additional default outbound header for all cloudevents -// when using an HTTP request. -func WithHeader(key, value string) Option { - return func(p *Protocol) error { - if p == nil { - return fmt.Errorf("http header option can not set nil protocol") - } - key = strings.TrimSpace(key) - if key != "" { - if p.RequestTemplate == nil { - p.RequestTemplate = &nethttp.Request{ - Method: nethttp.MethodPost, - } - } - if p.RequestTemplate.Header == nil { - p.RequestTemplate.Header = nethttp.Header{} - } - p.RequestTemplate.Header.Add(key, value) - return nil - } - return fmt.Errorf("http header option was empty string") - } -} - -// WithShutdownTimeout sets the shutdown timeout when the http server is being shutdown. -func WithShutdownTimeout(timeout time.Duration) Option { - return func(t *Protocol) error { - if t == nil { - return fmt.Errorf("http shutdown timeout option can not set nil protocol") - } - t.ShutdownTimeout = timeout - return nil - } -} - -func checkListen(t *Protocol, prefix string) error { - switch { - case t.listener.Load() != nil: - return fmt.Errorf("error setting %v: listener already set", prefix) - } - return nil -} - -// WithPort sets the listening port for StartReceiver. -// Only one of WithListener or WithPort is allowed. -func WithPort(port int) Option { - return func(t *Protocol) error { - if t == nil { - return fmt.Errorf("http port option can not set nil protocol") - } - if port < 0 || port > 65535 { - return fmt.Errorf("http port option was given an invalid port: %d", port) - } - if err := checkListen(t, "http port option"); err != nil { - return err - } - t.Port = port - return nil - } -} - -// WithListener sets the listener for StartReceiver. -// Only one of WithListener or WithPort is allowed. -func WithListener(l net.Listener) Option { - return func(t *Protocol) error { - if t == nil { - return fmt.Errorf("http listener option can not set nil protocol") - } - if err := checkListen(t, "http listener"); err != nil { - return err - } - t.listener.Store(l) - return nil - } -} - -// WithPath sets the path to receive cloudevents on for HTTP transports. -func WithPath(path string) Option { - return func(t *Protocol) error { - if t == nil { - return fmt.Errorf("http path option can not set nil protocol") - } - path = strings.TrimSpace(path) - if len(path) == 0 { - return fmt.Errorf("http path option was given an invalid path: %q", path) - } - t.Path = path - return nil - } -} - -// WithMethod sets the HTTP verb (GET, POST, PUT, etc.) to use -// when using an HTTP request. -func WithMethod(method string) Option { - return func(p *Protocol) error { - if p == nil { - return fmt.Errorf("http method option can not set nil protocol") - } - method = strings.TrimSpace(method) - if method != "" { - if p.RequestTemplate == nil { - p.RequestTemplate = &nethttp.Request{} - } - p.RequestTemplate.Method = method - return nil - } - return fmt.Errorf("http method option was empty string") - } -} - -// -// Middleware is a function that takes an existing http.Handler and wraps it in middleware, -// returning the wrapped http.Handler. -type Middleware func(next nethttp.Handler) nethttp.Handler - -// WithMiddleware adds an HTTP middleware to the transport. It may be specified multiple times. -// Middleware is applied to everything before it. For example -// `NewClient(WithMiddleware(foo), WithMiddleware(bar))` would result in `bar(foo(original))`. -func WithMiddleware(middleware Middleware) Option { - return func(t *Protocol) error { - if t == nil { - return fmt.Errorf("http middleware option can not set nil protocol") - } - t.middleware = append(t.middleware, middleware) - return nil - } -} - -// WithRoundTripper sets the HTTP RoundTripper. -func WithRoundTripper(roundTripper nethttp.RoundTripper) Option { - return func(t *Protocol) error { - if t == nil { - return fmt.Errorf("http round tripper option can not set nil protocol") - } - t.roundTripper = roundTripper - return nil - } -} - -// WithClient sets the protocol client -func WithClient(client nethttp.Client) Option { - return func(p *Protocol) error { - if p == nil { - return fmt.Errorf("client option can not set nil protocol") - } - p.Client = &client - return nil - } -} - -// WithGetHandlerFunc sets the http GET handler func -func WithGetHandlerFunc(fn nethttp.HandlerFunc) Option { - return func(p *Protocol) error { - if p == nil { - return fmt.Errorf("http GET handler func can not set nil protocol") - } - p.GetHandlerFn = fn - return nil - } -} - -// WithOptionsHandlerFunc sets the http OPTIONS handler func -func WithOptionsHandlerFunc(fn nethttp.HandlerFunc) Option { - return func(p *Protocol) error { - if p == nil { - return fmt.Errorf("http OPTIONS handler func can not set nil protocol") - } - p.OptionsHandlerFn = fn - return nil - } -} - -// WithDefaultOptionsHandlerFunc sets the options handler to be the built in handler and configures the options. -// methods: the supported methods reported to OPTIONS caller. -// rate: the rate limit reported to OPTIONS caller. -// origins: the prefix of the accepted origins, or "*". -// callback: preform the callback to ACK the OPTIONS request. -func WithDefaultOptionsHandlerFunc(methods []string, rate int, origins []string, callback bool) Option { - return func(p *Protocol) error { - if p == nil { - return fmt.Errorf("http OPTIONS handler func can not set nil protocol") - } - p.OptionsHandlerFn = p.DeleteHandlerFn - p.WebhookConfig = &WebhookConfig{ - AllowedMethods: methods, - AllowedRate: &rate, - AllowedOrigins: origins, - AutoACKCallback: callback, - } - return nil - } -} diff --git a/vendor/github.com/cloudevents/sdk-go/v2/protocol/http/protocol.go b/vendor/github.com/cloudevents/sdk-go/v2/protocol/http/protocol.go deleted file mode 100644 index 97372b4b..00000000 --- a/vendor/github.com/cloudevents/sdk-go/v2/protocol/http/protocol.go +++ /dev/null @@ -1,330 +0,0 @@ -package http - -import ( - "context" - "errors" - "fmt" - "io" - "net/http" - "net/url" - "sync" - "sync/atomic" - "time" - - "github.com/cloudevents/sdk-go/v2/binding" - cecontext "github.com/cloudevents/sdk-go/v2/context" - "github.com/cloudevents/sdk-go/v2/event" - "github.com/cloudevents/sdk-go/v2/protocol" -) - -const ( - // DefaultShutdownTimeout defines the default timeout given to the http.Server when calling Shutdown. - DefaultShutdownTimeout = time.Minute * 1 -) - -type msgErr struct { - msg *Message - respFn protocol.ResponseFn - err error -} - -// Protocol acts as both a http client and a http handler. -type Protocol struct { - Target *url.URL - RequestTemplate *http.Request - Client *http.Client - incoming chan msgErr - - // OptionsHandlerFn handles the OPTIONS method requests and is intended to - // implement the abuse protection spec: - // https://github.com/cloudevents/spec/blob/v1.0/http-webhook.md#4-abuse-protection - OptionsHandlerFn http.HandlerFunc - WebhookConfig *WebhookConfig - - GetHandlerFn http.HandlerFunc - DeleteHandlerFn http.HandlerFunc - - // To support Opener: - - // ShutdownTimeout defines the timeout given to the http.Server when calling Shutdown. - // If nil, DefaultShutdownTimeout is used. - ShutdownTimeout time.Duration - - // Port is the port configured to bind the receiver to. Defaults to 8080. - // If you want to know the effective port you're listening to, use GetListeningPort() - Port int - // Path is the path to bind the receiver to. Defaults to "/". - Path string - - // Receive Mutex - reMu sync.Mutex - // Handler is the handler the http Server will use. Use this to reuse the - // http server. If nil, the Protocol will create a one. - Handler *http.ServeMux - - listener atomic.Value - roundTripper http.RoundTripper - server *http.Server - handlerRegistered bool - middleware []Middleware -} - -func New(opts ...Option) (*Protocol, error) { - p := &Protocol{ - incoming: make(chan msgErr), - Port: -1, - } - if err := p.applyOptions(opts...); err != nil { - return nil, err - } - - if p.Client == nil { - p.Client = http.DefaultClient - } - - if p.roundTripper != nil { - p.Client.Transport = p.roundTripper - } - - if p.ShutdownTimeout == 0 { - p.ShutdownTimeout = DefaultShutdownTimeout - } - - return p, nil -} - -func (p *Protocol) applyOptions(opts ...Option) error { - for _, fn := range opts { - if err := fn(p); err != nil { - return err - } - } - return nil -} - -// Send implements binding.Sender -func (p *Protocol) Send(ctx context.Context, m binding.Message, transformers ...binding.Transformer) error { - if ctx == nil { - return fmt.Errorf("nil Context") - } else if m == nil { - return fmt.Errorf("nil Message") - } - - _, err := p.Request(ctx, m, transformers...) - return err -} - -// Request implements binding.Requester -func (p *Protocol) Request(ctx context.Context, m binding.Message, transformers ...binding.Transformer) (binding.Message, error) { - if ctx == nil { - return nil, fmt.Errorf("nil Context") - } else if m == nil { - return nil, fmt.Errorf("nil Message") - } - - var err error - defer func() { _ = m.Finish(err) }() - - req := p.makeRequest(ctx) - - if p.Client == nil || req == nil || req.URL == nil { - return nil, fmt.Errorf("not initialized: %#v", p) - } - - if err = WriteRequest(ctx, m, req, transformers...); err != nil { - return nil, err - } - - return p.do(ctx, req) -} - -func (p *Protocol) makeRequest(ctx context.Context) *http.Request { - // TODO: support custom headers from context? - req := &http.Request{ - Method: http.MethodPost, - Header: make(http.Header), - // TODO: HeaderFrom(ctx), - } - - if p.RequestTemplate != nil { - req.Method = p.RequestTemplate.Method - req.URL = p.RequestTemplate.URL - req.Close = p.RequestTemplate.Close - req.Host = p.RequestTemplate.Host - copyHeadersEnsure(p.RequestTemplate.Header, &req.Header) - } - - if p.Target != nil { - req.URL = p.Target - } - - // Override the default request with target from context. - if target := cecontext.TargetFrom(ctx); target != nil { - req.URL = target - } - return req.WithContext(ctx) -} - -// Ensure to is a non-nil map before copying -func copyHeadersEnsure(from http.Header, to *http.Header) { - if len(from) > 0 { - if *to == nil { - *to = http.Header{} - } - copyHeaders(from, *to) - } -} - -func copyHeaders(from, to http.Header) { - if from == nil || to == nil { - return - } - for header, values := range from { - for _, value := range values { - to.Add(header, value) - } - } -} - -// Receive the next incoming HTTP request as a CloudEvent. -// Returns non-nil error if the incoming HTTP request fails to parse as a CloudEvent -// Returns io.EOF if the receiver is closed. -func (p *Protocol) Receive(ctx context.Context) (binding.Message, error) { - if ctx == nil { - return nil, fmt.Errorf("nil Context") - } - - msg, fn, err := p.Respond(ctx) - // No-op the response when finish is invoked. - if msg != nil { - return binding.WithFinish(msg, func(err error) { - if fn != nil { - _ = fn(ctx, nil, nil) - } - }), err - } else { - return nil, err - } -} - -// Respond receives the next incoming HTTP request as a CloudEvent and waits -// for the response callback to invoked before continuing. -// Returns non-nil error if the incoming HTTP request fails to parse as a CloudEvent -// Returns io.EOF if the receiver is closed. -func (p *Protocol) Respond(ctx context.Context) (binding.Message, protocol.ResponseFn, error) { - if ctx == nil { - return nil, nil, fmt.Errorf("nil Context") - } - - select { - case in, ok := <-p.incoming: - if !ok { - return nil, nil, io.EOF - } - - if in.msg == nil { - return nil, in.respFn, in.err - } - return in.msg, in.respFn, in.err - - case <-ctx.Done(): - return nil, nil, io.EOF - } -} - -// ServeHTTP implements http.Handler. -// Blocks until ResponseFn is invoked. -func (p *Protocol) ServeHTTP(rw http.ResponseWriter, req *http.Request) { - // Filter the GET style methods: - switch req.Method { - case http.MethodOptions: - if p.OptionsHandlerFn == nil { - rw.WriteHeader(http.StatusMethodNotAllowed) - return - } - p.OptionsHandlerFn(rw, req) - return - - case http.MethodGet: - if p.GetHandlerFn == nil { - rw.WriteHeader(http.StatusMethodNotAllowed) - return - } - p.GetHandlerFn(rw, req) - return - - case http.MethodDelete: - if p.DeleteHandlerFn == nil { - rw.WriteHeader(http.StatusMethodNotAllowed) - return - } - p.DeleteHandlerFn(rw, req) - return - } - - m := NewMessageFromHttpRequest(req) - if m == nil { - // Should never get here unless ServeHTTP is called directly. - p.incoming <- msgErr{msg: nil, err: binding.ErrUnknownEncoding} - rw.WriteHeader(http.StatusBadRequest) - return // if there was no message, return. - } - - var finishErr error - m.OnFinish = func(err error) error { - finishErr = err - return nil - } - - wg := sync.WaitGroup{} - wg.Add(1) - var fn protocol.ResponseFn = func(ctx context.Context, respMsg binding.Message, res protocol.Result, transformers ...binding.Transformer) error { - // Unblock the ServeHTTP after the reply is written - defer func() { - wg.Done() - }() - - if finishErr != nil { - http.Error(rw, fmt.Sprintf("Cannot forward CloudEvent: %s", finishErr), http.StatusInternalServerError) - return finishErr - } - - status := http.StatusOK - if res != nil { - var result *Result - switch { - case protocol.ResultAs(res, &result): - if result.StatusCode > 100 && result.StatusCode < 600 { - status = result.StatusCode - } - - case !protocol.IsACK(res): - // Map client errors to http status code - validationError := event.ValidationError{} - if errors.As(res, &validationError) { - status = http.StatusBadRequest - rw.Header().Set("content-type", "text/plain") - rw.WriteHeader(status) - _, _ = rw.Write([]byte(validationError.Error())) - return validationError - } else if errors.Is(res, binding.ErrUnknownEncoding) { - status = http.StatusUnsupportedMediaType - } else { - status = http.StatusInternalServerError - } - } - } - - if respMsg != nil { - err := WriteResponseWriter(ctx, respMsg, status, rw, transformers...) - return respMsg.Finish(err) - } - - rw.WriteHeader(status) - return nil - } - - p.incoming <- msgErr{msg: m, respFn: fn} // Send to Request - // Block until ResponseFn is invoked - wg.Wait() -} diff --git a/vendor/github.com/cloudevents/sdk-go/v2/protocol/http/protocol_lifecycle.go b/vendor/github.com/cloudevents/sdk-go/v2/protocol/http/protocol_lifecycle.go deleted file mode 100644 index f3aafbd4..00000000 --- a/vendor/github.com/cloudevents/sdk-go/v2/protocol/http/protocol_lifecycle.go +++ /dev/null @@ -1,125 +0,0 @@ -package http - -import ( - "context" - "fmt" - "net" - "net/http" - "strings" - - "github.com/cloudevents/sdk-go/v2/protocol" - "go.opencensus.io/plugin/ochttp" - "go.opencensus.io/plugin/ochttp/propagation/tracecontext" -) - -var _ protocol.Opener = (*Protocol)(nil) - -func (p *Protocol) OpenInbound(ctx context.Context) error { - p.reMu.Lock() - defer p.reMu.Unlock() - - if p.Handler == nil { - p.Handler = http.NewServeMux() - } - - if !p.handlerRegistered { - // handler.Handle might panic if the user tries to use the same path as the sdk. - p.Handler.Handle(p.GetPath(), p) - p.handlerRegistered = true - } - - // After listener is invok - listener, err := p.listen() - if err != nil { - return err - } - - p.server = &http.Server{ - Addr: listener.Addr().String(), - Handler: &ochttp.Handler{ - Propagation: &tracecontext.HTTPFormat{}, - Handler: attachMiddleware(p.Handler, p.middleware), - FormatSpanName: formatSpanName, - }, - } - - // Shutdown - defer func() { - _ = p.server.Close() - p.server = nil - }() - - errChan := make(chan error, 1) - go func() { - errChan <- p.server.Serve(listener) - }() - - // wait for the server to return or ctx.Done(). - select { - case <-ctx.Done(): - // Try a gracefully shutdown. - ctx, cancel := context.WithTimeout(context.Background(), p.ShutdownTimeout) - defer cancel() - err := p.server.Shutdown(ctx) - <-errChan // Wait for server goroutine to exit - return err - case err := <-errChan: - return err - } -} - -// GetListeningPort returns the listening port. -// Returns -1 if it's not listening. -func (p *Protocol) GetListeningPort() int { - if listener := p.listener.Load(); listener != nil { - if tcpAddr, ok := listener.(net.Listener).Addr().(*net.TCPAddr); ok { - return tcpAddr.Port - } - } - return -1 -} - -func formatSpanName(r *http.Request) string { - return "cloudevents.http." + r.URL.Path -} - -// listen if not already listening, update t.Port -func (p *Protocol) listen() (net.Listener, error) { - if p.listener.Load() == nil { - port := 8080 - if p.Port != -1 { - port = p.Port - if port < 0 || port > 65535 { - return nil, fmt.Errorf("invalid port %d", port) - } - } - var err error - var listener net.Listener - if listener, err = net.Listen("tcp", fmt.Sprintf(":%d", port)); err != nil { - return nil, err - } - p.listener.Store(listener) - return listener, nil - } - return p.listener.Load().(net.Listener), nil -} - -// GetPath returns the path the transport is hosted on. If the path is '/', -// the transport will handle requests on any URI. To discover the true path -// a request was received on, inspect the context from Receive(cxt, ...) with -// TransportContextFrom(ctx). -func (p *Protocol) GetPath() string { - path := strings.TrimSpace(p.Path) - if len(path) > 0 { - return path - } - return "/" // default -} - -// attachMiddleware attaches the HTTP middleware to the specified handler. -func attachMiddleware(h http.Handler, middleware []Middleware) http.Handler { - for _, m := range middleware { - h = m(h) - } - return h -} diff --git a/vendor/github.com/cloudevents/sdk-go/v2/protocol/http/protocol_retry.go b/vendor/github.com/cloudevents/sdk-go/v2/protocol/http/protocol_retry.go deleted file mode 100644 index d3444b7c..00000000 --- a/vendor/github.com/cloudevents/sdk-go/v2/protocol/http/protocol_retry.go +++ /dev/null @@ -1,107 +0,0 @@ -package http - -import ( - "context" - "errors" - "go.uber.org/zap" - "net/http" - "net/url" - "time" - - "github.com/cloudevents/sdk-go/v2/binding" - cecontext "github.com/cloudevents/sdk-go/v2/context" - "github.com/cloudevents/sdk-go/v2/protocol" -) - -func (p *Protocol) do(ctx context.Context, req *http.Request) (binding.Message, error) { - params := cecontext.RetriesFrom(ctx) - - switch params.Strategy { - case cecontext.BackoffStrategyConstant, cecontext.BackoffStrategyLinear, cecontext.BackoffStrategyExponential: - return p.doWithRetry(ctx, params, req) - case cecontext.BackoffStrategyNone: - fallthrough - default: - return p.doOnce(req) - } -} - -func (p *Protocol) doOnce(req *http.Request) (binding.Message, protocol.Result) { - resp, err := p.Client.Do(req) - if err != nil { - return nil, protocol.NewReceipt(false, "%w", err) - } - - var result protocol.Result - if resp.StatusCode/100 == 2 { - result = protocol.ResultACK - } else { - result = protocol.ResultNACK - } - - return NewMessage(resp.Header, resp.Body), NewResult(resp.StatusCode, "%w", result) -} - -func (p *Protocol) doWithRetry(ctx context.Context, params *cecontext.RetryParams, req *http.Request) (binding.Message, error) { - then := time.Now() - retry := 0 - results := make([]protocol.Result, 0) - - for { - msg, result := p.doOnce(req) - - // Fast track common case. - if protocol.IsACK(result) { - return msg, NewRetriesResult(result, retry, then, results) - } - - // Try again? - // - // Make sure the error was something we should retry. - - { - var uErr *url.Error - if errors.As(result, &uErr) { - goto DoBackoff - } - } - - { - var httpResult *Result - if errors.As(result, &httpResult) { - // Potentially retry when: - // - 404 Not Found - // - 413 Payload Too Large with Retry-After (NOT SUPPORTED) - // - 425 Too Early - // - 429 Too Many Requests - // - 503 Service Unavailable (with or without Retry-After) (IGNORE Retry-After) - // - 504 Gateway Timeout - - sc := httpResult.StatusCode - if sc == 404 || sc == 425 || sc == 429 || sc == 503 || sc == 504 { - // retry! - goto DoBackoff - } else { - // Permanent error - cecontext.LoggerFrom(ctx).Debugw("status code not retryable, will not try again", - zap.Error(httpResult), - zap.Int("statusCode", sc)) - return msg, NewRetriesResult(result, retry, then, results) - } - } - } - - DoBackoff: - // Wait for the correct amount of backoff time. - - // total tries = retry + 1 - if err := params.Backoff(ctx, retry+1); err != nil { - // do not try again. - cecontext.LoggerFrom(ctx).Debugw("backoff error, will not try again", zap.Error(err)) - return msg, NewRetriesResult(result, retry, then, results) - } - - retry++ - results = append(results, result) - } -} diff --git a/vendor/github.com/cloudevents/sdk-go/v2/protocol/http/result.go b/vendor/github.com/cloudevents/sdk-go/v2/protocol/http/result.go deleted file mode 100644 index 149e6872..00000000 --- a/vendor/github.com/cloudevents/sdk-go/v2/protocol/http/result.go +++ /dev/null @@ -1,55 +0,0 @@ -package http - -import ( - "errors" - "fmt" - - "github.com/cloudevents/sdk-go/v2/protocol" -) - -// NewResult returns a fully populated http Result that should be used as -// a transport.Result. -func NewResult(statusCode int, messageFmt string, args ...interface{}) protocol.Result { - return &Result{ - StatusCode: statusCode, - Format: messageFmt, - Args: args, - } -} - -// Result wraps the fields required to make adjustments for http Responses. -type Result struct { - StatusCode int - Format string - Args []interface{} -} - -// make sure Result implements error. -var _ error = (*Result)(nil) - -// Is returns if the target error is a Result type checking target. -func (e *Result) Is(target error) bool { - if o, ok := target.(*Result); ok { - return e.StatusCode == o.StatusCode - } - - // Special case for nil == ACK - if o, ok := target.(*protocol.Receipt); ok { - if e == nil && o.ACK { - return true - } - } - - // Allow for wrapped errors. - if e != nil { - err := fmt.Errorf(e.Format, e.Args...) - return errors.Is(err, target) - } - return false -} - -// Error returns the string that is formed by using the format string with the -// provided args. -func (e *Result) Error() string { - return fmt.Sprintf("%d: %v", e.StatusCode, fmt.Errorf(e.Format, e.Args...)) -} diff --git a/vendor/github.com/cloudevents/sdk-go/v2/protocol/http/retries_result.go b/vendor/github.com/cloudevents/sdk-go/v2/protocol/http/retries_result.go deleted file mode 100644 index 0f25f705..00000000 --- a/vendor/github.com/cloudevents/sdk-go/v2/protocol/http/retries_result.go +++ /dev/null @@ -1,54 +0,0 @@ -package http - -import ( - "fmt" - "time" - - "github.com/cloudevents/sdk-go/v2/protocol" -) - -// NewRetriesResult returns a http RetriesResult that should be used as -// a transport.Result without retries -func NewRetriesResult(result protocol.Result, retries int, startTime time.Time, attempts []protocol.Result) protocol.Result { - rr := &RetriesResult{ - Result: result, - Retries: retries, - Duration: time.Since(startTime), - } - if len(attempts) > 0 { - rr.Attempts = attempts - } - return rr -} - -// RetriesResult wraps the fields required to make adjustments for http Responses. -type RetriesResult struct { - // The last result - protocol.Result - - // Retries is the number of times the request was tried - Retries int - - // Duration records the time spent retrying. Exclude the successful request (if any) - Duration time.Duration - - // Attempts of all failed requests. Exclude last result. - Attempts []protocol.Result -} - -// make sure RetriesResult implements error. -var _ error = (*RetriesResult)(nil) - -// Is returns if the target error is a RetriesResult type checking target. -func (e *RetriesResult) Is(target error) bool { - return protocol.ResultIs(e.Result, target) -} - -// Error returns the string that is formed by using the format string with the -// provided args. -func (e *RetriesResult) Error() string { - if e.Retries == 0 { - return e.Result.Error() - } - return fmt.Sprintf("%s (%dx)", e.Result.Error(), e.Retries) -} diff --git a/vendor/github.com/cloudevents/sdk-go/v2/protocol/http/write_request.go b/vendor/github.com/cloudevents/sdk-go/v2/protocol/http/write_request.go deleted file mode 100644 index e0c0d307..00000000 --- a/vendor/github.com/cloudevents/sdk-go/v2/protocol/http/write_request.go +++ /dev/null @@ -1,136 +0,0 @@ -package http - -import ( - "bytes" - "context" - "io" - "io/ioutil" - "net/http" - "strings" - - "github.com/cloudevents/sdk-go/v2/binding" - "github.com/cloudevents/sdk-go/v2/binding/format" - "github.com/cloudevents/sdk-go/v2/binding/spec" - "github.com/cloudevents/sdk-go/v2/types" -) - -// WriteRequest fills the provided httpRequest with the message m. -// Using context you can tweak the encoding processing (more details on binding.Write documentation). -func WriteRequest(ctx context.Context, m binding.Message, httpRequest *http.Request, transformers ...binding.Transformer) error { - structuredWriter := (*httpRequestWriter)(httpRequest) - binaryWriter := (*httpRequestWriter)(httpRequest) - - _, err := binding.Write( - ctx, - m, - structuredWriter, - binaryWriter, - transformers..., - ) - return err -} - -type httpRequestWriter http.Request - -func (b *httpRequestWriter) SetStructuredEvent(ctx context.Context, format format.Format, event io.Reader) error { - b.Header.Set(ContentType, format.MediaType()) - return b.setBody(event) -} - -func (b *httpRequestWriter) Start(ctx context.Context) error { - return nil -} - -func (b *httpRequestWriter) End(ctx context.Context) error { - return nil -} - -func (b *httpRequestWriter) SetData(data io.Reader) error { - return b.setBody(data) -} - -// setBody is a cherry-pick of the implementation in http.NewRequestWithContext -func (b *httpRequestWriter) setBody(body io.Reader) error { - rc, ok := body.(io.ReadCloser) - if !ok && body != nil { - rc = ioutil.NopCloser(body) - } - b.Body = rc - if body != nil { - switch v := body.(type) { - case *bytes.Buffer: - b.ContentLength = int64(v.Len()) - buf := v.Bytes() - b.GetBody = func() (io.ReadCloser, error) { - r := bytes.NewReader(buf) - return ioutil.NopCloser(r), nil - } - case *bytes.Reader: - b.ContentLength = int64(v.Len()) - snapshot := *v - b.GetBody = func() (io.ReadCloser, error) { - r := snapshot - return ioutil.NopCloser(&r), nil - } - case *strings.Reader: - b.ContentLength = int64(v.Len()) - snapshot := *v - b.GetBody = func() (io.ReadCloser, error) { - r := snapshot - return ioutil.NopCloser(&r), nil - } - default: - // This is where we'd set it to -1 (at least - // if body != NoBody) to mean unknown, but - // that broke people during the Go 1.8 testing - // period. People depend on it being 0 I - // guess. Maybe retry later. See Issue 18117. - } - // For client requests, Request.ContentLength of 0 - // means either actually 0, or unknown. The only way - // to explicitly say that the ContentLength is zero is - // to set the Body to nil. But turns out too much code - // depends on NewRequest returning a non-nil Body, - // so we use a well-known ReadCloser variable instead - // and have the http package also treat that sentinel - // variable to mean explicitly zero. - if b.GetBody != nil && b.ContentLength == 0 { - b.Body = http.NoBody - b.GetBody = func() (io.ReadCloser, error) { return http.NoBody, nil } - } - } - return nil -} - -func (b *httpRequestWriter) SetAttribute(attribute spec.Attribute, value interface{}) error { - mapping := attributeHeadersMapping[attribute.Name()] - if value == nil { - delete(b.Header, mapping) - return nil - } - - // Http headers, everything is a string! - s, err := types.Format(value) - if err != nil { - return err - } - b.Header[mapping] = append(b.Header[mapping], s) - return nil -} - -func (b *httpRequestWriter) SetExtension(name string, value interface{}) error { - if value == nil { - delete(b.Header, extNameToHeaderName(name)) - return nil - } - // Http headers, everything is a string! - s, err := types.Format(value) - if err != nil { - return err - } - b.Header[extNameToHeaderName(name)] = []string{s} - return nil -} - -var _ binding.StructuredWriter = (*httpRequestWriter)(nil) // Test it conforms to the interface -var _ binding.BinaryWriter = (*httpRequestWriter)(nil) // Test it conforms to the interface diff --git a/vendor/github.com/cloudevents/sdk-go/v2/protocol/http/write_responsewriter.go b/vendor/github.com/cloudevents/sdk-go/v2/protocol/http/write_responsewriter.go deleted file mode 100644 index 9646ca49..00000000 --- a/vendor/github.com/cloudevents/sdk-go/v2/protocol/http/write_responsewriter.go +++ /dev/null @@ -1,121 +0,0 @@ -package http - -import ( - "bytes" - "context" - "io" - "net/http" - "strconv" - "strings" - - "github.com/cloudevents/sdk-go/v2/binding" - "github.com/cloudevents/sdk-go/v2/binding/format" - "github.com/cloudevents/sdk-go/v2/binding/spec" - "github.com/cloudevents/sdk-go/v2/types" -) - -// WriteResponseWriter writes out to the the provided httpResponseWriter with the message m. -// Using context you can tweak the encoding processing (more details on binding.Write documentation). -func WriteResponseWriter(ctx context.Context, m binding.Message, status int, rw http.ResponseWriter, transformers ...binding.Transformer) error { - if status < 200 || status >= 600 { - status = http.StatusOK - } - writer := &httpResponseWriter{rw: rw, status: status} - - _, err := binding.Write( - ctx, - m, - writer, - writer, - transformers..., - ) - return err -} - -type httpResponseWriter struct { - rw http.ResponseWriter - status int - body io.Reader -} - -func (b *httpResponseWriter) SetStructuredEvent(ctx context.Context, format format.Format, event io.Reader) error { - b.rw.Header().Set(ContentType, format.MediaType()) - b.body = event - return b.finalizeWriter() -} - -func (b *httpResponseWriter) Start(ctx context.Context) error { - return nil -} - -func (b *httpResponseWriter) SetAttribute(attribute spec.Attribute, value interface{}) error { - mapping := attributeHeadersMapping[attribute.Name()] - if value == nil { - delete(b.rw.Header(), mapping) - } - - // Http headers, everything is a string! - s, err := types.Format(value) - if err != nil { - return err - } - b.rw.Header()[mapping] = append(b.rw.Header()[mapping], s) - return nil -} - -func (b *httpResponseWriter) SetExtension(name string, value interface{}) error { - if value == nil { - delete(b.rw.Header(), extNameToHeaderName(name)) - } - // Http headers, everything is a string! - s, err := types.Format(value) - if err != nil { - return err - } - b.rw.Header()[extNameToHeaderName(name)] = []string{s} - return nil -} - -func (b *httpResponseWriter) SetData(reader io.Reader) error { - b.body = reader - return nil -} - -func (b *httpResponseWriter) finalizeWriter() error { - if b.body != nil { - // Try to figure it out if we have a content-length - contentLength := -1 - switch v := b.body.(type) { - case *bytes.Buffer: - contentLength = v.Len() - case *bytes.Reader: - contentLength = v.Len() - case *strings.Reader: - contentLength = v.Len() - } - - if contentLength != -1 { - b.rw.Header().Add("Content-length", strconv.Itoa(contentLength)) - } - - // Finalize the headers. - b.rw.WriteHeader(b.status) - - // Write body. - _, err := io.Copy(b.rw, b.body) - if err != nil { - return err - } - } else { - // Finalize the headers. - b.rw.WriteHeader(b.status) - } - return nil -} - -func (b *httpResponseWriter) End(ctx context.Context) error { - return b.finalizeWriter() -} - -var _ binding.StructuredWriter = (*httpResponseWriter)(nil) // Test it conforms to the interface -var _ binding.BinaryWriter = (*httpResponseWriter)(nil) // Test it conforms to the interface diff --git a/vendor/github.com/cloudevents/sdk-go/v2/protocol/inbound.go b/vendor/github.com/cloudevents/sdk-go/v2/protocol/inbound.go deleted file mode 100644 index e67ed8ac..00000000 --- a/vendor/github.com/cloudevents/sdk-go/v2/protocol/inbound.go +++ /dev/null @@ -1,47 +0,0 @@ -package protocol - -import ( - "context" - - "github.com/cloudevents/sdk-go/v2/binding" -) - -// Receiver receives messages. -type Receiver interface { - // Receive blocks till a message is received or ctx expires. - // - // A non-nil error means the receiver is closed. - // io.EOF means it closed cleanly, any other value indicates an error. - // The caller is responsible for `Finish()` the returned message - Receive(ctx context.Context) (binding.Message, error) -} - -// ReceiveCloser is a Receiver that can be closed. -type ReceiveCloser interface { - Receiver - Closer -} - -// ResponseFn is the function callback provided from Responder.Respond to allow -// for a receiver to "reply" to a message it receives. -// transformers are applied when the message is written on the wire. -type ResponseFn func(ctx context.Context, m binding.Message, r Result, transformers ...binding.Transformer) error - -// Responder receives messages and is given a callback to respond. -type Responder interface { - // Receive blocks till a message is received or ctx expires. - // - // A non-nil error means the receiver is closed. - // io.EOF means it closed cleanly, any other value indicates an error. - // The caller is responsible for `Finish()` the returned message, - // while the protocol implementation is responsible for `Finish()` the response message. - // The caller MUST invoke ResponseFn, in order to avoid leaks. - // The correct flow for the caller is to finish the received message and then invoke the ResponseFn - Respond(ctx context.Context) (binding.Message, ResponseFn, error) -} - -// ResponderCloser is a Responder that can be closed. -type ResponderCloser interface { - Responder - Closer -} diff --git a/vendor/github.com/cloudevents/sdk-go/v2/protocol/lifecycle.go b/vendor/github.com/cloudevents/sdk-go/v2/protocol/lifecycle.go deleted file mode 100644 index 22ae08e0..00000000 --- a/vendor/github.com/cloudevents/sdk-go/v2/protocol/lifecycle.go +++ /dev/null @@ -1,18 +0,0 @@ -package protocol - -import ( - "context" -) - -// Opener is the common interface for things that need to be opened. -type Opener interface { - // OpenInbound is a blocking call and ctx is used to stop the Inbound message Receiver/Responder. - // Closing the context won't close the Receiver/Responder, aka it won't invoke Close(ctx). - OpenInbound(ctx context.Context) error -} - -// Closer is the common interface for things that can be closed. -// After invoking Close(ctx), you cannot reuse the object you closed. -type Closer interface { - Close(ctx context.Context) error -} diff --git a/vendor/github.com/cloudevents/sdk-go/v2/protocol/outbound.go b/vendor/github.com/cloudevents/sdk-go/v2/protocol/outbound.go deleted file mode 100644 index b0a87761..00000000 --- a/vendor/github.com/cloudevents/sdk-go/v2/protocol/outbound.go +++ /dev/null @@ -1,44 +0,0 @@ -package protocol - -import ( - "context" - - "github.com/cloudevents/sdk-go/v2/binding" -) - -// Sender sends messages. -type Sender interface { - // Send a message. - // - // Send returns when the "outbound" message has been sent. The Sender may - // still be expecting acknowledgment or holding other state for the message. - // - // m.Finish() is called when sending is finished (both succeeded or failed): - // expected acknowledgments (or errors) have been received, the Sender is - // no longer holding any state for the message. - // m.Finish() may be called during or after Send(). - // - // transformers are applied when the message is written on the wire. - Send(ctx context.Context, m binding.Message, transformers ...binding.Transformer) error -} - -// SendCloser is a Sender that can be closed. -type SendCloser interface { - Sender - Closer -} - -// Requester sends a message and receives a response -// -// Optional interface that may be implemented by protocols that support -// request/response correlation. -type Requester interface { - // Request sends m like Sender.Send() but also arranges to receive a response. - Request(ctx context.Context, m binding.Message, transformers ...binding.Transformer) (binding.Message, error) -} - -// RequesterCloser is a Requester that can be closed. -type RequesterCloser interface { - Requester - Closer -} diff --git a/vendor/github.com/cloudevents/sdk-go/v2/protocol/result.go b/vendor/github.com/cloudevents/sdk-go/v2/protocol/result.go deleted file mode 100644 index f9bd9f27..00000000 --- a/vendor/github.com/cloudevents/sdk-go/v2/protocol/result.go +++ /dev/null @@ -1,122 +0,0 @@ -package protocol - -import ( - "errors" - "fmt" -) - -// Result leverages go's 1.13 error wrapping. -type Result error - -// ResultIs reports whether any error in err's chain matches target. -// -// The chain consists of err itself followed by the sequence of errors obtained by -// repeatedly calling Unwrap. -// -// An error is considered to match a target if it is equal to that target or if -// it implements a method Is(error) bool such that Is(target) returns true. -// (text from errors/wrap.go) -var ResultIs = errors.Is - -// ResultAs finds the first error in err's chain that matches target, and if so, sets -// target to that error value and returns true. -// -// The chain consists of err itself followed by the sequence of errors obtained by -// repeatedly calling Unwrap. -// -// An error matches target if the error's concrete value is assignable to the value -// pointed to by target, or if the error has a method As(interface{}) bool such that -// As(target) returns true. In the latter case, the As method is responsible for -// setting target. -// -// As will panic if target is not a non-nil pointer to either a type that implements -// error, or to any interface type. As returns false if err is nil. -// (text from errors/wrap.go) -var ResultAs = errors.As - -func NewResult(messageFmt string, args ...interface{}) Result { - return fmt.Errorf(messageFmt, args...) -} - -// IsACK true means the recipient acknowledged the event. -func IsACK(target Result) bool { - // special case, nil target also means ACK. - if target == nil { - return true - } - - return ResultIs(target, ResultACK) -} - -// IsNACK true means the recipient did not acknowledge the event. -func IsNACK(target Result) bool { - return ResultIs(target, ResultNACK) -} - -// IsUndelivered true means the target result is not an ACK/NACK, but some other -// error unrelated to delivery not from the intended recipient. Likely target -// is an error that represents some part of the protocol is misconfigured or -// the event that was attempting to be sent was invalid. -func IsUndelivered(target Result) bool { - if target == nil { - // Short-circuit nil result is ACK. - return false - } - return !ResultIs(target, ResultACK) && !ResultIs(target, ResultNACK) -} - -var ( - ResultACK = NewReceipt(true, "") - ResultNACK = NewReceipt(false, "") -) - -// NewReceipt returns a fully populated protocol Receipt that should be used as -// a transport.Result. This type holds the base ACK/NACK results. -func NewReceipt(ack bool, messageFmt string, args ...interface{}) Result { - return &Receipt{ - Err: fmt.Errorf(messageFmt, args...), - ACK: ack, - } -} - -// Receipt wraps the fields required to understand if a protocol event is acknowledged. -type Receipt struct { - Err error - ACK bool -} - -// make sure Result implements error. -var _ error = (*Receipt)(nil) - -// Is returns if the target error is a Result type checking target. -func (e *Receipt) Is(target error) bool { - if o, ok := target.(*Receipt); ok { - if e == nil { - // Special case nil e as ACK. - return o.ACK - } - return e.ACK == o.ACK - } - // Allow for wrapped errors. - if e != nil { - return errors.Is(e.Err, target) - } - return false -} - -// Error returns the string that is formed by using the format string with the -// provided args. -func (e *Receipt) Error() string { - if e != nil { - return e.Err.Error() - } - return "" -} - -// Unwrap returns the wrapped error if exist or nil -func (e *Receipt) Unwrap() error { - if e != nil { - return errors.Unwrap(e.Err) - } - return nil -} diff --git a/vendor/github.com/cloudevents/sdk-go/v2/staticcheck.conf b/vendor/github.com/cloudevents/sdk-go/v2/staticcheck.conf deleted file mode 100644 index d6f26955..00000000 --- a/vendor/github.com/cloudevents/sdk-go/v2/staticcheck.conf +++ /dev/null @@ -1,3 +0,0 @@ -checks = [ - "all", "-ST1003", -] \ No newline at end of file diff --git a/vendor/github.com/cloudevents/sdk-go/v2/types/allocate.go b/vendor/github.com/cloudevents/sdk-go/v2/types/allocate.go deleted file mode 100644 index c38f7117..00000000 --- a/vendor/github.com/cloudevents/sdk-go/v2/types/allocate.go +++ /dev/null @@ -1,36 +0,0 @@ -package types - -import "reflect" - -// Allocate allocates a new instance of type t and returns: -// asPtr is of type t if t is a pointer type and of type &t otherwise -// asValue is a Value of type t pointing to the same data as asPtr -func Allocate(obj interface{}) (asPtr interface{}, asValue reflect.Value) { - if obj == nil { - return nil, reflect.Value{} - } - - switch t := reflect.TypeOf(obj); t.Kind() { - case reflect.Ptr: - reflectPtr := reflect.New(t.Elem()) - asPtr = reflectPtr.Interface() - asValue = reflectPtr - case reflect.Map: - reflectPtr := reflect.MakeMap(t) - asPtr = reflectPtr.Interface() - asValue = reflectPtr - case reflect.String: - reflectPtr := reflect.New(t) - asPtr = "" - asValue = reflectPtr.Elem() - case reflect.Slice: - reflectPtr := reflect.MakeSlice(t, 0, 0) - asPtr = reflectPtr.Interface() - asValue = reflectPtr - default: - reflectPtr := reflect.New(t) - asPtr = reflectPtr.Interface() - asValue = reflectPtr.Elem() - } - return -} diff --git a/vendor/github.com/cloudevents/sdk-go/v2/types/doc.go b/vendor/github.com/cloudevents/sdk-go/v2/types/doc.go deleted file mode 100644 index b1d9c29d..00000000 --- a/vendor/github.com/cloudevents/sdk-go/v2/types/doc.go +++ /dev/null @@ -1,41 +0,0 @@ -/* -Package types implements the CloudEvents type system. - -CloudEvents defines a set of abstract types for event context attributes. Each -type has a corresponding native Go type and a canonical string encoding. The -native Go types used to represent the CloudEvents types are: -bool, int32, string, []byte, *url.URL, time.Time - - +----------------+----------------+-----------------------------------+ - |CloudEvents Type|Native Type |Convertible From | - +================+================+===================================+ - |Bool |bool |bool | - +----------------+----------------+-----------------------------------+ - |Integer |int32 |Any numeric type with value in | - | | |range of int32 | - +----------------+----------------+-----------------------------------+ - |String |string |string | - +----------------+----------------+-----------------------------------+ - |Binary |[]byte |[]byte | - +----------------+----------------+-----------------------------------+ - |URI-Reference |*url.URL |url.URL, types.URIRef, types.URI | - +----------------+----------------+-----------------------------------+ - |URI |*url.URL |url.URL, types.URIRef, types.URI | - | | |Must be an absolute URI. | - +----------------+----------------+-----------------------------------+ - |Timestamp |time.Time |time.Time, types.Timestamp | - +----------------+----------------+-----------------------------------+ - -Extension attributes may be stored as a native type or a canonical string. The -To functions will convert to the desired from any convertible type -or from the canonical string form. - -The Parse and Format functions convert native types to/from -canonical strings. - -Note are no Parse or Format functions for URL or string. For URL use the -standard url.Parse() and url.URL.String(). The canonical string format of a -string is the string itself. - -*/ -package types diff --git a/vendor/github.com/cloudevents/sdk-go/v2/types/timestamp.go b/vendor/github.com/cloudevents/sdk-go/v2/types/timestamp.go deleted file mode 100644 index 3ae1c7de..00000000 --- a/vendor/github.com/cloudevents/sdk-go/v2/types/timestamp.go +++ /dev/null @@ -1,70 +0,0 @@ -package types - -import ( - "encoding/json" - "encoding/xml" - "fmt" - "time" -) - -// Timestamp wraps time.Time to normalize the time layout to RFC3339. It is -// intended to enforce compliance with the CloudEvents spec for their -// definition of Timestamp. Custom marshal methods are implemented to ensure -// the outbound Timestamp is a string in the RFC3339 layout. -type Timestamp struct { - time.Time -} - -// ParseTimestamp attempts to parse the given time assuming RFC3339 layout -func ParseTimestamp(s string) (*Timestamp, error) { - if s == "" { - return nil, nil - } - tt, err := ParseTime(s) - return &Timestamp{Time: tt}, err -} - -// MarshalJSON implements a custom json marshal method used when this type is -// marshaled using json.Marshal. -func (t *Timestamp) MarshalJSON() ([]byte, error) { - if t == nil || t.IsZero() { - return []byte(`""`), nil - } - return []byte(fmt.Sprintf("%q", t)), nil -} - -// UnmarshalJSON implements the json unmarshal method used when this type is -// unmarshaled using json.Unmarshal. -func (t *Timestamp) UnmarshalJSON(b []byte) error { - var timestamp string - if err := json.Unmarshal(b, ×tamp); err != nil { - return err - } - var err error - t.Time, err = ParseTime(timestamp) - return err -} - -// MarshalXML implements a custom xml marshal method used when this type is -// marshaled using xml.Marshal. -func (t *Timestamp) MarshalXML(e *xml.Encoder, start xml.StartElement) error { - if t == nil || t.IsZero() { - return e.EncodeElement(nil, start) - } - return e.EncodeElement(t.String(), start) -} - -// UnmarshalXML implements the xml unmarshal method used when this type is -// unmarshaled using xml.Unmarshal. -func (t *Timestamp) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error { - var timestamp string - if err := d.DecodeElement(×tamp, &start); err != nil { - return err - } - var err error - t.Time, err = ParseTime(timestamp) - return err -} - -// String outputs the time using RFC3339 format. -func (t Timestamp) String() string { return FormatTime(t.Time) } diff --git a/vendor/github.com/cloudevents/sdk-go/v2/types/uri.go b/vendor/github.com/cloudevents/sdk-go/v2/types/uri.go deleted file mode 100644 index 97248a24..00000000 --- a/vendor/github.com/cloudevents/sdk-go/v2/types/uri.go +++ /dev/null @@ -1,77 +0,0 @@ -package types - -import ( - "encoding/json" - "encoding/xml" - "fmt" - "net/url" -) - -// URI is a wrapper to url.URL. It is intended to enforce compliance with -// the CloudEvents spec for their definition of URI. Custom -// marshal methods are implemented to ensure the outbound URI object -// is a flat string. -type URI struct { - url.URL -} - -// ParseURI attempts to parse the given string as a URI. -func ParseURI(u string) *URI { - if u == "" { - return nil - } - pu, err := url.Parse(u) - if err != nil { - return nil - } - return &URI{URL: *pu} -} - -// MarshalJSON implements a custom json marshal method used when this type is -// marshaled using json.Marshal. -func (u URI) MarshalJSON() ([]byte, error) { - b := fmt.Sprintf("%q", u.String()) - return []byte(b), nil -} - -// UnmarshalJSON implements the json unmarshal method used when this type is -// unmarshaled using json.Unmarshal. -func (u *URI) UnmarshalJSON(b []byte) error { - var ref string - if err := json.Unmarshal(b, &ref); err != nil { - return err - } - r := ParseURI(ref) - if r != nil { - *u = *r - } - return nil -} - -// MarshalXML implements a custom xml marshal method used when this type is -// marshaled using xml.Marshal. -func (u URI) MarshalXML(e *xml.Encoder, start xml.StartElement) error { - return e.EncodeElement(u.String(), start) -} - -// UnmarshalXML implements the xml unmarshal method used when this type is -// unmarshaled using xml.Unmarshal. -func (u *URI) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error { - var ref string - if err := d.DecodeElement(&ref, &start); err != nil { - return err - } - r := ParseURI(ref) - if r != nil { - *u = *r - } - return nil -} - -// String returns the full string representation of the URI-Reference. -func (u *URI) String() string { - if u == nil { - return "" - } - return u.URL.String() -} diff --git a/vendor/github.com/cloudevents/sdk-go/v2/types/uriref.go b/vendor/github.com/cloudevents/sdk-go/v2/types/uriref.go deleted file mode 100644 index e19a1dbb..00000000 --- a/vendor/github.com/cloudevents/sdk-go/v2/types/uriref.go +++ /dev/null @@ -1,77 +0,0 @@ -package types - -import ( - "encoding/json" - "encoding/xml" - "fmt" - "net/url" -) - -// URIRef is a wrapper to url.URL. It is intended to enforce compliance with -// the CloudEvents spec for their definition of URI-Reference. Custom -// marshal methods are implemented to ensure the outbound URIRef object is -// is a flat string. -type URIRef struct { - url.URL -} - -// ParseURIRef attempts to parse the given string as a URI-Reference. -func ParseURIRef(u string) *URIRef { - if u == "" { - return nil - } - pu, err := url.Parse(u) - if err != nil { - return nil - } - return &URIRef{URL: *pu} -} - -// MarshalJSON implements a custom json marshal method used when this type is -// marshaled using json.Marshal. -func (u URIRef) MarshalJSON() ([]byte, error) { - b := fmt.Sprintf("%q", u.String()) - return []byte(b), nil -} - -// UnmarshalJSON implements the json unmarshal method used when this type is -// unmarshaled using json.Unmarshal. -func (u *URIRef) UnmarshalJSON(b []byte) error { - var ref string - if err := json.Unmarshal(b, &ref); err != nil { - return err - } - r := ParseURIRef(ref) - if r != nil { - *u = *r - } - return nil -} - -// MarshalXML implements a custom xml marshal method used when this type is -// marshaled using xml.Marshal. -func (u URIRef) MarshalXML(e *xml.Encoder, start xml.StartElement) error { - return e.EncodeElement(u.String(), start) -} - -// UnmarshalXML implements the xml unmarshal method used when this type is -// unmarshaled using xml.Unmarshal. -func (u *URIRef) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error { - var ref string - if err := d.DecodeElement(&ref, &start); err != nil { - return err - } - r := ParseURIRef(ref) - if r != nil { - *u = *r - } - return nil -} - -// String returns the full string representation of the URI-Reference. -func (u *URIRef) String() string { - if u == nil { - return "" - } - return u.URL.String() -} diff --git a/vendor/github.com/cloudevents/sdk-go/v2/types/value.go b/vendor/github.com/cloudevents/sdk-go/v2/types/value.go deleted file mode 100644 index adfbdd68..00000000 --- a/vendor/github.com/cloudevents/sdk-go/v2/types/value.go +++ /dev/null @@ -1,330 +0,0 @@ -package types - -import ( - "encoding/base64" - "fmt" - "math" - "net/url" - "reflect" - "strconv" - "time" -) - -// FormatBool returns canonical string format: "true" or "false" -func FormatBool(v bool) string { return strconv.FormatBool(v) } - -// FormatInteger returns canonical string format: decimal notation. -func FormatInteger(v int32) string { return strconv.Itoa(int(v)) } - -// FormatBinary returns canonical string format: standard base64 encoding -func FormatBinary(v []byte) string { return base64.StdEncoding.EncodeToString(v) } - -// FormatTime returns canonical string format: RFC3339 with nanoseconds -func FormatTime(v time.Time) string { return v.UTC().Format(time.RFC3339Nano) } - -// ParseBool parse canonical string format: "true" or "false" -func ParseBool(v string) (bool, error) { return strconv.ParseBool(v) } - -// ParseInteger parse canonical string format: decimal notation. -func ParseInteger(v string) (int32, error) { - // Accept floating-point but truncate to int32 as per CE spec. - f, err := strconv.ParseFloat(v, 64) - if err != nil { - return 0, err - } - if f > math.MaxInt32 || f < math.MinInt32 { - return 0, rangeErr(v) - } - return int32(f), nil -} - -// ParseBinary parse canonical string format: standard base64 encoding -func ParseBinary(v string) ([]byte, error) { return base64.StdEncoding.DecodeString(v) } - -// ParseTime parse canonical string format: RFC3339 with nanoseconds -func ParseTime(v string) (time.Time, error) { - t, err := time.Parse(time.RFC3339Nano, v) - if err != nil { - err := convertErr(time.Time{}, v) - err.extra = ": not in RFC3339 format" - return time.Time{}, err - } - return t, nil -} - -// Format returns the canonical string format of v, where v can be -// any type that is convertible to a CloudEvents type. -func Format(v interface{}) (string, error) { - v, err := Validate(v) - if err != nil { - return "", err - } - switch v := v.(type) { - case bool: - return FormatBool(v), nil - case int32: - return FormatInteger(v), nil - case string: - return v, nil - case []byte: - return FormatBinary(v), nil - case URI: - return v.String(), nil - case URIRef: - // url.URL is often passed by pointer so allow both - return v.String(), nil - case Timestamp: - return FormatTime(v.Time), nil - default: - return "", fmt.Errorf("%T is not a CloudEvents type", v) - } -} - -// Validate v is a valid CloudEvents attribute value, convert it to one of: -// bool, int32, string, []byte, types.URI, types.URIRef, types.Timestamp -func Validate(v interface{}) (interface{}, error) { - switch v := v.(type) { - case bool, int32, string, []byte: - return v, nil // Already a CloudEvents type, no validation needed. - - case uint, uintptr, uint8, uint16, uint32, uint64: - u := reflect.ValueOf(v).Uint() - if u > math.MaxInt32 { - return nil, rangeErr(v) - } - return int32(u), nil - case int, int8, int16, int64: - i := reflect.ValueOf(v).Int() - if i > math.MaxInt32 || i < math.MinInt32 { - return nil, rangeErr(v) - } - return int32(i), nil - case float32, float64: - f := reflect.ValueOf(v).Float() - if f > math.MaxInt32 || f < math.MinInt32 { - return nil, rangeErr(v) - } - return int32(f), nil - - case *url.URL: - if v == nil { - break - } - return URI{URL: *v}, nil - case url.URL: - return URI{URL: v}, nil - case *URIRef: - if v != nil { - return *v, nil - } - return nil, nil - case URIRef: - return v, nil - case *URI: - if v != nil { - return *v, nil - } - return nil, nil - case URI: - return v, nil - case time.Time: - return Timestamp{Time: v}, nil - case *time.Time: - if v == nil { - break - } - return Timestamp{Time: *v}, nil - case Timestamp: - return v, nil - } - rx := reflect.ValueOf(v) - if rx.Kind() == reflect.Ptr && !rx.IsNil() { - // Allow pointers-to convertible types - return Validate(rx.Elem().Interface()) - } - return nil, fmt.Errorf("invalid CloudEvents value: %#v", v) -} - -// Clone v clones a CloudEvents attribute value, which is one of the valid types: -// bool, int32, string, []byte, types.URI, types.URIRef, types.Timestamp -// Returns the same type -// Panics if the type is not valid -func Clone(v interface{}) interface{} { - if v == nil { - return nil - } - switch v := v.(type) { - case bool, int32, string, nil: - return v // Already a CloudEvents type, no validation needed. - case []byte: - clone := make([]byte, len(v)) - copy(clone, v) - return v - case url.URL: - return URI{v} - case *url.URL: - return &URI{*v} - case URIRef: - return v - case *URIRef: - return &URIRef{v.URL} - case URI: - return v - case *URI: - return &URI{v.URL} - case time.Time: - return Timestamp{v} - case *time.Time: - return &Timestamp{*v} - case Timestamp: - return v - case *Timestamp: - return &Timestamp{v.Time} - } - panic(fmt.Errorf("invalid CloudEvents value: %#v", v)) -} - -// ToBool accepts a bool value or canonical "true"/"false" string. -func ToBool(v interface{}) (bool, error) { - v, err := Validate(v) - if err != nil { - return false, err - } - switch v := v.(type) { - case bool: - return v, nil - case string: - return ParseBool(v) - default: - return false, convertErr(true, v) - } -} - -// ToInteger accepts any numeric value in int32 range, or canonical string. -func ToInteger(v interface{}) (int32, error) { - v, err := Validate(v) - if err != nil { - return 0, err - } - switch v := v.(type) { - case int32: - return v, nil - case string: - return ParseInteger(v) - default: - return 0, convertErr(int32(0), v) - } -} - -// ToString returns a string value unaltered. -// -// This function does not perform canonical string encoding, use one of the -// Format functions for that. -func ToString(v interface{}) (string, error) { - v, err := Validate(v) - if err != nil { - return "", err - } - switch v := v.(type) { - case string: - return v, nil - default: - return "", convertErr("", v) - } -} - -// ToBinary returns a []byte value, decoding from base64 string if necessary. -func ToBinary(v interface{}) ([]byte, error) { - v, err := Validate(v) - if err != nil { - return nil, err - } - switch v := v.(type) { - case []byte: - return v, nil - case string: - return base64.StdEncoding.DecodeString(v) - default: - return nil, convertErr([]byte(nil), v) - } -} - -// ToURL returns a *url.URL value, parsing from string if necessary. -func ToURL(v interface{}) (*url.URL, error) { - v, err := Validate(v) - if err != nil { - return nil, err - } - switch v := v.(type) { - case *URI: - return &v.URL, nil - case URI: - return &v.URL, nil - case *URIRef: - return &v.URL, nil - case URIRef: - return &v.URL, nil - case string: - u, err := url.Parse(v) - if err != nil { - return nil, err - } - return u, nil - default: - return nil, convertErr((*url.URL)(nil), v) - } -} - -// ToTime returns a time.Time value, parsing from RFC3339 string if necessary. -func ToTime(v interface{}) (time.Time, error) { - v, err := Validate(v) - if err != nil { - return time.Time{}, err - } - switch v := v.(type) { - case Timestamp: - return v.Time, nil - case string: - ts, err := time.Parse(time.RFC3339Nano, v) - if err != nil { - return time.Time{}, err - } - return ts, nil - default: - return time.Time{}, convertErr(time.Time{}, v) - } -} - -func IsZero(v interface{}) bool { - // Fast path - if v == nil { - return true - } - if s, ok := v.(string); ok && s == "" { - return true - } - return reflect.ValueOf(v).IsZero() -} - -type ConvertErr struct { - // Value being converted - Value interface{} - // Type of attempted conversion - Type reflect.Type - - extra string -} - -func (e *ConvertErr) Error() string { - return fmt.Sprintf("cannot convert %#v to %s%s", e.Value, e.Type, e.extra) -} - -func convertErr(target, v interface{}) *ConvertErr { - return &ConvertErr{Value: v, Type: reflect.TypeOf(target)} -} - -func rangeErr(v interface{}) error { - e := convertErr(int32(0), v) - e.extra = ": out of range" - return e -} diff --git a/vendor/github.com/davecgh/go-spew/LICENSE b/vendor/github.com/davecgh/go-spew/LICENSE deleted file mode 100644 index bc52e96f..00000000 --- a/vendor/github.com/davecgh/go-spew/LICENSE +++ /dev/null @@ -1,15 +0,0 @@ -ISC License - -Copyright (c) 2012-2016 Dave Collins - -Permission to use, copy, modify, and/or distribute this software for any -purpose with or without fee is hereby granted, provided that the above -copyright notice and this permission notice appear in all copies. - -THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES -WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR -ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF -OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. diff --git a/vendor/github.com/davecgh/go-spew/spew/bypass.go b/vendor/github.com/davecgh/go-spew/spew/bypass.go deleted file mode 100644 index 79299478..00000000 --- a/vendor/github.com/davecgh/go-spew/spew/bypass.go +++ /dev/null @@ -1,145 +0,0 @@ -// Copyright (c) 2015-2016 Dave Collins -// -// Permission to use, copy, modify, and distribute this software for any -// purpose with or without fee is hereby granted, provided that the above -// copyright notice and this permission notice appear in all copies. -// -// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES -// WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -// MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR -// ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -// WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -// ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF -// OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -// NOTE: Due to the following build constraints, this file will only be compiled -// when the code is not running on Google App Engine, compiled by GopherJS, and -// "-tags safe" is not added to the go build command line. The "disableunsafe" -// tag is deprecated and thus should not be used. -// Go versions prior to 1.4 are disabled because they use a different layout -// for interfaces which make the implementation of unsafeReflectValue more complex. -// +build !js,!appengine,!safe,!disableunsafe,go1.4 - -package spew - -import ( - "reflect" - "unsafe" -) - -const ( - // UnsafeDisabled is a build-time constant which specifies whether or - // not access to the unsafe package is available. - UnsafeDisabled = false - - // ptrSize is the size of a pointer on the current arch. - ptrSize = unsafe.Sizeof((*byte)(nil)) -) - -type flag uintptr - -var ( - // flagRO indicates whether the value field of a reflect.Value - // is read-only. - flagRO flag - - // flagAddr indicates whether the address of the reflect.Value's - // value may be taken. - flagAddr flag -) - -// flagKindMask holds the bits that make up the kind -// part of the flags field. In all the supported versions, -// it is in the lower 5 bits. -const flagKindMask = flag(0x1f) - -// Different versions of Go have used different -// bit layouts for the flags type. This table -// records the known combinations. -var okFlags = []struct { - ro, addr flag -}{{ - // From Go 1.4 to 1.5 - ro: 1 << 5, - addr: 1 << 7, -}, { - // Up to Go tip. - ro: 1<<5 | 1<<6, - addr: 1 << 8, -}} - -var flagValOffset = func() uintptr { - field, ok := reflect.TypeOf(reflect.Value{}).FieldByName("flag") - if !ok { - panic("reflect.Value has no flag field") - } - return field.Offset -}() - -// flagField returns a pointer to the flag field of a reflect.Value. -func flagField(v *reflect.Value) *flag { - return (*flag)(unsafe.Pointer(uintptr(unsafe.Pointer(v)) + flagValOffset)) -} - -// unsafeReflectValue converts the passed reflect.Value into a one that bypasses -// the typical safety restrictions preventing access to unaddressable and -// unexported data. It works by digging the raw pointer to the underlying -// value out of the protected value and generating a new unprotected (unsafe) -// reflect.Value to it. -// -// This allows us to check for implementations of the Stringer and error -// interfaces to be used for pretty printing ordinarily unaddressable and -// inaccessible values such as unexported struct fields. -func unsafeReflectValue(v reflect.Value) reflect.Value { - if !v.IsValid() || (v.CanInterface() && v.CanAddr()) { - return v - } - flagFieldPtr := flagField(&v) - *flagFieldPtr &^= flagRO - *flagFieldPtr |= flagAddr - return v -} - -// Sanity checks against future reflect package changes -// to the type or semantics of the Value.flag field. -func init() { - field, ok := reflect.TypeOf(reflect.Value{}).FieldByName("flag") - if !ok { - panic("reflect.Value has no flag field") - } - if field.Type.Kind() != reflect.TypeOf(flag(0)).Kind() { - panic("reflect.Value flag field has changed kind") - } - type t0 int - var t struct { - A t0 - // t0 will have flagEmbedRO set. - t0 - // a will have flagStickyRO set - a t0 - } - vA := reflect.ValueOf(t).FieldByName("A") - va := reflect.ValueOf(t).FieldByName("a") - vt0 := reflect.ValueOf(t).FieldByName("t0") - - // Infer flagRO from the difference between the flags - // for the (otherwise identical) fields in t. - flagPublic := *flagField(&vA) - flagWithRO := *flagField(&va) | *flagField(&vt0) - flagRO = flagPublic ^ flagWithRO - - // Infer flagAddr from the difference between a value - // taken from a pointer and not. - vPtrA := reflect.ValueOf(&t).Elem().FieldByName("A") - flagNoPtr := *flagField(&vA) - flagPtr := *flagField(&vPtrA) - flagAddr = flagNoPtr ^ flagPtr - - // Check that the inferred flags tally with one of the known versions. - for _, f := range okFlags { - if flagRO == f.ro && flagAddr == f.addr { - return - } - } - panic("reflect.Value read-only flag has changed semantics") -} diff --git a/vendor/github.com/davecgh/go-spew/spew/bypasssafe.go b/vendor/github.com/davecgh/go-spew/spew/bypasssafe.go deleted file mode 100644 index 205c28d6..00000000 --- a/vendor/github.com/davecgh/go-spew/spew/bypasssafe.go +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright (c) 2015-2016 Dave Collins -// -// Permission to use, copy, modify, and distribute this software for any -// purpose with or without fee is hereby granted, provided that the above -// copyright notice and this permission notice appear in all copies. -// -// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES -// WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -// MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR -// ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -// WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -// ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF -// OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -// NOTE: Due to the following build constraints, this file will only be compiled -// when the code is running on Google App Engine, compiled by GopherJS, or -// "-tags safe" is added to the go build command line. The "disableunsafe" -// tag is deprecated and thus should not be used. -// +build js appengine safe disableunsafe !go1.4 - -package spew - -import "reflect" - -const ( - // UnsafeDisabled is a build-time constant which specifies whether or - // not access to the unsafe package is available. - UnsafeDisabled = true -) - -// unsafeReflectValue typically converts the passed reflect.Value into a one -// that bypasses the typical safety restrictions preventing access to -// unaddressable and unexported data. However, doing this relies on access to -// the unsafe package. This is a stub version which simply returns the passed -// reflect.Value when the unsafe package is not available. -func unsafeReflectValue(v reflect.Value) reflect.Value { - return v -} diff --git a/vendor/github.com/davecgh/go-spew/spew/common.go b/vendor/github.com/davecgh/go-spew/spew/common.go deleted file mode 100644 index 1be8ce94..00000000 --- a/vendor/github.com/davecgh/go-spew/spew/common.go +++ /dev/null @@ -1,341 +0,0 @@ -/* - * Copyright (c) 2013-2016 Dave Collins - * - * Permission to use, copy, modify, and distribute this software for any - * purpose with or without fee is hereby granted, provided that the above - * copyright notice and this permission notice appear in all copies. - * - * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - */ - -package spew - -import ( - "bytes" - "fmt" - "io" - "reflect" - "sort" - "strconv" -) - -// Some constants in the form of bytes to avoid string overhead. This mirrors -// the technique used in the fmt package. -var ( - panicBytes = []byte("(PANIC=") - plusBytes = []byte("+") - iBytes = []byte("i") - trueBytes = []byte("true") - falseBytes = []byte("false") - interfaceBytes = []byte("(interface {})") - commaNewlineBytes = []byte(",\n") - newlineBytes = []byte("\n") - openBraceBytes = []byte("{") - openBraceNewlineBytes = []byte("{\n") - closeBraceBytes = []byte("}") - asteriskBytes = []byte("*") - colonBytes = []byte(":") - colonSpaceBytes = []byte(": ") - openParenBytes = []byte("(") - closeParenBytes = []byte(")") - spaceBytes = []byte(" ") - pointerChainBytes = []byte("->") - nilAngleBytes = []byte("") - maxNewlineBytes = []byte("\n") - maxShortBytes = []byte("") - circularBytes = []byte("") - circularShortBytes = []byte("") - invalidAngleBytes = []byte("") - openBracketBytes = []byte("[") - closeBracketBytes = []byte("]") - percentBytes = []byte("%") - precisionBytes = []byte(".") - openAngleBytes = []byte("<") - closeAngleBytes = []byte(">") - openMapBytes = []byte("map[") - closeMapBytes = []byte("]") - lenEqualsBytes = []byte("len=") - capEqualsBytes = []byte("cap=") -) - -// hexDigits is used to map a decimal value to a hex digit. -var hexDigits = "0123456789abcdef" - -// catchPanic handles any panics that might occur during the handleMethods -// calls. -func catchPanic(w io.Writer, v reflect.Value) { - if err := recover(); err != nil { - w.Write(panicBytes) - fmt.Fprintf(w, "%v", err) - w.Write(closeParenBytes) - } -} - -// handleMethods attempts to call the Error and String methods on the underlying -// type the passed reflect.Value represents and outputes the result to Writer w. -// -// It handles panics in any called methods by catching and displaying the error -// as the formatted value. -func handleMethods(cs *ConfigState, w io.Writer, v reflect.Value) (handled bool) { - // We need an interface to check if the type implements the error or - // Stringer interface. However, the reflect package won't give us an - // interface on certain things like unexported struct fields in order - // to enforce visibility rules. We use unsafe, when it's available, - // to bypass these restrictions since this package does not mutate the - // values. - if !v.CanInterface() { - if UnsafeDisabled { - return false - } - - v = unsafeReflectValue(v) - } - - // Choose whether or not to do error and Stringer interface lookups against - // the base type or a pointer to the base type depending on settings. - // Technically calling one of these methods with a pointer receiver can - // mutate the value, however, types which choose to satisify an error or - // Stringer interface with a pointer receiver should not be mutating their - // state inside these interface methods. - if !cs.DisablePointerMethods && !UnsafeDisabled && !v.CanAddr() { - v = unsafeReflectValue(v) - } - if v.CanAddr() { - v = v.Addr() - } - - // Is it an error or Stringer? - switch iface := v.Interface().(type) { - case error: - defer catchPanic(w, v) - if cs.ContinueOnMethod { - w.Write(openParenBytes) - w.Write([]byte(iface.Error())) - w.Write(closeParenBytes) - w.Write(spaceBytes) - return false - } - - w.Write([]byte(iface.Error())) - return true - - case fmt.Stringer: - defer catchPanic(w, v) - if cs.ContinueOnMethod { - w.Write(openParenBytes) - w.Write([]byte(iface.String())) - w.Write(closeParenBytes) - w.Write(spaceBytes) - return false - } - w.Write([]byte(iface.String())) - return true - } - return false -} - -// printBool outputs a boolean value as true or false to Writer w. -func printBool(w io.Writer, val bool) { - if val { - w.Write(trueBytes) - } else { - w.Write(falseBytes) - } -} - -// printInt outputs a signed integer value to Writer w. -func printInt(w io.Writer, val int64, base int) { - w.Write([]byte(strconv.FormatInt(val, base))) -} - -// printUint outputs an unsigned integer value to Writer w. -func printUint(w io.Writer, val uint64, base int) { - w.Write([]byte(strconv.FormatUint(val, base))) -} - -// printFloat outputs a floating point value using the specified precision, -// which is expected to be 32 or 64bit, to Writer w. -func printFloat(w io.Writer, val float64, precision int) { - w.Write([]byte(strconv.FormatFloat(val, 'g', -1, precision))) -} - -// printComplex outputs a complex value using the specified float precision -// for the real and imaginary parts to Writer w. -func printComplex(w io.Writer, c complex128, floatPrecision int) { - r := real(c) - w.Write(openParenBytes) - w.Write([]byte(strconv.FormatFloat(r, 'g', -1, floatPrecision))) - i := imag(c) - if i >= 0 { - w.Write(plusBytes) - } - w.Write([]byte(strconv.FormatFloat(i, 'g', -1, floatPrecision))) - w.Write(iBytes) - w.Write(closeParenBytes) -} - -// printHexPtr outputs a uintptr formatted as hexadecimal with a leading '0x' -// prefix to Writer w. -func printHexPtr(w io.Writer, p uintptr) { - // Null pointer. - num := uint64(p) - if num == 0 { - w.Write(nilAngleBytes) - return - } - - // Max uint64 is 16 bytes in hex + 2 bytes for '0x' prefix - buf := make([]byte, 18) - - // It's simpler to construct the hex string right to left. - base := uint64(16) - i := len(buf) - 1 - for num >= base { - buf[i] = hexDigits[num%base] - num /= base - i-- - } - buf[i] = hexDigits[num] - - // Add '0x' prefix. - i-- - buf[i] = 'x' - i-- - buf[i] = '0' - - // Strip unused leading bytes. - buf = buf[i:] - w.Write(buf) -} - -// valuesSorter implements sort.Interface to allow a slice of reflect.Value -// elements to be sorted. -type valuesSorter struct { - values []reflect.Value - strings []string // either nil or same len and values - cs *ConfigState -} - -// newValuesSorter initializes a valuesSorter instance, which holds a set of -// surrogate keys on which the data should be sorted. It uses flags in -// ConfigState to decide if and how to populate those surrogate keys. -func newValuesSorter(values []reflect.Value, cs *ConfigState) sort.Interface { - vs := &valuesSorter{values: values, cs: cs} - if canSortSimply(vs.values[0].Kind()) { - return vs - } - if !cs.DisableMethods { - vs.strings = make([]string, len(values)) - for i := range vs.values { - b := bytes.Buffer{} - if !handleMethods(cs, &b, vs.values[i]) { - vs.strings = nil - break - } - vs.strings[i] = b.String() - } - } - if vs.strings == nil && cs.SpewKeys { - vs.strings = make([]string, len(values)) - for i := range vs.values { - vs.strings[i] = Sprintf("%#v", vs.values[i].Interface()) - } - } - return vs -} - -// canSortSimply tests whether a reflect.Kind is a primitive that can be sorted -// directly, or whether it should be considered for sorting by surrogate keys -// (if the ConfigState allows it). -func canSortSimply(kind reflect.Kind) bool { - // This switch parallels valueSortLess, except for the default case. - switch kind { - case reflect.Bool: - return true - case reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Int: - return true - case reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uint: - return true - case reflect.Float32, reflect.Float64: - return true - case reflect.String: - return true - case reflect.Uintptr: - return true - case reflect.Array: - return true - } - return false -} - -// Len returns the number of values in the slice. It is part of the -// sort.Interface implementation. -func (s *valuesSorter) Len() int { - return len(s.values) -} - -// Swap swaps the values at the passed indices. It is part of the -// sort.Interface implementation. -func (s *valuesSorter) Swap(i, j int) { - s.values[i], s.values[j] = s.values[j], s.values[i] - if s.strings != nil { - s.strings[i], s.strings[j] = s.strings[j], s.strings[i] - } -} - -// valueSortLess returns whether the first value should sort before the second -// value. It is used by valueSorter.Less as part of the sort.Interface -// implementation. -func valueSortLess(a, b reflect.Value) bool { - switch a.Kind() { - case reflect.Bool: - return !a.Bool() && b.Bool() - case reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Int: - return a.Int() < b.Int() - case reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uint: - return a.Uint() < b.Uint() - case reflect.Float32, reflect.Float64: - return a.Float() < b.Float() - case reflect.String: - return a.String() < b.String() - case reflect.Uintptr: - return a.Uint() < b.Uint() - case reflect.Array: - // Compare the contents of both arrays. - l := a.Len() - for i := 0; i < l; i++ { - av := a.Index(i) - bv := b.Index(i) - if av.Interface() == bv.Interface() { - continue - } - return valueSortLess(av, bv) - } - } - return a.String() < b.String() -} - -// Less returns whether the value at index i should sort before the -// value at index j. It is part of the sort.Interface implementation. -func (s *valuesSorter) Less(i, j int) bool { - if s.strings == nil { - return valueSortLess(s.values[i], s.values[j]) - } - return s.strings[i] < s.strings[j] -} - -// sortValues is a sort function that handles both native types and any type that -// can be converted to error or Stringer. Other inputs are sorted according to -// their Value.String() value to ensure display stability. -func sortValues(values []reflect.Value, cs *ConfigState) { - if len(values) == 0 { - return - } - sort.Sort(newValuesSorter(values, cs)) -} diff --git a/vendor/github.com/davecgh/go-spew/spew/config.go b/vendor/github.com/davecgh/go-spew/spew/config.go deleted file mode 100644 index 2e3d22f3..00000000 --- a/vendor/github.com/davecgh/go-spew/spew/config.go +++ /dev/null @@ -1,306 +0,0 @@ -/* - * Copyright (c) 2013-2016 Dave Collins - * - * Permission to use, copy, modify, and distribute this software for any - * purpose with or without fee is hereby granted, provided that the above - * copyright notice and this permission notice appear in all copies. - * - * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - */ - -package spew - -import ( - "bytes" - "fmt" - "io" - "os" -) - -// ConfigState houses the configuration options used by spew to format and -// display values. There is a global instance, Config, that is used to control -// all top-level Formatter and Dump functionality. Each ConfigState instance -// provides methods equivalent to the top-level functions. -// -// The zero value for ConfigState provides no indentation. You would typically -// want to set it to a space or a tab. -// -// Alternatively, you can use NewDefaultConfig to get a ConfigState instance -// with default settings. See the documentation of NewDefaultConfig for default -// values. -type ConfigState struct { - // Indent specifies the string to use for each indentation level. The - // global config instance that all top-level functions use set this to a - // single space by default. If you would like more indentation, you might - // set this to a tab with "\t" or perhaps two spaces with " ". - Indent string - - // MaxDepth controls the maximum number of levels to descend into nested - // data structures. The default, 0, means there is no limit. - // - // NOTE: Circular data structures are properly detected, so it is not - // necessary to set this value unless you specifically want to limit deeply - // nested data structures. - MaxDepth int - - // DisableMethods specifies whether or not error and Stringer interfaces are - // invoked for types that implement them. - DisableMethods bool - - // DisablePointerMethods specifies whether or not to check for and invoke - // error and Stringer interfaces on types which only accept a pointer - // receiver when the current type is not a pointer. - // - // NOTE: This might be an unsafe action since calling one of these methods - // with a pointer receiver could technically mutate the value, however, - // in practice, types which choose to satisify an error or Stringer - // interface with a pointer receiver should not be mutating their state - // inside these interface methods. As a result, this option relies on - // access to the unsafe package, so it will not have any effect when - // running in environments without access to the unsafe package such as - // Google App Engine or with the "safe" build tag specified. - DisablePointerMethods bool - - // DisablePointerAddresses specifies whether to disable the printing of - // pointer addresses. This is useful when diffing data structures in tests. - DisablePointerAddresses bool - - // DisableCapacities specifies whether to disable the printing of capacities - // for arrays, slices, maps and channels. This is useful when diffing - // data structures in tests. - DisableCapacities bool - - // ContinueOnMethod specifies whether or not recursion should continue once - // a custom error or Stringer interface is invoked. The default, false, - // means it will print the results of invoking the custom error or Stringer - // interface and return immediately instead of continuing to recurse into - // the internals of the data type. - // - // NOTE: This flag does not have any effect if method invocation is disabled - // via the DisableMethods or DisablePointerMethods options. - ContinueOnMethod bool - - // SortKeys specifies map keys should be sorted before being printed. Use - // this to have a more deterministic, diffable output. Note that only - // native types (bool, int, uint, floats, uintptr and string) and types - // that support the error or Stringer interfaces (if methods are - // enabled) are supported, with other types sorted according to the - // reflect.Value.String() output which guarantees display stability. - SortKeys bool - - // SpewKeys specifies that, as a last resort attempt, map keys should - // be spewed to strings and sorted by those strings. This is only - // considered if SortKeys is true. - SpewKeys bool -} - -// Config is the active configuration of the top-level functions. -// The configuration can be changed by modifying the contents of spew.Config. -var Config = ConfigState{Indent: " "} - -// Errorf is a wrapper for fmt.Errorf that treats each argument as if it were -// passed with a Formatter interface returned by c.NewFormatter. It returns -// the formatted string as a value that satisfies error. See NewFormatter -// for formatting details. -// -// This function is shorthand for the following syntax: -// -// fmt.Errorf(format, c.NewFormatter(a), c.NewFormatter(b)) -func (c *ConfigState) Errorf(format string, a ...interface{}) (err error) { - return fmt.Errorf(format, c.convertArgs(a)...) -} - -// Fprint is a wrapper for fmt.Fprint that treats each argument as if it were -// passed with a Formatter interface returned by c.NewFormatter. It returns -// the number of bytes written and any write error encountered. See -// NewFormatter for formatting details. -// -// This function is shorthand for the following syntax: -// -// fmt.Fprint(w, c.NewFormatter(a), c.NewFormatter(b)) -func (c *ConfigState) Fprint(w io.Writer, a ...interface{}) (n int, err error) { - return fmt.Fprint(w, c.convertArgs(a)...) -} - -// Fprintf is a wrapper for fmt.Fprintf that treats each argument as if it were -// passed with a Formatter interface returned by c.NewFormatter. It returns -// the number of bytes written and any write error encountered. See -// NewFormatter for formatting details. -// -// This function is shorthand for the following syntax: -// -// fmt.Fprintf(w, format, c.NewFormatter(a), c.NewFormatter(b)) -func (c *ConfigState) Fprintf(w io.Writer, format string, a ...interface{}) (n int, err error) { - return fmt.Fprintf(w, format, c.convertArgs(a)...) -} - -// Fprintln is a wrapper for fmt.Fprintln that treats each argument as if it -// passed with a Formatter interface returned by c.NewFormatter. See -// NewFormatter for formatting details. -// -// This function is shorthand for the following syntax: -// -// fmt.Fprintln(w, c.NewFormatter(a), c.NewFormatter(b)) -func (c *ConfigState) Fprintln(w io.Writer, a ...interface{}) (n int, err error) { - return fmt.Fprintln(w, c.convertArgs(a)...) -} - -// Print is a wrapper for fmt.Print that treats each argument as if it were -// passed with a Formatter interface returned by c.NewFormatter. It returns -// the number of bytes written and any write error encountered. See -// NewFormatter for formatting details. -// -// This function is shorthand for the following syntax: -// -// fmt.Print(c.NewFormatter(a), c.NewFormatter(b)) -func (c *ConfigState) Print(a ...interface{}) (n int, err error) { - return fmt.Print(c.convertArgs(a)...) -} - -// Printf is a wrapper for fmt.Printf that treats each argument as if it were -// passed with a Formatter interface returned by c.NewFormatter. It returns -// the number of bytes written and any write error encountered. See -// NewFormatter for formatting details. -// -// This function is shorthand for the following syntax: -// -// fmt.Printf(format, c.NewFormatter(a), c.NewFormatter(b)) -func (c *ConfigState) Printf(format string, a ...interface{}) (n int, err error) { - return fmt.Printf(format, c.convertArgs(a)...) -} - -// Println is a wrapper for fmt.Println that treats each argument as if it were -// passed with a Formatter interface returned by c.NewFormatter. It returns -// the number of bytes written and any write error encountered. See -// NewFormatter for formatting details. -// -// This function is shorthand for the following syntax: -// -// fmt.Println(c.NewFormatter(a), c.NewFormatter(b)) -func (c *ConfigState) Println(a ...interface{}) (n int, err error) { - return fmt.Println(c.convertArgs(a)...) -} - -// Sprint is a wrapper for fmt.Sprint that treats each argument as if it were -// passed with a Formatter interface returned by c.NewFormatter. It returns -// the resulting string. See NewFormatter for formatting details. -// -// This function is shorthand for the following syntax: -// -// fmt.Sprint(c.NewFormatter(a), c.NewFormatter(b)) -func (c *ConfigState) Sprint(a ...interface{}) string { - return fmt.Sprint(c.convertArgs(a)...) -} - -// Sprintf is a wrapper for fmt.Sprintf that treats each argument as if it were -// passed with a Formatter interface returned by c.NewFormatter. It returns -// the resulting string. See NewFormatter for formatting details. -// -// This function is shorthand for the following syntax: -// -// fmt.Sprintf(format, c.NewFormatter(a), c.NewFormatter(b)) -func (c *ConfigState) Sprintf(format string, a ...interface{}) string { - return fmt.Sprintf(format, c.convertArgs(a)...) -} - -// Sprintln is a wrapper for fmt.Sprintln that treats each argument as if it -// were passed with a Formatter interface returned by c.NewFormatter. It -// returns the resulting string. See NewFormatter for formatting details. -// -// This function is shorthand for the following syntax: -// -// fmt.Sprintln(c.NewFormatter(a), c.NewFormatter(b)) -func (c *ConfigState) Sprintln(a ...interface{}) string { - return fmt.Sprintln(c.convertArgs(a)...) -} - -/* -NewFormatter returns a custom formatter that satisfies the fmt.Formatter -interface. As a result, it integrates cleanly with standard fmt package -printing functions. The formatter is useful for inline printing of smaller data -types similar to the standard %v format specifier. - -The custom formatter only responds to the %v (most compact), %+v (adds pointer -addresses), %#v (adds types), and %#+v (adds types and pointer addresses) verb -combinations. Any other verbs such as %x and %q will be sent to the the -standard fmt package for formatting. In addition, the custom formatter ignores -the width and precision arguments (however they will still work on the format -specifiers not handled by the custom formatter). - -Typically this function shouldn't be called directly. It is much easier to make -use of the custom formatter by calling one of the convenience functions such as -c.Printf, c.Println, or c.Printf. -*/ -func (c *ConfigState) NewFormatter(v interface{}) fmt.Formatter { - return newFormatter(c, v) -} - -// Fdump formats and displays the passed arguments to io.Writer w. It formats -// exactly the same as Dump. -func (c *ConfigState) Fdump(w io.Writer, a ...interface{}) { - fdump(c, w, a...) -} - -/* -Dump displays the passed parameters to standard out with newlines, customizable -indentation, and additional debug information such as complete types and all -pointer addresses used to indirect to the final value. It provides the -following features over the built-in printing facilities provided by the fmt -package: - - * Pointers are dereferenced and followed - * Circular data structures are detected and handled properly - * Custom Stringer/error interfaces are optionally invoked, including - on unexported types - * Custom types which only implement the Stringer/error interfaces via - a pointer receiver are optionally invoked when passing non-pointer - variables - * Byte arrays and slices are dumped like the hexdump -C command which - includes offsets, byte values in hex, and ASCII output - -The configuration options are controlled by modifying the public members -of c. See ConfigState for options documentation. - -See Fdump if you would prefer dumping to an arbitrary io.Writer or Sdump to -get the formatted result as a string. -*/ -func (c *ConfigState) Dump(a ...interface{}) { - fdump(c, os.Stdout, a...) -} - -// Sdump returns a string with the passed arguments formatted exactly the same -// as Dump. -func (c *ConfigState) Sdump(a ...interface{}) string { - var buf bytes.Buffer - fdump(c, &buf, a...) - return buf.String() -} - -// convertArgs accepts a slice of arguments and returns a slice of the same -// length with each argument converted to a spew Formatter interface using -// the ConfigState associated with s. -func (c *ConfigState) convertArgs(args []interface{}) (formatters []interface{}) { - formatters = make([]interface{}, len(args)) - for index, arg := range args { - formatters[index] = newFormatter(c, arg) - } - return formatters -} - -// NewDefaultConfig returns a ConfigState with the following default settings. -// -// Indent: " " -// MaxDepth: 0 -// DisableMethods: false -// DisablePointerMethods: false -// ContinueOnMethod: false -// SortKeys: false -func NewDefaultConfig() *ConfigState { - return &ConfigState{Indent: " "} -} diff --git a/vendor/github.com/davecgh/go-spew/spew/doc.go b/vendor/github.com/davecgh/go-spew/spew/doc.go deleted file mode 100644 index aacaac6f..00000000 --- a/vendor/github.com/davecgh/go-spew/spew/doc.go +++ /dev/null @@ -1,211 +0,0 @@ -/* - * Copyright (c) 2013-2016 Dave Collins - * - * Permission to use, copy, modify, and distribute this software for any - * purpose with or without fee is hereby granted, provided that the above - * copyright notice and this permission notice appear in all copies. - * - * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - */ - -/* -Package spew implements a deep pretty printer for Go data structures to aid in -debugging. - -A quick overview of the additional features spew provides over the built-in -printing facilities for Go data types are as follows: - - * Pointers are dereferenced and followed - * Circular data structures are detected and handled properly - * Custom Stringer/error interfaces are optionally invoked, including - on unexported types - * Custom types which only implement the Stringer/error interfaces via - a pointer receiver are optionally invoked when passing non-pointer - variables - * Byte arrays and slices are dumped like the hexdump -C command which - includes offsets, byte values in hex, and ASCII output (only when using - Dump style) - -There are two different approaches spew allows for dumping Go data structures: - - * Dump style which prints with newlines, customizable indentation, - and additional debug information such as types and all pointer addresses - used to indirect to the final value - * A custom Formatter interface that integrates cleanly with the standard fmt - package and replaces %v, %+v, %#v, and %#+v to provide inline printing - similar to the default %v while providing the additional functionality - outlined above and passing unsupported format verbs such as %x and %q - along to fmt - -Quick Start - -This section demonstrates how to quickly get started with spew. See the -sections below for further details on formatting and configuration options. - -To dump a variable with full newlines, indentation, type, and pointer -information use Dump, Fdump, or Sdump: - spew.Dump(myVar1, myVar2, ...) - spew.Fdump(someWriter, myVar1, myVar2, ...) - str := spew.Sdump(myVar1, myVar2, ...) - -Alternatively, if you would prefer to use format strings with a compacted inline -printing style, use the convenience wrappers Printf, Fprintf, etc with -%v (most compact), %+v (adds pointer addresses), %#v (adds types), or -%#+v (adds types and pointer addresses): - spew.Printf("myVar1: %v -- myVar2: %+v", myVar1, myVar2) - spew.Printf("myVar3: %#v -- myVar4: %#+v", myVar3, myVar4) - spew.Fprintf(someWriter, "myVar1: %v -- myVar2: %+v", myVar1, myVar2) - spew.Fprintf(someWriter, "myVar3: %#v -- myVar4: %#+v", myVar3, myVar4) - -Configuration Options - -Configuration of spew is handled by fields in the ConfigState type. For -convenience, all of the top-level functions use a global state available -via the spew.Config global. - -It is also possible to create a ConfigState instance that provides methods -equivalent to the top-level functions. This allows concurrent configuration -options. See the ConfigState documentation for more details. - -The following configuration options are available: - * Indent - String to use for each indentation level for Dump functions. - It is a single space by default. A popular alternative is "\t". - - * MaxDepth - Maximum number of levels to descend into nested data structures. - There is no limit by default. - - * DisableMethods - Disables invocation of error and Stringer interface methods. - Method invocation is enabled by default. - - * DisablePointerMethods - Disables invocation of error and Stringer interface methods on types - which only accept pointer receivers from non-pointer variables. - Pointer method invocation is enabled by default. - - * DisablePointerAddresses - DisablePointerAddresses specifies whether to disable the printing of - pointer addresses. This is useful when diffing data structures in tests. - - * DisableCapacities - DisableCapacities specifies whether to disable the printing of - capacities for arrays, slices, maps and channels. This is useful when - diffing data structures in tests. - - * ContinueOnMethod - Enables recursion into types after invoking error and Stringer interface - methods. Recursion after method invocation is disabled by default. - - * SortKeys - Specifies map keys should be sorted before being printed. Use - this to have a more deterministic, diffable output. Note that - only native types (bool, int, uint, floats, uintptr and string) - and types which implement error or Stringer interfaces are - supported with other types sorted according to the - reflect.Value.String() output which guarantees display - stability. Natural map order is used by default. - - * SpewKeys - Specifies that, as a last resort attempt, map keys should be - spewed to strings and sorted by those strings. This is only - considered if SortKeys is true. - -Dump Usage - -Simply call spew.Dump with a list of variables you want to dump: - - spew.Dump(myVar1, myVar2, ...) - -You may also call spew.Fdump if you would prefer to output to an arbitrary -io.Writer. For example, to dump to standard error: - - spew.Fdump(os.Stderr, myVar1, myVar2, ...) - -A third option is to call spew.Sdump to get the formatted output as a string: - - str := spew.Sdump(myVar1, myVar2, ...) - -Sample Dump Output - -See the Dump example for details on the setup of the types and variables being -shown here. - - (main.Foo) { - unexportedField: (*main.Bar)(0xf84002e210)({ - flag: (main.Flag) flagTwo, - data: (uintptr) - }), - ExportedField: (map[interface {}]interface {}) (len=1) { - (string) (len=3) "one": (bool) true - } - } - -Byte (and uint8) arrays and slices are displayed uniquely like the hexdump -C -command as shown. - ([]uint8) (len=32 cap=32) { - 00000000 11 12 13 14 15 16 17 18 19 1a 1b 1c 1d 1e 1f 20 |............... | - 00000010 21 22 23 24 25 26 27 28 29 2a 2b 2c 2d 2e 2f 30 |!"#$%&'()*+,-./0| - 00000020 31 32 |12| - } - -Custom Formatter - -Spew provides a custom formatter that implements the fmt.Formatter interface -so that it integrates cleanly with standard fmt package printing functions. The -formatter is useful for inline printing of smaller data types similar to the -standard %v format specifier. - -The custom formatter only responds to the %v (most compact), %+v (adds pointer -addresses), %#v (adds types), or %#+v (adds types and pointer addresses) verb -combinations. Any other verbs such as %x and %q will be sent to the the -standard fmt package for formatting. In addition, the custom formatter ignores -the width and precision arguments (however they will still work on the format -specifiers not handled by the custom formatter). - -Custom Formatter Usage - -The simplest way to make use of the spew custom formatter is to call one of the -convenience functions such as spew.Printf, spew.Println, or spew.Printf. The -functions have syntax you are most likely already familiar with: - - spew.Printf("myVar1: %v -- myVar2: %+v", myVar1, myVar2) - spew.Printf("myVar3: %#v -- myVar4: %#+v", myVar3, myVar4) - spew.Println(myVar, myVar2) - spew.Fprintf(os.Stderr, "myVar1: %v -- myVar2: %+v", myVar1, myVar2) - spew.Fprintf(os.Stderr, "myVar3: %#v -- myVar4: %#+v", myVar3, myVar4) - -See the Index for the full list convenience functions. - -Sample Formatter Output - -Double pointer to a uint8: - %v: <**>5 - %+v: <**>(0xf8400420d0->0xf8400420c8)5 - %#v: (**uint8)5 - %#+v: (**uint8)(0xf8400420d0->0xf8400420c8)5 - -Pointer to circular struct with a uint8 field and a pointer to itself: - %v: <*>{1 <*>} - %+v: <*>(0xf84003e260){ui8:1 c:<*>(0xf84003e260)} - %#v: (*main.circular){ui8:(uint8)1 c:(*main.circular)} - %#+v: (*main.circular)(0xf84003e260){ui8:(uint8)1 c:(*main.circular)(0xf84003e260)} - -See the Printf example for details on the setup of variables being shown -here. - -Errors - -Since it is possible for custom Stringer/error interfaces to panic, spew -detects them and handles them internally by printing the panic information -inline with the output. Since spew is intended to provide deep pretty printing -capabilities on structures, it intentionally does not return any errors. -*/ -package spew diff --git a/vendor/github.com/davecgh/go-spew/spew/dump.go b/vendor/github.com/davecgh/go-spew/spew/dump.go deleted file mode 100644 index f78d89fc..00000000 --- a/vendor/github.com/davecgh/go-spew/spew/dump.go +++ /dev/null @@ -1,509 +0,0 @@ -/* - * Copyright (c) 2013-2016 Dave Collins - * - * Permission to use, copy, modify, and distribute this software for any - * purpose with or without fee is hereby granted, provided that the above - * copyright notice and this permission notice appear in all copies. - * - * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - */ - -package spew - -import ( - "bytes" - "encoding/hex" - "fmt" - "io" - "os" - "reflect" - "regexp" - "strconv" - "strings" -) - -var ( - // uint8Type is a reflect.Type representing a uint8. It is used to - // convert cgo types to uint8 slices for hexdumping. - uint8Type = reflect.TypeOf(uint8(0)) - - // cCharRE is a regular expression that matches a cgo char. - // It is used to detect character arrays to hexdump them. - cCharRE = regexp.MustCompile(`^.*\._Ctype_char$`) - - // cUnsignedCharRE is a regular expression that matches a cgo unsigned - // char. It is used to detect unsigned character arrays to hexdump - // them. - cUnsignedCharRE = regexp.MustCompile(`^.*\._Ctype_unsignedchar$`) - - // cUint8tCharRE is a regular expression that matches a cgo uint8_t. - // It is used to detect uint8_t arrays to hexdump them. - cUint8tCharRE = regexp.MustCompile(`^.*\._Ctype_uint8_t$`) -) - -// dumpState contains information about the state of a dump operation. -type dumpState struct { - w io.Writer - depth int - pointers map[uintptr]int - ignoreNextType bool - ignoreNextIndent bool - cs *ConfigState -} - -// indent performs indentation according to the depth level and cs.Indent -// option. -func (d *dumpState) indent() { - if d.ignoreNextIndent { - d.ignoreNextIndent = false - return - } - d.w.Write(bytes.Repeat([]byte(d.cs.Indent), d.depth)) -} - -// unpackValue returns values inside of non-nil interfaces when possible. -// This is useful for data types like structs, arrays, slices, and maps which -// can contain varying types packed inside an interface. -func (d *dumpState) unpackValue(v reflect.Value) reflect.Value { - if v.Kind() == reflect.Interface && !v.IsNil() { - v = v.Elem() - } - return v -} - -// dumpPtr handles formatting of pointers by indirecting them as necessary. -func (d *dumpState) dumpPtr(v reflect.Value) { - // Remove pointers at or below the current depth from map used to detect - // circular refs. - for k, depth := range d.pointers { - if depth >= d.depth { - delete(d.pointers, k) - } - } - - // Keep list of all dereferenced pointers to show later. - pointerChain := make([]uintptr, 0) - - // Figure out how many levels of indirection there are by dereferencing - // pointers and unpacking interfaces down the chain while detecting circular - // references. - nilFound := false - cycleFound := false - indirects := 0 - ve := v - for ve.Kind() == reflect.Ptr { - if ve.IsNil() { - nilFound = true - break - } - indirects++ - addr := ve.Pointer() - pointerChain = append(pointerChain, addr) - if pd, ok := d.pointers[addr]; ok && pd < d.depth { - cycleFound = true - indirects-- - break - } - d.pointers[addr] = d.depth - - ve = ve.Elem() - if ve.Kind() == reflect.Interface { - if ve.IsNil() { - nilFound = true - break - } - ve = ve.Elem() - } - } - - // Display type information. - d.w.Write(openParenBytes) - d.w.Write(bytes.Repeat(asteriskBytes, indirects)) - d.w.Write([]byte(ve.Type().String())) - d.w.Write(closeParenBytes) - - // Display pointer information. - if !d.cs.DisablePointerAddresses && len(pointerChain) > 0 { - d.w.Write(openParenBytes) - for i, addr := range pointerChain { - if i > 0 { - d.w.Write(pointerChainBytes) - } - printHexPtr(d.w, addr) - } - d.w.Write(closeParenBytes) - } - - // Display dereferenced value. - d.w.Write(openParenBytes) - switch { - case nilFound: - d.w.Write(nilAngleBytes) - - case cycleFound: - d.w.Write(circularBytes) - - default: - d.ignoreNextType = true - d.dump(ve) - } - d.w.Write(closeParenBytes) -} - -// dumpSlice handles formatting of arrays and slices. Byte (uint8 under -// reflection) arrays and slices are dumped in hexdump -C fashion. -func (d *dumpState) dumpSlice(v reflect.Value) { - // Determine whether this type should be hex dumped or not. Also, - // for types which should be hexdumped, try to use the underlying data - // first, then fall back to trying to convert them to a uint8 slice. - var buf []uint8 - doConvert := false - doHexDump := false - numEntries := v.Len() - if numEntries > 0 { - vt := v.Index(0).Type() - vts := vt.String() - switch { - // C types that need to be converted. - case cCharRE.MatchString(vts): - fallthrough - case cUnsignedCharRE.MatchString(vts): - fallthrough - case cUint8tCharRE.MatchString(vts): - doConvert = true - - // Try to use existing uint8 slices and fall back to converting - // and copying if that fails. - case vt.Kind() == reflect.Uint8: - // We need an addressable interface to convert the type - // to a byte slice. However, the reflect package won't - // give us an interface on certain things like - // unexported struct fields in order to enforce - // visibility rules. We use unsafe, when available, to - // bypass these restrictions since this package does not - // mutate the values. - vs := v - if !vs.CanInterface() || !vs.CanAddr() { - vs = unsafeReflectValue(vs) - } - if !UnsafeDisabled { - vs = vs.Slice(0, numEntries) - - // Use the existing uint8 slice if it can be - // type asserted. - iface := vs.Interface() - if slice, ok := iface.([]uint8); ok { - buf = slice - doHexDump = true - break - } - } - - // The underlying data needs to be converted if it can't - // be type asserted to a uint8 slice. - doConvert = true - } - - // Copy and convert the underlying type if needed. - if doConvert && vt.ConvertibleTo(uint8Type) { - // Convert and copy each element into a uint8 byte - // slice. - buf = make([]uint8, numEntries) - for i := 0; i < numEntries; i++ { - vv := v.Index(i) - buf[i] = uint8(vv.Convert(uint8Type).Uint()) - } - doHexDump = true - } - } - - // Hexdump the entire slice as needed. - if doHexDump { - indent := strings.Repeat(d.cs.Indent, d.depth) - str := indent + hex.Dump(buf) - str = strings.Replace(str, "\n", "\n"+indent, -1) - str = strings.TrimRight(str, d.cs.Indent) - d.w.Write([]byte(str)) - return - } - - // Recursively call dump for each item. - for i := 0; i < numEntries; i++ { - d.dump(d.unpackValue(v.Index(i))) - if i < (numEntries - 1) { - d.w.Write(commaNewlineBytes) - } else { - d.w.Write(newlineBytes) - } - } -} - -// dump is the main workhorse for dumping a value. It uses the passed reflect -// value to figure out what kind of object we are dealing with and formats it -// appropriately. It is a recursive function, however circular data structures -// are detected and handled properly. -func (d *dumpState) dump(v reflect.Value) { - // Handle invalid reflect values immediately. - kind := v.Kind() - if kind == reflect.Invalid { - d.w.Write(invalidAngleBytes) - return - } - - // Handle pointers specially. - if kind == reflect.Ptr { - d.indent() - d.dumpPtr(v) - return - } - - // Print type information unless already handled elsewhere. - if !d.ignoreNextType { - d.indent() - d.w.Write(openParenBytes) - d.w.Write([]byte(v.Type().String())) - d.w.Write(closeParenBytes) - d.w.Write(spaceBytes) - } - d.ignoreNextType = false - - // Display length and capacity if the built-in len and cap functions - // work with the value's kind and the len/cap itself is non-zero. - valueLen, valueCap := 0, 0 - switch v.Kind() { - case reflect.Array, reflect.Slice, reflect.Chan: - valueLen, valueCap = v.Len(), v.Cap() - case reflect.Map, reflect.String: - valueLen = v.Len() - } - if valueLen != 0 || !d.cs.DisableCapacities && valueCap != 0 { - d.w.Write(openParenBytes) - if valueLen != 0 { - d.w.Write(lenEqualsBytes) - printInt(d.w, int64(valueLen), 10) - } - if !d.cs.DisableCapacities && valueCap != 0 { - if valueLen != 0 { - d.w.Write(spaceBytes) - } - d.w.Write(capEqualsBytes) - printInt(d.w, int64(valueCap), 10) - } - d.w.Write(closeParenBytes) - d.w.Write(spaceBytes) - } - - // Call Stringer/error interfaces if they exist and the handle methods flag - // is enabled - if !d.cs.DisableMethods { - if (kind != reflect.Invalid) && (kind != reflect.Interface) { - if handled := handleMethods(d.cs, d.w, v); handled { - return - } - } - } - - switch kind { - case reflect.Invalid: - // Do nothing. We should never get here since invalid has already - // been handled above. - - case reflect.Bool: - printBool(d.w, v.Bool()) - - case reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Int: - printInt(d.w, v.Int(), 10) - - case reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uint: - printUint(d.w, v.Uint(), 10) - - case reflect.Float32: - printFloat(d.w, v.Float(), 32) - - case reflect.Float64: - printFloat(d.w, v.Float(), 64) - - case reflect.Complex64: - printComplex(d.w, v.Complex(), 32) - - case reflect.Complex128: - printComplex(d.w, v.Complex(), 64) - - case reflect.Slice: - if v.IsNil() { - d.w.Write(nilAngleBytes) - break - } - fallthrough - - case reflect.Array: - d.w.Write(openBraceNewlineBytes) - d.depth++ - if (d.cs.MaxDepth != 0) && (d.depth > d.cs.MaxDepth) { - d.indent() - d.w.Write(maxNewlineBytes) - } else { - d.dumpSlice(v) - } - d.depth-- - d.indent() - d.w.Write(closeBraceBytes) - - case reflect.String: - d.w.Write([]byte(strconv.Quote(v.String()))) - - case reflect.Interface: - // The only time we should get here is for nil interfaces due to - // unpackValue calls. - if v.IsNil() { - d.w.Write(nilAngleBytes) - } - - case reflect.Ptr: - // Do nothing. We should never get here since pointers have already - // been handled above. - - case reflect.Map: - // nil maps should be indicated as different than empty maps - if v.IsNil() { - d.w.Write(nilAngleBytes) - break - } - - d.w.Write(openBraceNewlineBytes) - d.depth++ - if (d.cs.MaxDepth != 0) && (d.depth > d.cs.MaxDepth) { - d.indent() - d.w.Write(maxNewlineBytes) - } else { - numEntries := v.Len() - keys := v.MapKeys() - if d.cs.SortKeys { - sortValues(keys, d.cs) - } - for i, key := range keys { - d.dump(d.unpackValue(key)) - d.w.Write(colonSpaceBytes) - d.ignoreNextIndent = true - d.dump(d.unpackValue(v.MapIndex(key))) - if i < (numEntries - 1) { - d.w.Write(commaNewlineBytes) - } else { - d.w.Write(newlineBytes) - } - } - } - d.depth-- - d.indent() - d.w.Write(closeBraceBytes) - - case reflect.Struct: - d.w.Write(openBraceNewlineBytes) - d.depth++ - if (d.cs.MaxDepth != 0) && (d.depth > d.cs.MaxDepth) { - d.indent() - d.w.Write(maxNewlineBytes) - } else { - vt := v.Type() - numFields := v.NumField() - for i := 0; i < numFields; i++ { - d.indent() - vtf := vt.Field(i) - d.w.Write([]byte(vtf.Name)) - d.w.Write(colonSpaceBytes) - d.ignoreNextIndent = true - d.dump(d.unpackValue(v.Field(i))) - if i < (numFields - 1) { - d.w.Write(commaNewlineBytes) - } else { - d.w.Write(newlineBytes) - } - } - } - d.depth-- - d.indent() - d.w.Write(closeBraceBytes) - - case reflect.Uintptr: - printHexPtr(d.w, uintptr(v.Uint())) - - case reflect.UnsafePointer, reflect.Chan, reflect.Func: - printHexPtr(d.w, v.Pointer()) - - // There were not any other types at the time this code was written, but - // fall back to letting the default fmt package handle it in case any new - // types are added. - default: - if v.CanInterface() { - fmt.Fprintf(d.w, "%v", v.Interface()) - } else { - fmt.Fprintf(d.w, "%v", v.String()) - } - } -} - -// fdump is a helper function to consolidate the logic from the various public -// methods which take varying writers and config states. -func fdump(cs *ConfigState, w io.Writer, a ...interface{}) { - for _, arg := range a { - if arg == nil { - w.Write(interfaceBytes) - w.Write(spaceBytes) - w.Write(nilAngleBytes) - w.Write(newlineBytes) - continue - } - - d := dumpState{w: w, cs: cs} - d.pointers = make(map[uintptr]int) - d.dump(reflect.ValueOf(arg)) - d.w.Write(newlineBytes) - } -} - -// Fdump formats and displays the passed arguments to io.Writer w. It formats -// exactly the same as Dump. -func Fdump(w io.Writer, a ...interface{}) { - fdump(&Config, w, a...) -} - -// Sdump returns a string with the passed arguments formatted exactly the same -// as Dump. -func Sdump(a ...interface{}) string { - var buf bytes.Buffer - fdump(&Config, &buf, a...) - return buf.String() -} - -/* -Dump displays the passed parameters to standard out with newlines, customizable -indentation, and additional debug information such as complete types and all -pointer addresses used to indirect to the final value. It provides the -following features over the built-in printing facilities provided by the fmt -package: - - * Pointers are dereferenced and followed - * Circular data structures are detected and handled properly - * Custom Stringer/error interfaces are optionally invoked, including - on unexported types - * Custom types which only implement the Stringer/error interfaces via - a pointer receiver are optionally invoked when passing non-pointer - variables - * Byte arrays and slices are dumped like the hexdump -C command which - includes offsets, byte values in hex, and ASCII output - -The configuration options are controlled by an exported package global, -spew.Config. See ConfigState for options documentation. - -See Fdump if you would prefer dumping to an arbitrary io.Writer or Sdump to -get the formatted result as a string. -*/ -func Dump(a ...interface{}) { - fdump(&Config, os.Stdout, a...) -} diff --git a/vendor/github.com/davecgh/go-spew/spew/format.go b/vendor/github.com/davecgh/go-spew/spew/format.go deleted file mode 100644 index b04edb7d..00000000 --- a/vendor/github.com/davecgh/go-spew/spew/format.go +++ /dev/null @@ -1,419 +0,0 @@ -/* - * Copyright (c) 2013-2016 Dave Collins - * - * Permission to use, copy, modify, and distribute this software for any - * purpose with or without fee is hereby granted, provided that the above - * copyright notice and this permission notice appear in all copies. - * - * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - */ - -package spew - -import ( - "bytes" - "fmt" - "reflect" - "strconv" - "strings" -) - -// supportedFlags is a list of all the character flags supported by fmt package. -const supportedFlags = "0-+# " - -// formatState implements the fmt.Formatter interface and contains information -// about the state of a formatting operation. The NewFormatter function can -// be used to get a new Formatter which can be used directly as arguments -// in standard fmt package printing calls. -type formatState struct { - value interface{} - fs fmt.State - depth int - pointers map[uintptr]int - ignoreNextType bool - cs *ConfigState -} - -// buildDefaultFormat recreates the original format string without precision -// and width information to pass in to fmt.Sprintf in the case of an -// unrecognized type. Unless new types are added to the language, this -// function won't ever be called. -func (f *formatState) buildDefaultFormat() (format string) { - buf := bytes.NewBuffer(percentBytes) - - for _, flag := range supportedFlags { - if f.fs.Flag(int(flag)) { - buf.WriteRune(flag) - } - } - - buf.WriteRune('v') - - format = buf.String() - return format -} - -// constructOrigFormat recreates the original format string including precision -// and width information to pass along to the standard fmt package. This allows -// automatic deferral of all format strings this package doesn't support. -func (f *formatState) constructOrigFormat(verb rune) (format string) { - buf := bytes.NewBuffer(percentBytes) - - for _, flag := range supportedFlags { - if f.fs.Flag(int(flag)) { - buf.WriteRune(flag) - } - } - - if width, ok := f.fs.Width(); ok { - buf.WriteString(strconv.Itoa(width)) - } - - if precision, ok := f.fs.Precision(); ok { - buf.Write(precisionBytes) - buf.WriteString(strconv.Itoa(precision)) - } - - buf.WriteRune(verb) - - format = buf.String() - return format -} - -// unpackValue returns values inside of non-nil interfaces when possible and -// ensures that types for values which have been unpacked from an interface -// are displayed when the show types flag is also set. -// This is useful for data types like structs, arrays, slices, and maps which -// can contain varying types packed inside an interface. -func (f *formatState) unpackValue(v reflect.Value) reflect.Value { - if v.Kind() == reflect.Interface { - f.ignoreNextType = false - if !v.IsNil() { - v = v.Elem() - } - } - return v -} - -// formatPtr handles formatting of pointers by indirecting them as necessary. -func (f *formatState) formatPtr(v reflect.Value) { - // Display nil if top level pointer is nil. - showTypes := f.fs.Flag('#') - if v.IsNil() && (!showTypes || f.ignoreNextType) { - f.fs.Write(nilAngleBytes) - return - } - - // Remove pointers at or below the current depth from map used to detect - // circular refs. - for k, depth := range f.pointers { - if depth >= f.depth { - delete(f.pointers, k) - } - } - - // Keep list of all dereferenced pointers to possibly show later. - pointerChain := make([]uintptr, 0) - - // Figure out how many levels of indirection there are by derferencing - // pointers and unpacking interfaces down the chain while detecting circular - // references. - nilFound := false - cycleFound := false - indirects := 0 - ve := v - for ve.Kind() == reflect.Ptr { - if ve.IsNil() { - nilFound = true - break - } - indirects++ - addr := ve.Pointer() - pointerChain = append(pointerChain, addr) - if pd, ok := f.pointers[addr]; ok && pd < f.depth { - cycleFound = true - indirects-- - break - } - f.pointers[addr] = f.depth - - ve = ve.Elem() - if ve.Kind() == reflect.Interface { - if ve.IsNil() { - nilFound = true - break - } - ve = ve.Elem() - } - } - - // Display type or indirection level depending on flags. - if showTypes && !f.ignoreNextType { - f.fs.Write(openParenBytes) - f.fs.Write(bytes.Repeat(asteriskBytes, indirects)) - f.fs.Write([]byte(ve.Type().String())) - f.fs.Write(closeParenBytes) - } else { - if nilFound || cycleFound { - indirects += strings.Count(ve.Type().String(), "*") - } - f.fs.Write(openAngleBytes) - f.fs.Write([]byte(strings.Repeat("*", indirects))) - f.fs.Write(closeAngleBytes) - } - - // Display pointer information depending on flags. - if f.fs.Flag('+') && (len(pointerChain) > 0) { - f.fs.Write(openParenBytes) - for i, addr := range pointerChain { - if i > 0 { - f.fs.Write(pointerChainBytes) - } - printHexPtr(f.fs, addr) - } - f.fs.Write(closeParenBytes) - } - - // Display dereferenced value. - switch { - case nilFound: - f.fs.Write(nilAngleBytes) - - case cycleFound: - f.fs.Write(circularShortBytes) - - default: - f.ignoreNextType = true - f.format(ve) - } -} - -// format is the main workhorse for providing the Formatter interface. It -// uses the passed reflect value to figure out what kind of object we are -// dealing with and formats it appropriately. It is a recursive function, -// however circular data structures are detected and handled properly. -func (f *formatState) format(v reflect.Value) { - // Handle invalid reflect values immediately. - kind := v.Kind() - if kind == reflect.Invalid { - f.fs.Write(invalidAngleBytes) - return - } - - // Handle pointers specially. - if kind == reflect.Ptr { - f.formatPtr(v) - return - } - - // Print type information unless already handled elsewhere. - if !f.ignoreNextType && f.fs.Flag('#') { - f.fs.Write(openParenBytes) - f.fs.Write([]byte(v.Type().String())) - f.fs.Write(closeParenBytes) - } - f.ignoreNextType = false - - // Call Stringer/error interfaces if they exist and the handle methods - // flag is enabled. - if !f.cs.DisableMethods { - if (kind != reflect.Invalid) && (kind != reflect.Interface) { - if handled := handleMethods(f.cs, f.fs, v); handled { - return - } - } - } - - switch kind { - case reflect.Invalid: - // Do nothing. We should never get here since invalid has already - // been handled above. - - case reflect.Bool: - printBool(f.fs, v.Bool()) - - case reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Int: - printInt(f.fs, v.Int(), 10) - - case reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uint: - printUint(f.fs, v.Uint(), 10) - - case reflect.Float32: - printFloat(f.fs, v.Float(), 32) - - case reflect.Float64: - printFloat(f.fs, v.Float(), 64) - - case reflect.Complex64: - printComplex(f.fs, v.Complex(), 32) - - case reflect.Complex128: - printComplex(f.fs, v.Complex(), 64) - - case reflect.Slice: - if v.IsNil() { - f.fs.Write(nilAngleBytes) - break - } - fallthrough - - case reflect.Array: - f.fs.Write(openBracketBytes) - f.depth++ - if (f.cs.MaxDepth != 0) && (f.depth > f.cs.MaxDepth) { - f.fs.Write(maxShortBytes) - } else { - numEntries := v.Len() - for i := 0; i < numEntries; i++ { - if i > 0 { - f.fs.Write(spaceBytes) - } - f.ignoreNextType = true - f.format(f.unpackValue(v.Index(i))) - } - } - f.depth-- - f.fs.Write(closeBracketBytes) - - case reflect.String: - f.fs.Write([]byte(v.String())) - - case reflect.Interface: - // The only time we should get here is for nil interfaces due to - // unpackValue calls. - if v.IsNil() { - f.fs.Write(nilAngleBytes) - } - - case reflect.Ptr: - // Do nothing. We should never get here since pointers have already - // been handled above. - - case reflect.Map: - // nil maps should be indicated as different than empty maps - if v.IsNil() { - f.fs.Write(nilAngleBytes) - break - } - - f.fs.Write(openMapBytes) - f.depth++ - if (f.cs.MaxDepth != 0) && (f.depth > f.cs.MaxDepth) { - f.fs.Write(maxShortBytes) - } else { - keys := v.MapKeys() - if f.cs.SortKeys { - sortValues(keys, f.cs) - } - for i, key := range keys { - if i > 0 { - f.fs.Write(spaceBytes) - } - f.ignoreNextType = true - f.format(f.unpackValue(key)) - f.fs.Write(colonBytes) - f.ignoreNextType = true - f.format(f.unpackValue(v.MapIndex(key))) - } - } - f.depth-- - f.fs.Write(closeMapBytes) - - case reflect.Struct: - numFields := v.NumField() - f.fs.Write(openBraceBytes) - f.depth++ - if (f.cs.MaxDepth != 0) && (f.depth > f.cs.MaxDepth) { - f.fs.Write(maxShortBytes) - } else { - vt := v.Type() - for i := 0; i < numFields; i++ { - if i > 0 { - f.fs.Write(spaceBytes) - } - vtf := vt.Field(i) - if f.fs.Flag('+') || f.fs.Flag('#') { - f.fs.Write([]byte(vtf.Name)) - f.fs.Write(colonBytes) - } - f.format(f.unpackValue(v.Field(i))) - } - } - f.depth-- - f.fs.Write(closeBraceBytes) - - case reflect.Uintptr: - printHexPtr(f.fs, uintptr(v.Uint())) - - case reflect.UnsafePointer, reflect.Chan, reflect.Func: - printHexPtr(f.fs, v.Pointer()) - - // There were not any other types at the time this code was written, but - // fall back to letting the default fmt package handle it if any get added. - default: - format := f.buildDefaultFormat() - if v.CanInterface() { - fmt.Fprintf(f.fs, format, v.Interface()) - } else { - fmt.Fprintf(f.fs, format, v.String()) - } - } -} - -// Format satisfies the fmt.Formatter interface. See NewFormatter for usage -// details. -func (f *formatState) Format(fs fmt.State, verb rune) { - f.fs = fs - - // Use standard formatting for verbs that are not v. - if verb != 'v' { - format := f.constructOrigFormat(verb) - fmt.Fprintf(fs, format, f.value) - return - } - - if f.value == nil { - if fs.Flag('#') { - fs.Write(interfaceBytes) - } - fs.Write(nilAngleBytes) - return - } - - f.format(reflect.ValueOf(f.value)) -} - -// newFormatter is a helper function to consolidate the logic from the various -// public methods which take varying config states. -func newFormatter(cs *ConfigState, v interface{}) fmt.Formatter { - fs := &formatState{value: v, cs: cs} - fs.pointers = make(map[uintptr]int) - return fs -} - -/* -NewFormatter returns a custom formatter that satisfies the fmt.Formatter -interface. As a result, it integrates cleanly with standard fmt package -printing functions. The formatter is useful for inline printing of smaller data -types similar to the standard %v format specifier. - -The custom formatter only responds to the %v (most compact), %+v (adds pointer -addresses), %#v (adds types), or %#+v (adds types and pointer addresses) verb -combinations. Any other verbs such as %x and %q will be sent to the the -standard fmt package for formatting. In addition, the custom formatter ignores -the width and precision arguments (however they will still work on the format -specifiers not handled by the custom formatter). - -Typically this function shouldn't be called directly. It is much easier to make -use of the custom formatter by calling one of the convenience functions such as -Printf, Println, or Fprintf. -*/ -func NewFormatter(v interface{}) fmt.Formatter { - return newFormatter(&Config, v) -} diff --git a/vendor/github.com/davecgh/go-spew/spew/spew.go b/vendor/github.com/davecgh/go-spew/spew/spew.go deleted file mode 100644 index 32c0e338..00000000 --- a/vendor/github.com/davecgh/go-spew/spew/spew.go +++ /dev/null @@ -1,148 +0,0 @@ -/* - * Copyright (c) 2013-2016 Dave Collins - * - * Permission to use, copy, modify, and distribute this software for any - * purpose with or without fee is hereby granted, provided that the above - * copyright notice and this permission notice appear in all copies. - * - * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR - * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF - * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - */ - -package spew - -import ( - "fmt" - "io" -) - -// Errorf is a wrapper for fmt.Errorf that treats each argument as if it were -// passed with a default Formatter interface returned by NewFormatter. It -// returns the formatted string as a value that satisfies error. See -// NewFormatter for formatting details. -// -// This function is shorthand for the following syntax: -// -// fmt.Errorf(format, spew.NewFormatter(a), spew.NewFormatter(b)) -func Errorf(format string, a ...interface{}) (err error) { - return fmt.Errorf(format, convertArgs(a)...) -} - -// Fprint is a wrapper for fmt.Fprint that treats each argument as if it were -// passed with a default Formatter interface returned by NewFormatter. It -// returns the number of bytes written and any write error encountered. See -// NewFormatter for formatting details. -// -// This function is shorthand for the following syntax: -// -// fmt.Fprint(w, spew.NewFormatter(a), spew.NewFormatter(b)) -func Fprint(w io.Writer, a ...interface{}) (n int, err error) { - return fmt.Fprint(w, convertArgs(a)...) -} - -// Fprintf is a wrapper for fmt.Fprintf that treats each argument as if it were -// passed with a default Formatter interface returned by NewFormatter. It -// returns the number of bytes written and any write error encountered. See -// NewFormatter for formatting details. -// -// This function is shorthand for the following syntax: -// -// fmt.Fprintf(w, format, spew.NewFormatter(a), spew.NewFormatter(b)) -func Fprintf(w io.Writer, format string, a ...interface{}) (n int, err error) { - return fmt.Fprintf(w, format, convertArgs(a)...) -} - -// Fprintln is a wrapper for fmt.Fprintln that treats each argument as if it -// passed with a default Formatter interface returned by NewFormatter. See -// NewFormatter for formatting details. -// -// This function is shorthand for the following syntax: -// -// fmt.Fprintln(w, spew.NewFormatter(a), spew.NewFormatter(b)) -func Fprintln(w io.Writer, a ...interface{}) (n int, err error) { - return fmt.Fprintln(w, convertArgs(a)...) -} - -// Print is a wrapper for fmt.Print that treats each argument as if it were -// passed with a default Formatter interface returned by NewFormatter. It -// returns the number of bytes written and any write error encountered. See -// NewFormatter for formatting details. -// -// This function is shorthand for the following syntax: -// -// fmt.Print(spew.NewFormatter(a), spew.NewFormatter(b)) -func Print(a ...interface{}) (n int, err error) { - return fmt.Print(convertArgs(a)...) -} - -// Printf is a wrapper for fmt.Printf that treats each argument as if it were -// passed with a default Formatter interface returned by NewFormatter. It -// returns the number of bytes written and any write error encountered. See -// NewFormatter for formatting details. -// -// This function is shorthand for the following syntax: -// -// fmt.Printf(format, spew.NewFormatter(a), spew.NewFormatter(b)) -func Printf(format string, a ...interface{}) (n int, err error) { - return fmt.Printf(format, convertArgs(a)...) -} - -// Println is a wrapper for fmt.Println that treats each argument as if it were -// passed with a default Formatter interface returned by NewFormatter. It -// returns the number of bytes written and any write error encountered. See -// NewFormatter for formatting details. -// -// This function is shorthand for the following syntax: -// -// fmt.Println(spew.NewFormatter(a), spew.NewFormatter(b)) -func Println(a ...interface{}) (n int, err error) { - return fmt.Println(convertArgs(a)...) -} - -// Sprint is a wrapper for fmt.Sprint that treats each argument as if it were -// passed with a default Formatter interface returned by NewFormatter. It -// returns the resulting string. See NewFormatter for formatting details. -// -// This function is shorthand for the following syntax: -// -// fmt.Sprint(spew.NewFormatter(a), spew.NewFormatter(b)) -func Sprint(a ...interface{}) string { - return fmt.Sprint(convertArgs(a)...) -} - -// Sprintf is a wrapper for fmt.Sprintf that treats each argument as if it were -// passed with a default Formatter interface returned by NewFormatter. It -// returns the resulting string. See NewFormatter for formatting details. -// -// This function is shorthand for the following syntax: -// -// fmt.Sprintf(format, spew.NewFormatter(a), spew.NewFormatter(b)) -func Sprintf(format string, a ...interface{}) string { - return fmt.Sprintf(format, convertArgs(a)...) -} - -// Sprintln is a wrapper for fmt.Sprintln that treats each argument as if it -// were passed with a default Formatter interface returned by NewFormatter. It -// returns the resulting string. See NewFormatter for formatting details. -// -// This function is shorthand for the following syntax: -// -// fmt.Sprintln(spew.NewFormatter(a), spew.NewFormatter(b)) -func Sprintln(a ...interface{}) string { - return fmt.Sprintln(convertArgs(a)...) -} - -// convertArgs accepts a slice of arguments and returns a slice of the same -// length with each argument converted to a default spew Formatter interface. -func convertArgs(args []interface{}) (formatters []interface{}) { - formatters = make([]interface{}, len(args)) - for index, arg := range args { - formatters[index] = NewFormatter(arg) - } - return formatters -} diff --git a/vendor/github.com/eapache/go-resiliency/LICENSE b/vendor/github.com/eapache/go-resiliency/LICENSE deleted file mode 100644 index 698a3f51..00000000 --- a/vendor/github.com/eapache/go-resiliency/LICENSE +++ /dev/null @@ -1,22 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2014 Evan Huus - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - diff --git a/vendor/github.com/eapache/go-resiliency/breaker/README.md b/vendor/github.com/eapache/go-resiliency/breaker/README.md deleted file mode 100644 index 2d1b3d93..00000000 --- a/vendor/github.com/eapache/go-resiliency/breaker/README.md +++ /dev/null @@ -1,34 +0,0 @@ -circuit-breaker -=============== - -[![Build Status](https://travis-ci.org/eapache/go-resiliency.svg?branch=master)](https://travis-ci.org/eapache/go-resiliency) -[![GoDoc](https://godoc.org/github.com/eapache/go-resiliency/breaker?status.svg)](https://godoc.org/github.com/eapache/go-resiliency/breaker) -[![Code of Conduct](https://img.shields.io/badge/code%20of%20conduct-active-blue.svg)](https://eapache.github.io/conduct.html) - -The circuit-breaker resiliency pattern for golang. - -Creating a breaker takes three parameters: -- error threshold (for opening the breaker) -- success threshold (for closing the breaker) -- timeout (how long to keep the breaker open) - -```go -b := breaker.New(3, 1, 5*time.Second) - -for { - result := b.Run(func() error { - // communicate with some external service and - // return an error if the communication failed - return nil - }) - - switch result { - case nil: - // success! - case breaker.ErrBreakerOpen: - // our function wasn't run because the breaker was open - default: - // some other error - } -} -``` diff --git a/vendor/github.com/eapache/go-resiliency/breaker/breaker.go b/vendor/github.com/eapache/go-resiliency/breaker/breaker.go deleted file mode 100644 index f88ca724..00000000 --- a/vendor/github.com/eapache/go-resiliency/breaker/breaker.go +++ /dev/null @@ -1,161 +0,0 @@ -// Package breaker implements the circuit-breaker resiliency pattern for Go. -package breaker - -import ( - "errors" - "sync" - "sync/atomic" - "time" -) - -// ErrBreakerOpen is the error returned from Run() when the function is not executed -// because the breaker is currently open. -var ErrBreakerOpen = errors.New("circuit breaker is open") - -const ( - closed uint32 = iota - open - halfOpen -) - -// Breaker implements the circuit-breaker resiliency pattern -type Breaker struct { - errorThreshold, successThreshold int - timeout time.Duration - - lock sync.Mutex - state uint32 - errors, successes int - lastError time.Time -} - -// New constructs a new circuit-breaker that starts closed. -// From closed, the breaker opens if "errorThreshold" errors are seen -// without an error-free period of at least "timeout". From open, the -// breaker half-closes after "timeout". From half-open, the breaker closes -// after "successThreshold" consecutive successes, or opens on a single error. -func New(errorThreshold, successThreshold int, timeout time.Duration) *Breaker { - return &Breaker{ - errorThreshold: errorThreshold, - successThreshold: successThreshold, - timeout: timeout, - } -} - -// Run will either return ErrBreakerOpen immediately if the circuit-breaker is -// already open, or it will run the given function and pass along its return -// value. It is safe to call Run concurrently on the same Breaker. -func (b *Breaker) Run(work func() error) error { - state := atomic.LoadUint32(&b.state) - - if state == open { - return ErrBreakerOpen - } - - return b.doWork(state, work) -} - -// Go will either return ErrBreakerOpen immediately if the circuit-breaker is -// already open, or it will run the given function in a separate goroutine. -// If the function is run, Go will return nil immediately, and will *not* return -// the return value of the function. It is safe to call Go concurrently on the -// same Breaker. -func (b *Breaker) Go(work func() error) error { - state := atomic.LoadUint32(&b.state) - - if state == open { - return ErrBreakerOpen - } - - // errcheck complains about ignoring the error return value, but - // that's on purpose; if you want an error from a goroutine you have to - // get it over a channel or something - go b.doWork(state, work) - - return nil -} - -func (b *Breaker) doWork(state uint32, work func() error) error { - var panicValue interface{} - - result := func() error { - defer func() { - panicValue = recover() - }() - return work() - }() - - if result == nil && panicValue == nil && state == closed { - // short-circuit the normal, success path without contending - // on the lock - return nil - } - - // oh well, I guess we have to contend on the lock - b.processResult(result, panicValue) - - if panicValue != nil { - // as close as Go lets us come to a "rethrow" although unfortunately - // we lose the original panicing location - panic(panicValue) - } - - return result -} - -func (b *Breaker) processResult(result error, panicValue interface{}) { - b.lock.Lock() - defer b.lock.Unlock() - - if result == nil && panicValue == nil { - if b.state == halfOpen { - b.successes++ - if b.successes == b.successThreshold { - b.closeBreaker() - } - } - } else { - if b.errors > 0 { - expiry := b.lastError.Add(b.timeout) - if time.Now().After(expiry) { - b.errors = 0 - } - } - - switch b.state { - case closed: - b.errors++ - if b.errors == b.errorThreshold { - b.openBreaker() - } else { - b.lastError = time.Now() - } - case halfOpen: - b.openBreaker() - } - } -} - -func (b *Breaker) openBreaker() { - b.changeState(open) - go b.timer() -} - -func (b *Breaker) closeBreaker() { - b.changeState(closed) -} - -func (b *Breaker) timer() { - time.Sleep(b.timeout) - - b.lock.Lock() - defer b.lock.Unlock() - - b.changeState(halfOpen) -} - -func (b *Breaker) changeState(newState uint32) { - b.errors = 0 - b.successes = 0 - atomic.StoreUint32(&b.state, newState) -} diff --git a/vendor/github.com/eapache/go-xerial-snappy/.gitignore b/vendor/github.com/eapache/go-xerial-snappy/.gitignore deleted file mode 100644 index daf913b1..00000000 --- a/vendor/github.com/eapache/go-xerial-snappy/.gitignore +++ /dev/null @@ -1,24 +0,0 @@ -# Compiled Object files, Static and Dynamic libs (Shared Objects) -*.o -*.a -*.so - -# Folders -_obj -_test - -# Architecture specific extensions/prefixes -*.[568vq] -[568vq].out - -*.cgo1.go -*.cgo2.c -_cgo_defun.c -_cgo_gotypes.go -_cgo_export.* - -_testmain.go - -*.exe -*.test -*.prof diff --git a/vendor/github.com/eapache/go-xerial-snappy/.travis.yml b/vendor/github.com/eapache/go-xerial-snappy/.travis.yml deleted file mode 100644 index d6cf4f1f..00000000 --- a/vendor/github.com/eapache/go-xerial-snappy/.travis.yml +++ /dev/null @@ -1,7 +0,0 @@ -language: go - -go: -- 1.5.4 -- 1.6.1 - -sudo: false diff --git a/vendor/github.com/eapache/go-xerial-snappy/LICENSE b/vendor/github.com/eapache/go-xerial-snappy/LICENSE deleted file mode 100644 index 5bf3688d..00000000 --- a/vendor/github.com/eapache/go-xerial-snappy/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2016 Evan Huus - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/vendor/github.com/eapache/go-xerial-snappy/README.md b/vendor/github.com/eapache/go-xerial-snappy/README.md deleted file mode 100644 index 3f2695c7..00000000 --- a/vendor/github.com/eapache/go-xerial-snappy/README.md +++ /dev/null @@ -1,13 +0,0 @@ -# go-xerial-snappy - -[![Build Status](https://travis-ci.org/eapache/go-xerial-snappy.svg?branch=master)](https://travis-ci.org/eapache/go-xerial-snappy) - -Xerial-compatible Snappy framing support for golang. - -Packages using Xerial for snappy encoding use a framing format incompatible with -basically everything else in existence. This package wraps Go's built-in snappy -package to support it. - -Apps that use this format include Apache Kafka (see -https://github.com/dpkp/kafka-python/issues/126#issuecomment-35478921 for -details). diff --git a/vendor/github.com/eapache/go-xerial-snappy/fuzz.go b/vendor/github.com/eapache/go-xerial-snappy/fuzz.go deleted file mode 100644 index 6a46f478..00000000 --- a/vendor/github.com/eapache/go-xerial-snappy/fuzz.go +++ /dev/null @@ -1,16 +0,0 @@ -// +build gofuzz - -package snappy - -func Fuzz(data []byte) int { - decode, err := Decode(data) - if decode == nil && err == nil { - panic("nil error with nil result") - } - - if err != nil { - return 0 - } - - return 1 -} diff --git a/vendor/github.com/eapache/go-xerial-snappy/snappy.go b/vendor/github.com/eapache/go-xerial-snappy/snappy.go deleted file mode 100644 index ea8f7afe..00000000 --- a/vendor/github.com/eapache/go-xerial-snappy/snappy.go +++ /dev/null @@ -1,131 +0,0 @@ -package snappy - -import ( - "bytes" - "encoding/binary" - "errors" - - master "github.com/golang/snappy" -) - -const ( - sizeOffset = 16 - sizeBytes = 4 -) - -var ( - xerialHeader = []byte{130, 83, 78, 65, 80, 80, 89, 0} - - // This is xerial version 1 and minimally compatible with version 1 - xerialVersionInfo = []byte{0, 0, 0, 1, 0, 0, 0, 1} - - // ErrMalformed is returned by the decoder when the xerial framing - // is malformed - ErrMalformed = errors.New("malformed xerial framing") -) - -func min(x, y int) int { - if x < y { - return x - } - return y -} - -// Encode encodes data as snappy with no framing header. -func Encode(src []byte) []byte { - return master.Encode(nil, src) -} - -// EncodeStream *appends* to the specified 'dst' the compressed -// 'src' in xerial framing format. If 'dst' does not have enough -// capacity, then a new slice will be allocated. If 'dst' has -// non-zero length, then if *must* have been built using this function. -func EncodeStream(dst, src []byte) []byte { - if len(dst) == 0 { - dst = append(dst, xerialHeader...) - dst = append(dst, xerialVersionInfo...) - } - - // Snappy encode in blocks of maximum 32KB - var ( - max = len(src) - blockSize = 32 * 1024 - pos = 0 - chunk []byte - ) - - for pos < max { - newPos := min(pos + blockSize, max) - chunk = master.Encode(chunk[:cap(chunk)], src[pos:newPos]) - - // First encode the compressed size (big-endian) - // Put* panics if the buffer is too small, so pad 4 bytes first - origLen := len(dst) - dst = append(dst, dst[0:4]...) - binary.BigEndian.PutUint32(dst[origLen:], uint32(len(chunk))) - - // And now the compressed data - dst = append(dst, chunk...) - pos = newPos - } - return dst -} - -// Decode decodes snappy data whether it is traditional unframed -// or includes the xerial framing format. -func Decode(src []byte) ([]byte, error) { - return DecodeInto(nil, src) -} - -// DecodeInto decodes snappy data whether it is traditional unframed -// or includes the xerial framing format into the specified `dst`. -// It is assumed that the entirety of `dst` including all capacity is available -// for use by this function. If `dst` is nil *or* insufficiently large to hold -// the decoded `src`, new space will be allocated. -func DecodeInto(dst, src []byte) ([]byte, error) { - var max = len(src) - if max < len(xerialHeader) { - return nil, ErrMalformed - } - - if !bytes.Equal(src[:8], xerialHeader) { - return master.Decode(dst[:cap(dst)], src) - } - - if max < sizeOffset+sizeBytes { - return nil, ErrMalformed - } - - if dst == nil { - dst = make([]byte, 0, len(src)) - } - - dst = dst[:0] - var ( - pos = sizeOffset - chunk []byte - err error - ) - - for pos+sizeBytes <= max { - size := int(binary.BigEndian.Uint32(src[pos : pos+sizeBytes])) - pos += sizeBytes - - nextPos := pos + size - // On architectures where int is 32-bytes wide size + pos could - // overflow so we need to check the low bound as well as the - // high - if nextPos < pos || nextPos > max { - return nil, ErrMalformed - } - - chunk, err = master.Decode(chunk[:cap(chunk)], src[pos:nextPos]) - - if err != nil { - return nil, err - } - pos = nextPos - dst = append(dst, chunk...) - } - return dst, nil -} diff --git a/vendor/github.com/eapache/queue/.gitignore b/vendor/github.com/eapache/queue/.gitignore deleted file mode 100644 index 83656241..00000000 --- a/vendor/github.com/eapache/queue/.gitignore +++ /dev/null @@ -1,23 +0,0 @@ -# Compiled Object files, Static and Dynamic libs (Shared Objects) -*.o -*.a -*.so - -# Folders -_obj -_test - -# Architecture specific extensions/prefixes -*.[568vq] -[568vq].out - -*.cgo1.go -*.cgo2.c -_cgo_defun.c -_cgo_gotypes.go -_cgo_export.* - -_testmain.go - -*.exe -*.test diff --git a/vendor/github.com/eapache/queue/.travis.yml b/vendor/github.com/eapache/queue/.travis.yml deleted file mode 100644 index 235a40a4..00000000 --- a/vendor/github.com/eapache/queue/.travis.yml +++ /dev/null @@ -1,7 +0,0 @@ -language: go -sudo: false - -go: - - 1.2 - - 1.3 - - 1.4 diff --git a/vendor/github.com/eapache/queue/LICENSE b/vendor/github.com/eapache/queue/LICENSE deleted file mode 100644 index d5f36dbc..00000000 --- a/vendor/github.com/eapache/queue/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2014 Evan Huus - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. \ No newline at end of file diff --git a/vendor/github.com/eapache/queue/README.md b/vendor/github.com/eapache/queue/README.md deleted file mode 100644 index 8e782335..00000000 --- a/vendor/github.com/eapache/queue/README.md +++ /dev/null @@ -1,16 +0,0 @@ -Queue -===== - -[![Build Status](https://travis-ci.org/eapache/queue.svg)](https://travis-ci.org/eapache/queue) -[![GoDoc](https://godoc.org/github.com/eapache/queue?status.png)](https://godoc.org/github.com/eapache/queue) -[![Code of Conduct](https://img.shields.io/badge/code%20of%20conduct-active-blue.svg)](https://eapache.github.io/conduct.html) - -A fast Golang queue using a ring-buffer, based on the version suggested by Dariusz Górecki. -Using this instead of other, simpler, queue implementations (slice+append or linked list) provides -substantial memory and time benefits, and fewer GC pauses. - -The queue implemented here is as fast as it is in part because it is *not* thread-safe. - -Follows semantic versioning using https://gopkg.in/ - import from -[`gopkg.in/eapache/queue.v1`](https://gopkg.in/eapache/queue.v1) -for guaranteed API stability. diff --git a/vendor/github.com/eapache/queue/queue.go b/vendor/github.com/eapache/queue/queue.go deleted file mode 100644 index 71d1acdf..00000000 --- a/vendor/github.com/eapache/queue/queue.go +++ /dev/null @@ -1,102 +0,0 @@ -/* -Package queue provides a fast, ring-buffer queue based on the version suggested by Dariusz Górecki. -Using this instead of other, simpler, queue implementations (slice+append or linked list) provides -substantial memory and time benefits, and fewer GC pauses. - -The queue implemented here is as fast as it is for an additional reason: it is *not* thread-safe. -*/ -package queue - -// minQueueLen is smallest capacity that queue may have. -// Must be power of 2 for bitwise modulus: x % n == x & (n - 1). -const minQueueLen = 16 - -// Queue represents a single instance of the queue data structure. -type Queue struct { - buf []interface{} - head, tail, count int -} - -// New constructs and returns a new Queue. -func New() *Queue { - return &Queue{ - buf: make([]interface{}, minQueueLen), - } -} - -// Length returns the number of elements currently stored in the queue. -func (q *Queue) Length() int { - return q.count -} - -// resizes the queue to fit exactly twice its current contents -// this can result in shrinking if the queue is less than half-full -func (q *Queue) resize() { - newBuf := make([]interface{}, q.count<<1) - - if q.tail > q.head { - copy(newBuf, q.buf[q.head:q.tail]) - } else { - n := copy(newBuf, q.buf[q.head:]) - copy(newBuf[n:], q.buf[:q.tail]) - } - - q.head = 0 - q.tail = q.count - q.buf = newBuf -} - -// Add puts an element on the end of the queue. -func (q *Queue) Add(elem interface{}) { - if q.count == len(q.buf) { - q.resize() - } - - q.buf[q.tail] = elem - // bitwise modulus - q.tail = (q.tail + 1) & (len(q.buf) - 1) - q.count++ -} - -// Peek returns the element at the head of the queue. This call panics -// if the queue is empty. -func (q *Queue) Peek() interface{} { - if q.count <= 0 { - panic("queue: Peek() called on empty queue") - } - return q.buf[q.head] -} - -// Get returns the element at index i in the queue. If the index is -// invalid, the call will panic. This method accepts both positive and -// negative index values. Index 0 refers to the first element, and -// index -1 refers to the last. -func (q *Queue) Get(i int) interface{} { - // If indexing backwards, convert to positive index. - if i < 0 { - i += q.count - } - if i < 0 || i >= q.count { - panic("queue: Get() called with index out of range") - } - // bitwise modulus - return q.buf[(q.head+i)&(len(q.buf)-1)] -} - -// Remove removes and returns the element from the front of the queue. If the -// queue is empty, the call will panic. -func (q *Queue) Remove() interface{} { - if q.count <= 0 { - panic("queue: Remove() called on empty queue") - } - ret := q.buf[q.head] - q.buf[q.head] = nil - // bitwise modulus - q.head = (q.head + 1) & (len(q.buf) - 1) - q.count-- - // Resize down if buffer 1/4 full. - if len(q.buf) > minQueueLen && (q.count<<2) == len(q.buf) { - q.resize() - } - return ret -} diff --git a/vendor/github.com/globalsign/mgo/.gitignore b/vendor/github.com/globalsign/mgo/.gitignore deleted file mode 100644 index 9a3120f6..00000000 --- a/vendor/github.com/globalsign/mgo/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -_harness -.vscode \ No newline at end of file diff --git a/vendor/github.com/globalsign/mgo/.travis.yml b/vendor/github.com/globalsign/mgo/.travis.yml deleted file mode 100644 index f1f89e96..00000000 --- a/vendor/github.com/globalsign/mgo/.travis.yml +++ /dev/null @@ -1,49 +0,0 @@ -language: go - -go_import_path: github.com/globalsign/mgo - -go: - - 1.9.x - - 1.10.x - -env: - global: - - BUCKET=https://s3.eu-west-2.amazonaws.com/globalsign-mgo - - FASTDL=https://fastdl.mongodb.org/linux - matrix: - - MONGODB=x86_64-ubuntu1404-3.0.15 - - MONGODB=x86_64-ubuntu1404-3.2.17 - - MONGODB=x86_64-ubuntu1404-3.4.10 - - MONGODB=x86_64-ubuntu1404-3.6.0 - -install: - - - wget $FASTDL/mongodb-linux-$MONGODB.tgz - - tar xzvf mongodb-linux-$MONGODB.tgz - - export PATH=$PWD/mongodb-linux-$MONGODB/bin:$PATH - - - wget $BUCKET/daemontools.tar.gz - - tar xzvf daemontools.tar.gz - - export PATH=$PWD/daemontools:$PATH - - - go get gopkg.in/check.v1 - - go get gopkg.in/yaml.v2 - - go get gopkg.in/tomb.v2 - - go get golang.org/x/lint/golint - -before_script: - - golint ./... | grep -v 'ID' | cat - - go vet github.com/globalsign/mgo/bson github.com/globalsign/mgo/txn github.com/globalsign/mgo - - export NOIPV6=1 - - make startdb - -script: - - (cd bson && go test -check.v) - - go test -check.v -fast - - (cd txn && go test -check.v) - - make stopdb - -git: - depth: 3 - -# vim:sw=4:ts=4:et diff --git a/vendor/github.com/globalsign/mgo/CONTRIBUTING.md b/vendor/github.com/globalsign/mgo/CONTRIBUTING.md deleted file mode 100644 index 79539955..00000000 --- a/vendor/github.com/globalsign/mgo/CONTRIBUTING.md +++ /dev/null @@ -1,14 +0,0 @@ -Contributing -------------------------- - -We really appreciate contributions, but they must meet the following requirements: - -* A PR should have a brief description of the problem/feature being proposed -* Pull requests should target the `development` branch -* Existing tests should pass and any new code should be covered with it's own test(s) (use [travis-ci](https://travis-ci.org)) -* New functions should be [documented](https://blog.golang.org/godoc-documenting-go-code) clearly -* Code should pass `golint`, `go vet` and `go fmt` - -We merge PRs into `development`, which is then tested in a sharded, replicated environment in our datacenter for regressions. Once everyone is happy, we merge to master - this is to maintain a bit of quality control past the usual PR process. - -**Thanks** for helping! diff --git a/vendor/github.com/globalsign/mgo/LICENSE b/vendor/github.com/globalsign/mgo/LICENSE deleted file mode 100644 index 770c7672..00000000 --- a/vendor/github.com/globalsign/mgo/LICENSE +++ /dev/null @@ -1,25 +0,0 @@ -mgo - MongoDB driver for Go - -Copyright (c) 2010-2013 - Gustavo Niemeyer - -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -1. Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. -2. Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR -ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/globalsign/mgo/Makefile b/vendor/github.com/globalsign/mgo/Makefile deleted file mode 100644 index d1027d45..00000000 --- a/vendor/github.com/globalsign/mgo/Makefile +++ /dev/null @@ -1,5 +0,0 @@ -startdb: - @harness/setup.sh start - -stopdb: - @harness/setup.sh stop diff --git a/vendor/github.com/globalsign/mgo/README.md b/vendor/github.com/globalsign/mgo/README.md deleted file mode 100644 index 76fd0554..00000000 --- a/vendor/github.com/globalsign/mgo/README.md +++ /dev/null @@ -1,105 +0,0 @@ -[![Build Status](https://travis-ci.org/globalsign/mgo.svg?branch=master)](https://travis-ci.org/globalsign/mgo) [![GoDoc](https://godoc.org/github.com/globalsign/mgo?status.svg)](https://godoc.org/github.com/globalsign/mgo) - -The MongoDB driver for Go -------------------------- - -This fork has had a few improvements by ourselves as well as several PR's merged from the original mgo repo that are currently awaiting review. -Changes are mostly geared towards performance improvements and bug fixes, though a few new features have been added. - -Further PR's (with tests) are welcome, but please maintain backwards compatibility. - -Detailed documentation of the API is available at -[GoDoc](https://godoc.org/github.com/globalsign/mgo). - -A [sub-package](https://godoc.org/github.com/globalsign/mgo/bson) that implements the [BSON](http://bsonspec.org) specification is also included, and may be used independently of the driver. - -## Supported Versions - -`mgo` is known to work well on (and has integration tests against) MongoDB v3.0, 3.2, 3.4 and 3.6. - -MongoDB 4.0 is currently experimental - we would happily accept PRs to help improve support! - -## Changes -* Fixes attempting to authenticate before every query ([details](https://github.com/go-mgo/mgo/issues/254)) -* Removes bulk update / delete batch size limitations ([details](https://github.com/go-mgo/mgo/issues/288)) -* Adds native support for `time.Duration` marshalling ([details](https://github.com/go-mgo/mgo/pull/373)) -* Reduce memory footprint / garbage collection pressure by reusing buffers ([details](https://github.com/go-mgo/mgo/pull/229), [more](https://github.com/globalsign/mgo/pull/56)) -* Support majority read concerns ([details](https://github.com/globalsign/mgo/pull/2)) -* Improved connection handling ([details](https://github.com/globalsign/mgo/pull/5)) -* Hides SASL warnings ([details](https://github.com/globalsign/mgo/pull/7)) -* Support for partial indexes ([details](https://github.com/domodwyer/mgo/commit/5efe8eccb028238d93c222828cae4806aeae9f51)) -* Fixes timezone handling ([details](https://github.com/go-mgo/mgo/pull/464)) -* Integration tests run against MongoDB 3.2 & 3.4 releases ([details](https://github.com/globalsign/mgo/pull/4), [more](https://github.com/globalsign/mgo/pull/24), [more](https://github.com/globalsign/mgo/pull/35)) -* Improved multi-document transaction performance ([details](https://github.com/globalsign/mgo/pull/10), [more](https://github.com/globalsign/mgo/pull/11), [more](https://github.com/globalsign/mgo/pull/16)) -* Fixes cursor timeouts ([details](https://jira.mongodb.org/browse/SERVER-24899)) -* Support index hints and timeouts for count queries ([details](https://github.com/globalsign/mgo/pull/17)) -* Don't panic when handling indexed `int64` fields ([details](https://github.com/go-mgo/mgo/issues/475)) -* Supports dropping all indexes on a collection ([details](https://github.com/globalsign/mgo/pull/25)) -* Annotates log entries/profiler output with optional appName on 3.4+ ([details](https://github.com/globalsign/mgo/pull/28)) -* Support for read-only [views](https://docs.mongodb.com/manual/core/views/) in 3.4+ ([details](https://github.com/globalsign/mgo/pull/33)) -* Support for [collations](https://docs.mongodb.com/manual/reference/collation/) in 3.4+ ([details](https://github.com/globalsign/mgo/pull/37), [more](https://github.com/globalsign/mgo/pull/166)) -* Provide BSON constants for convenience/sanity ([details](https://github.com/globalsign/mgo/pull/41)) -* Consistently unmarshal time.Time values as UTC ([details](https://github.com/globalsign/mgo/pull/42)) -* Enforces best practise coding guidelines ([details](https://github.com/globalsign/mgo/pull/44)) -* GetBSON correctly handles structs with both fields and pointers ([details](https://github.com/globalsign/mgo/pull/40)) -* Improved bson.Raw unmarshalling performance ([details](https://github.com/globalsign/mgo/pull/49)) -* Minimise socket connection timeouts due to excessive locking ([details](https://github.com/globalsign/mgo/pull/52)) -* Natively support X509 client authentication ([details](https://github.com/globalsign/mgo/pull/55)) -* Gracefully recover from a temporarily unreachable server ([details](https://github.com/globalsign/mgo/pull/69)) -* Use JSON tags when no explicit BSON are tags set ([details](https://github.com/globalsign/mgo/pull/91)) -* Support [$changeStream](https://docs.mongodb.com/manual/changeStreams/) tailing on 3.6+ ([details](https://github.com/globalsign/mgo/pull/97)) -* Fix deadlock in cluster synchronisation ([details](https://github.com/globalsign/mgo/issues/120)) -* Implement `maxIdleTimeout` for pooled connections ([details](https://github.com/globalsign/mgo/pull/116)) -* Connection pool waiting improvements ([details](https://github.com/globalsign/mgo/pull/115)) -* Fixes BSON encoding for `$in` and friends ([details](https://github.com/globalsign/mgo/pull/128)) -* Add BSON stream encoders ([details](https://github.com/globalsign/mgo/pull/127)) -* Add integer map key support in the BSON encoder ([details](https://github.com/globalsign/mgo/pull/140)) -* Support aggregation [collations](https://docs.mongodb.com/manual/reference/collation/) ([details](https://github.com/globalsign/mgo/pull/144)) -* Support encoding of inline struct references ([details](https://github.com/globalsign/mgo/pull/146)) -* Improved windows test harness ([details](https://github.com/globalsign/mgo/pull/158)) -* Improved type and nil handling in the BSON codec ([details](https://github.com/globalsign/mgo/pull/147/files), [more](https://github.com/globalsign/mgo/pull/181)) -* Separated network read/write timeouts ([details](https://github.com/globalsign/mgo/pull/161)) -* Expanded dial string configuration options ([details](https://github.com/globalsign/mgo/pull/162)) -* Implement MongoTimestamp ([details](https://github.com/globalsign/mgo/pull/171)) -* Support setting `writeConcern` for `findAndModify` operations ([details](https://github.com/globalsign/mgo/pull/185)) -* Add `ssl` to the dial string options ([details](https://github.com/globalsign/mgo/pull/184)) - - ---- - -### Thanks to -* @aksentyev -* @bachue -* @bozaro -* @BenLubar -* @carldunham -* @carter2000 -* @cedric-cordenier -* @cezarsa -* @DaytonG -* @ddspog -* @drichelson -* @dvic -* @eaglerayp -* @feliixx -* @fmpwizard -* @gazoon -* @gedge -* @gnawux -* @idy -* @jameinel -* @jefferickson -* @johnlawsharrison -* @KJTsanaktsidis -* @larrycinnabar -* @mapete94 -* @maxnoel -* @mcspring -* @Mei-Zhao -* @peterdeka -* @Reenjii -* @roobre -* @smoya -* @steve-gray -* @tbruyelle -* @wgallagher diff --git a/vendor/github.com/globalsign/mgo/auth.go b/vendor/github.com/globalsign/mgo/auth.go deleted file mode 100644 index 75d2ebc3..00000000 --- a/vendor/github.com/globalsign/mgo/auth.go +++ /dev/null @@ -1,467 +0,0 @@ -// mgo - MongoDB driver for Go -// -// Copyright (c) 2010-2012 - Gustavo Niemeyer -// -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are met: -// -// 1. Redistributions of source code must retain the above copyright notice, this -// list of conditions and the following disclaimer. -// 2. Redistributions in binary form must reproduce the above copyright notice, -// this list of conditions and the following disclaimer in the documentation -// and/or other materials provided with the distribution. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR -// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -package mgo - -import ( - "crypto/md5" - "crypto/sha1" - "encoding/hex" - "errors" - "fmt" - "sync" - - "github.com/globalsign/mgo/bson" - "github.com/globalsign/mgo/internal/scram" -) - -type authCmd struct { - Authenticate int - - Nonce string - User string - Key string -} - -type startSaslCmd struct { - StartSASL int `bson:"startSasl"` -} - -type authResult struct { - ErrMsg string - Ok bool -} - -type getNonceCmd struct { - GetNonce int -} - -type getNonceResult struct { - Nonce string - Err string `bson:"$err"` - Code int -} - -type logoutCmd struct { - Logout int -} - -type saslCmd struct { - Start int `bson:"saslStart,omitempty"` - Continue int `bson:"saslContinue,omitempty"` - ConversationId int `bson:"conversationId,omitempty"` - Mechanism string `bson:"mechanism,omitempty"` - Payload []byte -} - -type saslResult struct { - Ok bool `bson:"ok"` - NotOk bool `bson:"code"` // Server <= 2.3.2 returns ok=1 & code>0 on errors (WTF?) - Done bool - - ConversationId int `bson:"conversationId"` - Payload []byte - ErrMsg string -} - -type saslStepper interface { - Step(serverData []byte) (clientData []byte, done bool, err error) - Close() -} - -func (socket *mongoSocket) getNonce() (nonce string, err error) { - socket.Lock() - for socket.cachedNonce == "" && socket.dead == nil { - debugf("Socket %p to %s: waiting for nonce", socket, socket.addr) - socket.gotNonce.Wait() - } - if socket.cachedNonce == "mongos" { - socket.Unlock() - return "", errors.New("Can't authenticate with mongos; see http://j.mp/mongos-auth") - } - debugf("Socket %p to %s: got nonce", socket, socket.addr) - nonce, err = socket.cachedNonce, socket.dead - socket.cachedNonce = "" - socket.Unlock() - if err != nil { - nonce = "" - } - return -} - -func (socket *mongoSocket) resetNonce() { - debugf("Socket %p to %s: requesting a new nonce", socket, socket.addr) - op := &queryOp{} - op.query = &getNonceCmd{GetNonce: 1} - op.collection = "admin.$cmd" - op.limit = -1 - op.replyFunc = func(err error, reply *replyOp, docNum int, docData []byte) { - if err != nil { - socket.kill(errors.New("getNonce: "+err.Error()), true) - return - } - result := &getNonceResult{} - err = bson.Unmarshal(docData, &result) - if err != nil { - socket.kill(errors.New("Failed to unmarshal nonce: "+err.Error()), true) - return - } - debugf("Socket %p to %s: nonce unmarshalled: %#v", socket, socket.addr, result) - if result.Code == 13390 { - // mongos doesn't yet support auth (see http://j.mp/mongos-auth) - result.Nonce = "mongos" - } else if result.Nonce == "" { - var msg string - if result.Err != "" { - msg = fmt.Sprintf("Got an empty nonce: %s (%d)", result.Err, result.Code) - } else { - msg = "Got an empty nonce" - } - socket.kill(errors.New(msg), true) - return - } - socket.Lock() - if socket.cachedNonce != "" { - socket.Unlock() - panic("resetNonce: nonce already cached") - } - socket.cachedNonce = result.Nonce - socket.gotNonce.Signal() - socket.Unlock() - } - err := socket.Query(op) - if err != nil { - socket.kill(errors.New("resetNonce: "+err.Error()), true) - } -} - -func (socket *mongoSocket) Login(cred Credential) error { - socket.Lock() - if cred.Mechanism == "" && socket.serverInfo.MaxWireVersion >= 3 { - cred.Mechanism = "SCRAM-SHA-1" - } - for _, sockCred := range socket.creds { - if sockCred == cred { - debugf("Socket %p to %s: login: db=%q user=%q (already logged in)", socket, socket.addr, cred.Source, cred.Username) - socket.Unlock() - return nil - } - } - if socket.dropLogout(cred) { - debugf("Socket %p to %s: login: db=%q user=%q (cached)", socket, socket.addr, cred.Source, cred.Username) - socket.creds = append(socket.creds, cred) - socket.Unlock() - return nil - } - socket.Unlock() - - debugf("Socket %p to %s: login: db=%q user=%q", socket, socket.addr, cred.Source, cred.Username) - - var err error - switch cred.Mechanism { - case "", "MONGODB-CR", "MONGO-CR": // Name changed to MONGODB-CR in SERVER-8501. - err = socket.loginClassic(cred) - case "PLAIN": - err = socket.loginPlain(cred) - case "MONGODB-X509": - err = socket.loginX509(cred) - default: - // Try SASL for everything else, if it is available. - err = socket.loginSASL(cred) - } - - if err != nil { - debugf("Socket %p to %s: login error: %s", socket, socket.addr, err) - } else { - debugf("Socket %p to %s: login successful", socket, socket.addr) - } - return err -} - -func (socket *mongoSocket) loginClassic(cred Credential) error { - // Note that this only works properly because this function is - // synchronous, which means the nonce won't get reset while we're - // using it and any other login requests will block waiting for a - // new nonce provided in the defer call below. - nonce, err := socket.getNonce() - if err != nil { - return err - } - defer socket.resetNonce() - - psum := md5.New() - psum.Write([]byte(cred.Username + ":mongo:" + cred.Password)) - - ksum := md5.New() - ksum.Write([]byte(nonce + cred.Username)) - ksum.Write([]byte(hex.EncodeToString(psum.Sum(nil)))) - - key := hex.EncodeToString(ksum.Sum(nil)) - - cmd := authCmd{Authenticate: 1, User: cred.Username, Nonce: nonce, Key: key} - res := authResult{} - return socket.loginRun(cred.Source, &cmd, &res, func() error { - if !res.Ok { - return errors.New(res.ErrMsg) - } - socket.Lock() - socket.dropAuth(cred.Source) - socket.creds = append(socket.creds, cred) - socket.Unlock() - return nil - }) -} - -type authX509Cmd struct { - Authenticate int - User string - Mechanism string -} - -func (socket *mongoSocket) loginX509(cred Credential) error { - cmd := authX509Cmd{Authenticate: 1, User: cred.Username, Mechanism: "MONGODB-X509"} - res := authResult{} - return socket.loginRun(cred.Source, &cmd, &res, func() error { - if !res.Ok { - return errors.New(res.ErrMsg) - } - socket.Lock() - socket.dropAuth(cred.Source) - socket.creds = append(socket.creds, cred) - socket.Unlock() - return nil - }) -} - -func (socket *mongoSocket) loginPlain(cred Credential) error { - cmd := saslCmd{Start: 1, Mechanism: "PLAIN", Payload: []byte("\x00" + cred.Username + "\x00" + cred.Password)} - res := authResult{} - return socket.loginRun(cred.Source, &cmd, &res, func() error { - if !res.Ok { - return errors.New(res.ErrMsg) - } - socket.Lock() - socket.dropAuth(cred.Source) - socket.creds = append(socket.creds, cred) - socket.Unlock() - return nil - }) -} - -func (socket *mongoSocket) loginSASL(cred Credential) error { - var sasl saslStepper - var err error - if cred.Mechanism == "SCRAM-SHA-1" { - // SCRAM is handled without external libraries. - sasl = saslNewScram(cred) - } else if len(cred.ServiceHost) > 0 { - sasl, err = saslNew(cred, cred.ServiceHost) - } else { - sasl, err = saslNew(cred, socket.Server().Addr) - } - if err != nil { - return err - } - defer sasl.Close() - - // The goal of this logic is to carry a locked socket until the - // local SASL step confirms the auth is valid; the socket needs to be - // locked so that concurrent action doesn't leave the socket in an - // auth state that doesn't reflect the operations that took place. - // As a simple case, imagine inverting login=>logout to logout=>login. - // - // The logic below works because the lock func isn't called concurrently. - locked := false - lock := func(b bool) { - if locked != b { - locked = b - if b { - socket.Lock() - } else { - socket.Unlock() - } - } - } - - lock(true) - defer lock(false) - - start := 1 - cmd := saslCmd{} - res := saslResult{} - for { - payload, done, err := sasl.Step(res.Payload) - if err != nil { - return err - } - if done && res.Done { - socket.dropAuth(cred.Source) - socket.creds = append(socket.creds, cred) - break - } - lock(false) - - cmd = saslCmd{ - Start: start, - Continue: 1 - start, - ConversationId: res.ConversationId, - Mechanism: cred.Mechanism, - Payload: payload, - } - start = 0 - err = socket.loginRun(cred.Source, &cmd, &res, func() error { - // See the comment on lock for why this is necessary. - lock(true) - if !res.Ok || res.NotOk { - return fmt.Errorf("server returned error on SASL authentication step: %s", res.ErrMsg) - } - return nil - }) - if err != nil { - return err - } - if done && res.Done { - socket.dropAuth(cred.Source) - socket.creds = append(socket.creds, cred) - break - } - } - - return nil -} - -func saslNewScram(cred Credential) *saslScram { - credsum := md5.New() - credsum.Write([]byte(cred.Username + ":mongo:" + cred.Password)) - client := scram.NewClient(sha1.New, cred.Username, hex.EncodeToString(credsum.Sum(nil))) - return &saslScram{cred: cred, client: client} -} - -type saslScram struct { - cred Credential - client *scram.Client -} - -func (s *saslScram) Close() {} - -func (s *saslScram) Step(serverData []byte) (clientData []byte, done bool, err error) { - more := s.client.Step(serverData) - return s.client.Out(), !more, s.client.Err() -} - -func (socket *mongoSocket) loginRun(db string, query, result interface{}, f func() error) error { - var mutex sync.Mutex - var replyErr error - mutex.Lock() - - op := queryOp{} - op.query = query - op.collection = db + ".$cmd" - op.limit = -1 - op.replyFunc = func(err error, reply *replyOp, docNum int, docData []byte) { - defer mutex.Unlock() - - if err != nil { - replyErr = err - return - } - - err = bson.Unmarshal(docData, result) - if err != nil { - replyErr = err - } else { - // Must handle this within the read loop for the socket, so - // that concurrent login requests are properly ordered. - replyErr = f() - } - } - - err := socket.Query(&op) - if err != nil { - return err - } - mutex.Lock() // Wait. - return replyErr -} - -func (socket *mongoSocket) Logout(db string) { - socket.Lock() - cred, found := socket.dropAuth(db) - if found { - debugf("Socket %p to %s: logout: db=%q (flagged)", socket, socket.addr, db) - socket.logout = append(socket.logout, cred) - } - socket.Unlock() -} - -func (socket *mongoSocket) LogoutAll() { - socket.Lock() - if l := len(socket.creds); l > 0 { - debugf("Socket %p to %s: logout all (flagged %d)", socket, socket.addr, l) - socket.logout = append(socket.logout, socket.creds...) - socket.creds = socket.creds[0:0] - } - socket.Unlock() -} - -func (socket *mongoSocket) flushLogout() (ops []interface{}) { - socket.Lock() - if l := len(socket.logout); l > 0 { - debugf("Socket %p to %s: logout all (flushing %d)", socket, socket.addr, l) - for i := 0; i != l; i++ { - op := queryOp{} - op.query = &logoutCmd{1} - op.collection = socket.logout[i].Source + ".$cmd" - op.limit = -1 - ops = append(ops, &op) - } - socket.logout = socket.logout[0:0] - } - socket.Unlock() - return -} - -func (socket *mongoSocket) dropAuth(db string) (cred Credential, found bool) { - for i, sockCred := range socket.creds { - if sockCred.Source == db { - copy(socket.creds[i:], socket.creds[i+1:]) - socket.creds = socket.creds[:len(socket.creds)-1] - return sockCred, true - } - } - return cred, false -} - -func (socket *mongoSocket) dropLogout(cred Credential) (found bool) { - for i, sockCred := range socket.logout { - if sockCred == cred { - copy(socket.logout[i:], socket.logout[i+1:]) - socket.logout = socket.logout[:len(socket.logout)-1] - return true - } - } - return false -} diff --git a/vendor/github.com/globalsign/mgo/bson/LICENSE b/vendor/github.com/globalsign/mgo/bson/LICENSE deleted file mode 100644 index 89032601..00000000 --- a/vendor/github.com/globalsign/mgo/bson/LICENSE +++ /dev/null @@ -1,25 +0,0 @@ -BSON library for Go - -Copyright (c) 2010-2012 - Gustavo Niemeyer - -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -1. Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. -2. Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR -ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/globalsign/mgo/bson/README.md b/vendor/github.com/globalsign/mgo/bson/README.md deleted file mode 100644 index 5c5819e6..00000000 --- a/vendor/github.com/globalsign/mgo/bson/README.md +++ /dev/null @@ -1,12 +0,0 @@ -[![GoDoc](https://godoc.org/github.com/globalsign/mgo/bson?status.svg)](https://godoc.org/github.com/globalsign/mgo/bson) - -An Implementation of BSON for Go --------------------------------- - -Package bson is an implementation of the [BSON specification](http://bsonspec.org) for Go. - -While the BSON package implements the BSON spec as faithfully as possible, there -is some MongoDB specific behaviour (such as map keys `$in`, `$all`, etc) in the -`bson` package. The priority is for backwards compatibility for the `mgo` -driver, though fixes for obviously buggy behaviour is welcome (and features, etc -behind feature flags). diff --git a/vendor/github.com/globalsign/mgo/bson/bson.go b/vendor/github.com/globalsign/mgo/bson/bson.go deleted file mode 100644 index eb87ef62..00000000 --- a/vendor/github.com/globalsign/mgo/bson/bson.go +++ /dev/null @@ -1,836 +0,0 @@ -// BSON library for Go -// -// Copyright (c) 2010-2012 - Gustavo Niemeyer -// -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are met: -// -// 1. Redistributions of source code must retain the above copyright notice, this -// list of conditions and the following disclaimer. -// 2. Redistributions in binary form must reproduce the above copyright notice, -// this list of conditions and the following disclaimer in the documentation -// and/or other materials provided with the distribution. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR -// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -// Package bson is an implementation of the BSON specification for Go: -// -// http://bsonspec.org -// -// It was created as part of the mgo MongoDB driver for Go, but is standalone -// and may be used on its own without the driver. -package bson - -import ( - "bytes" - "crypto/md5" - "crypto/rand" - "encoding/binary" - "encoding/hex" - "encoding/json" - "errors" - "fmt" - "io" - "math" - "os" - "reflect" - "runtime" - "strings" - "sync" - "sync/atomic" - "time" -) - -//go:generate go run bson_corpus_spec_test_generator.go - -// -------------------------------------------------------------------------- -// The public API. - -// Element types constants from BSON specification. -const ( - ElementFloat64 byte = 0x01 - ElementString byte = 0x02 - ElementDocument byte = 0x03 - ElementArray byte = 0x04 - ElementBinary byte = 0x05 - Element06 byte = 0x06 - ElementObjectId byte = 0x07 - ElementBool byte = 0x08 - ElementDatetime byte = 0x09 - ElementNil byte = 0x0A - ElementRegEx byte = 0x0B - ElementDBPointer byte = 0x0C - ElementJavaScriptWithoutScope byte = 0x0D - ElementSymbol byte = 0x0E - ElementJavaScriptWithScope byte = 0x0F - ElementInt32 byte = 0x10 - ElementTimestamp byte = 0x11 - ElementInt64 byte = 0x12 - ElementDecimal128 byte = 0x13 - ElementMinKey byte = 0xFF - ElementMaxKey byte = 0x7F - - BinaryGeneric byte = 0x00 - BinaryFunction byte = 0x01 - BinaryBinaryOld byte = 0x02 - BinaryUUIDOld byte = 0x03 - BinaryUUID byte = 0x04 - BinaryMD5 byte = 0x05 - BinaryUserDefined byte = 0x80 -) - -// Getter interface: a value implementing the bson.Getter interface will have its GetBSON -// method called when the given value has to be marshalled, and the result -// of this method will be marshaled in place of the actual object. -// -// If GetBSON returns return a non-nil error, the marshalling procedure -// will stop and error out with the provided value. -type Getter interface { - GetBSON() (interface{}, error) -} - -// Setter interface: a value implementing the bson.Setter interface will receive the BSON -// value via the SetBSON method during unmarshaling, and the object -// itself will not be changed as usual. -// -// If setting the value works, the method should return nil or alternatively -// bson.ErrSetZero to set the respective field to its zero value (nil for -// pointer types). If SetBSON returns a value of type bson.TypeError, the -// BSON value will be omitted from a map or slice being decoded and the -// unmarshalling will continue. If it returns any other non-nil error, the -// unmarshalling procedure will stop and error out with the provided value. -// -// This interface is generally useful in pointer receivers, since the method -// will want to change the receiver. A type field that implements the Setter -// interface doesn't have to be a pointer, though. -// -// Unlike the usual behavior, unmarshalling onto a value that implements a -// Setter interface will NOT reset the value to its zero state. This allows -// the value to decide by itself how to be unmarshalled. -// -// For example: -// -// type MyString string -// -// func (s *MyString) SetBSON(raw bson.Raw) error { -// return raw.Unmarshal(s) -// } -// -type Setter interface { - SetBSON(raw Raw) error -} - -// ErrSetZero may be returned from a SetBSON method to have the value set to -// its respective zero value. When used in pointer values, this will set the -// field to nil rather than to the pre-allocated value. -var ErrSetZero = errors.New("set to zero") - -// M is a convenient alias for a map[string]interface{} map, useful for -// dealing with BSON in a native way. For instance: -// -// bson.M{"a": 1, "b": true} -// -// There's no special handling for this type in addition to what's done anyway -// for an equivalent map type. Elements in the map will be dumped in an -// undefined ordered. See also the bson.D type for an ordered alternative. -type M map[string]interface{} - -// D represents a BSON document containing ordered elements. For example: -// -// bson.D{{"a", 1}, {"b", true}} -// -// In some situations, such as when creating indexes for MongoDB, the order in -// which the elements are defined is important. If the order is not important, -// using a map is generally more comfortable. See bson.M and bson.RawD. -type D []DocElem - -// DocElem is an element of the bson.D document representation. -type DocElem struct { - Name string - Value interface{} -} - -// Map returns a map out of the ordered element name/value pairs in d. -func (d D) Map() (m M) { - m = make(M, len(d)) - for _, item := range d { - m[item.Name] = item.Value - } - return m -} - -// The Raw type represents raw unprocessed BSON documents and elements. -// Kind is the kind of element as defined per the BSON specification, and -// Data is the raw unprocessed data for the respective element. -// Using this type it is possible to unmarshal or marshal values partially. -// -// Relevant documentation: -// -// http://bsonspec.org/#/specification -// -type Raw struct { - Kind byte - Data []byte -} - -// RawD represents a BSON document containing raw unprocessed elements. -// This low-level representation may be useful when lazily processing -// documents of uncertain content, or when manipulating the raw content -// documents in general. -type RawD []RawDocElem - -// RawDocElem elements of RawD type. -type RawDocElem struct { - Name string - Value Raw -} - -// ObjectId is a unique ID identifying a BSON value. It must be exactly 12 bytes -// long. MongoDB objects by default have such a property set in their "_id" -// property. -// -// http://www.mongodb.org/display/DOCS/Object+Ids -type ObjectId string - -// ObjectIdHex returns an ObjectId from the provided hex representation. -// Calling this function with an invalid hex representation will -// cause a runtime panic. See the IsObjectIdHex function. -func ObjectIdHex(s string) ObjectId { - d, err := hex.DecodeString(s) - if err != nil || len(d) != 12 { - panic(fmt.Sprintf("invalid input to ObjectIdHex: %q", s)) - } - return ObjectId(d) -} - -// IsObjectIdHex returns whether s is a valid hex representation of -// an ObjectId. See the ObjectIdHex function. -func IsObjectIdHex(s string) bool { - if len(s) != 24 { - return false - } - _, err := hex.DecodeString(s) - return err == nil -} - -// objectIdCounter is atomically incremented when generating a new ObjectId -// using NewObjectId() function. It's used as a counter part of an id. -var objectIdCounter = readRandomUint32() - -// readRandomUint32 returns a random objectIdCounter. -func readRandomUint32() uint32 { - var b [4]byte - _, err := io.ReadFull(rand.Reader, b[:]) - if err != nil { - panic(fmt.Errorf("cannot read random object id: %v", err)) - } - return uint32((uint32(b[0]) << 0) | (uint32(b[1]) << 8) | (uint32(b[2]) << 16) | (uint32(b[3]) << 24)) -} - -// machineId stores machine id generated once and used in subsequent calls -// to NewObjectId function. -var machineId = readMachineId() -var processId = os.Getpid() - -// readMachineId generates and returns a machine id. -// If this function fails to get the hostname it will cause a runtime error. -func readMachineId() []byte { - var sum [3]byte - id := sum[:] - hostname, err1 := os.Hostname() - if err1 != nil { - _, err2 := io.ReadFull(rand.Reader, id) - if err2 != nil { - panic(fmt.Errorf("cannot get hostname: %v; %v", err1, err2)) - } - return id - } - hw := md5.New() - hw.Write([]byte(hostname)) - copy(id, hw.Sum(nil)) - return id -} - -// NewObjectId returns a new unique ObjectId. -func NewObjectId() ObjectId { - var b [12]byte - // Timestamp, 4 bytes, big endian - binary.BigEndian.PutUint32(b[:], uint32(time.Now().Unix())) - // Machine, first 3 bytes of md5(hostname) - b[4] = machineId[0] - b[5] = machineId[1] - b[6] = machineId[2] - // Pid, 2 bytes, specs don't specify endianness, but we use big endian. - b[7] = byte(processId >> 8) - b[8] = byte(processId) - // Increment, 3 bytes, big endian - i := atomic.AddUint32(&objectIdCounter, 1) - b[9] = byte(i >> 16) - b[10] = byte(i >> 8) - b[11] = byte(i) - return ObjectId(b[:]) -} - -// NewObjectIdWithTime returns a dummy ObjectId with the timestamp part filled -// with the provided number of seconds from epoch UTC, and all other parts -// filled with zeroes. It's not safe to insert a document with an id generated -// by this method, it is useful only for queries to find documents with ids -// generated before or after the specified timestamp. -func NewObjectIdWithTime(t time.Time) ObjectId { - var b [12]byte - binary.BigEndian.PutUint32(b[:4], uint32(t.Unix())) - return ObjectId(string(b[:])) -} - -// String returns a hex string representation of the id. -// Example: ObjectIdHex("4d88e15b60f486e428412dc9"). -func (id ObjectId) String() string { - return fmt.Sprintf(`ObjectIdHex("%x")`, string(id)) -} - -// Hex returns a hex representation of the ObjectId. -func (id ObjectId) Hex() string { - return hex.EncodeToString([]byte(id)) -} - -// MarshalJSON turns a bson.ObjectId into a json.Marshaller. -func (id ObjectId) MarshalJSON() ([]byte, error) { - return []byte(fmt.Sprintf(`"%x"`, string(id))), nil -} - -var nullBytes = []byte("null") - -// UnmarshalJSON turns *bson.ObjectId into a json.Unmarshaller. -func (id *ObjectId) UnmarshalJSON(data []byte) error { - if len(data) > 0 && (data[0] == '{' || data[0] == 'O') { - var v struct { - Id json.RawMessage `json:"$oid"` - Func struct { - Id json.RawMessage - } `json:"$oidFunc"` - } - err := jdec(data, &v) - if err == nil { - if len(v.Id) > 0 { - data = []byte(v.Id) - } else { - data = []byte(v.Func.Id) - } - } - } - if len(data) == 2 && data[0] == '"' && data[1] == '"' || bytes.Equal(data, nullBytes) { - *id = "" - return nil - } - if len(data) != 26 || data[0] != '"' || data[25] != '"' { - return fmt.Errorf("invalid ObjectId in JSON: %s", string(data)) - } - var buf [12]byte - _, err := hex.Decode(buf[:], data[1:25]) - if err != nil { - return fmt.Errorf("invalid ObjectId in JSON: %s (%s)", string(data), err) - } - *id = ObjectId(string(buf[:])) - return nil -} - -// MarshalText turns bson.ObjectId into an encoding.TextMarshaler. -func (id ObjectId) MarshalText() ([]byte, error) { - return []byte(fmt.Sprintf("%x", string(id))), nil -} - -// UnmarshalText turns *bson.ObjectId into an encoding.TextUnmarshaler. -func (id *ObjectId) UnmarshalText(data []byte) error { - if len(data) == 1 && data[0] == ' ' || len(data) == 0 { - *id = "" - return nil - } - if len(data) != 24 { - return fmt.Errorf("invalid ObjectId: %s", data) - } - var buf [12]byte - _, err := hex.Decode(buf[:], data[:]) - if err != nil { - return fmt.Errorf("invalid ObjectId: %s (%s)", data, err) - } - *id = ObjectId(string(buf[:])) - return nil -} - -// Valid returns true if id is valid. A valid id must contain exactly 12 bytes. -func (id ObjectId) Valid() bool { - return len(id) == 12 -} - -// byteSlice returns byte slice of id from start to end. -// Calling this function with an invalid id will cause a runtime panic. -func (id ObjectId) byteSlice(start, end int) []byte { - if len(id) != 12 { - panic(fmt.Sprintf("invalid ObjectId: %q", string(id))) - } - return []byte(string(id)[start:end]) -} - -// Time returns the timestamp part of the id. -// It's a runtime error to call this method with an invalid id. -func (id ObjectId) Time() time.Time { - // First 4 bytes of ObjectId is 32-bit big-endian seconds from epoch. - secs := int64(binary.BigEndian.Uint32(id.byteSlice(0, 4))) - return time.Unix(secs, 0) -} - -// Machine returns the 3-byte machine id part of the id. -// It's a runtime error to call this method with an invalid id. -func (id ObjectId) Machine() []byte { - return id.byteSlice(4, 7) -} - -// Pid returns the process id part of the id. -// It's a runtime error to call this method with an invalid id. -func (id ObjectId) Pid() uint16 { - return binary.BigEndian.Uint16(id.byteSlice(7, 9)) -} - -// Counter returns the incrementing value part of the id. -// It's a runtime error to call this method with an invalid id. -func (id ObjectId) Counter() int32 { - b := id.byteSlice(9, 12) - // Counter is stored as big-endian 3-byte value - return int32(uint32(b[0])<<16 | uint32(b[1])<<8 | uint32(b[2])) -} - -// The Symbol type is similar to a string and is used in languages with a -// distinct symbol type. -type Symbol string - -// Now returns the current time with millisecond precision. MongoDB stores -// timestamps with the same precision, so a Time returned from this method -// will not change after a roundtrip to the database. That's the only reason -// why this function exists. Using the time.Now function also works fine -// otherwise. -func Now() time.Time { - return time.Unix(0, time.Now().UnixNano()/1e6*1e6) -} - -// MongoTimestamp is a special internal type used by MongoDB that for some -// strange reason has its own datatype defined in BSON. -type MongoTimestamp int64 - -// Time returns the time part of ts which is stored with second precision. -func (ts MongoTimestamp) Time() time.Time { - return time.Unix(int64(uint64(ts)>>32), 0) -} - -// Counter returns the counter part of ts. -func (ts MongoTimestamp) Counter() uint32 { - return uint32(ts) -} - -// NewMongoTimestamp creates a timestamp using the given -// date `t` (with second precision) and counter `c` (unique for `t`). -// -// Returns an error if time `t` is not between 1970-01-01T00:00:00Z -// and 2106-02-07T06:28:15Z (inclusive). -// -// Note that two MongoTimestamps should never have the same (time, counter) combination: -// the caller must ensure the counter `c` is increased if creating multiple MongoTimestamp -// values for the same time `t` (ignoring fractions of seconds). -func NewMongoTimestamp(t time.Time, c uint32) (MongoTimestamp, error) { - u := t.Unix() - if u < 0 || u > math.MaxUint32 { - return -1, errors.New("invalid value for time") - } - - i := int64(u<<32 | int64(c)) - - return MongoTimestamp(i), nil -} - -type orderKey int64 - -// MaxKey is a special value that compares higher than all other possible BSON -// values in a MongoDB database. -var MaxKey = orderKey(1<<63 - 1) - -// MinKey is a special value that compares lower than all other possible BSON -// values in a MongoDB database. -var MinKey = orderKey(-1 << 63) - -type undefined struct{} - -// Undefined represents the undefined BSON value. -var Undefined undefined - -// Binary is a representation for non-standard binary values. Any kind should -// work, but the following are known as of this writing: -// -// 0x00 - Generic. This is decoded as []byte(data), not Binary{0x00, data}. -// 0x01 - Function (!?) -// 0x02 - Obsolete generic. -// 0x03 - UUID -// 0x05 - MD5 -// 0x80 - User defined. -// -type Binary struct { - Kind byte - Data []byte -} - -// RegEx represents a regular expression. The Options field may contain -// individual characters defining the way in which the pattern should be -// applied, and must be sorted. Valid options as of this writing are 'i' for -// case insensitive matching, 'm' for multi-line matching, 'x' for verbose -// mode, 'l' to make \w, \W, and similar be locale-dependent, 's' for dot-all -// mode (a '.' matches everything), and 'u' to make \w, \W, and similar match -// unicode. The value of the Options parameter is not verified before being -// marshaled into the BSON format. -type RegEx struct { - Pattern string - Options string -} - -// JavaScript is a type that holds JavaScript code. If Scope is non-nil, it -// will be marshaled as a mapping from identifiers to values that may be -// used when evaluating the provided Code. -type JavaScript struct { - Code string - Scope interface{} -} - -// DBPointer refers to a document id in a namespace. -// -// This type is deprecated in the BSON specification and should not be used -// except for backwards compatibility with ancient applications. -type DBPointer struct { - Namespace string - Id ObjectId -} - -const initialBufferSize = 64 - -func handleErr(err *error) { - if r := recover(); r != nil { - if _, ok := r.(runtime.Error); ok { - panic(r) - } else if _, ok := r.(externalPanic); ok { - panic(r) - } else if s, ok := r.(string); ok { - *err = errors.New(s) - } else if e, ok := r.(error); ok { - *err = e - } else { - panic(r) - } - } -} - -// Marshal serializes the in value, which may be a map or a struct value. -// In the case of struct values, only exported fields will be serialized, -// and the order of serialized fields will match that of the struct itself. -// The lowercased field name is used as the key for each exported field, -// but this behavior may be changed using the respective field tag. -// The tag may also contain flags to tweak the marshalling behavior for -// the field. The tag formats accepted are: -// -// "[][,[,]]" -// -// `(...) bson:"[][,[,]]" (...)` -// -// The following flags are currently supported: -// -// omitempty Only include the field if it's not set to the zero -// value for the type or to empty slices or maps. -// -// minsize Marshal an int64 value as an int32, if that's feasible -// while preserving the numeric value. -// -// inline Inline the field, which must be a struct or a map, -// causing all of its fields or keys to be processed as if -// they were part of the outer struct. For maps, keys must -// not conflict with the bson keys of other struct fields. -// -// Some examples: -// -// type T struct { -// A bool -// B int "myb" -// C string "myc,omitempty" -// D string `bson:",omitempty" json:"jsonkey"` -// E int64 ",minsize" -// F int64 "myf,omitempty,minsize" -// } -// -func Marshal(in interface{}) (out []byte, err error) { - return MarshalBuffer(in, make([]byte, 0, initialBufferSize)) -} - -// MarshalBuffer behaves the same way as Marshal, except that instead of -// allocating a new byte slice it tries to use the received byte slice and -// only allocates more memory if necessary to fit the marshaled value. -func MarshalBuffer(in interface{}, buf []byte) (out []byte, err error) { - defer handleErr(&err) - e := &encoder{buf} - e.addDoc(reflect.ValueOf(in)) - return e.out, nil -} - -// Unmarshal deserializes data from in into the out value. The out value -// must be a map, a pointer to a struct, or a pointer to a bson.D value. -// In the case of struct values, only exported fields will be deserialized. -// The lowercased field name is used as the key for each exported field, -// but this behavior may be changed using the respective field tag. -// The tag may also contain flags to tweak the marshalling behavior for -// the field. The tag formats accepted are: -// -// "[][,[,]]" -// -// `(...) bson:"[][,[,]]" (...)` -// -// The following flags are currently supported during unmarshal (see the -// Marshal method for other flags): -// -// inline Inline the field, which must be a struct or a map. -// Inlined structs are handled as if its fields were part -// of the outer struct. An inlined map causes keys that do -// not match any other struct field to be inserted in the -// map rather than being discarded as usual. -// -// The target field or element types of out may not necessarily match -// the BSON values of the provided data. The following conversions are -// made automatically: -// -// - Numeric types are converted if at least the integer part of the -// value would be preserved correctly -// - Bools are converted to numeric types as 1 or 0 -// - Numeric types are converted to bools as true if not 0 or false otherwise -// - Binary and string BSON data is converted to a string, array or byte slice -// -// If the value would not fit the type and cannot be converted, it's -// silently skipped. -// -// Pointer values are initialized when necessary. -func Unmarshal(in []byte, out interface{}) (err error) { - if raw, ok := out.(*Raw); ok { - raw.Kind = 3 - raw.Data = in - return nil - } - defer handleErr(&err) - v := reflect.ValueOf(out) - switch v.Kind() { - case reflect.Ptr: - fallthrough - case reflect.Map: - d := newDecoder(in) - d.readDocTo(v) - if d.i < len(d.in) { - return errors.New("document is corrupted") - } - case reflect.Struct: - return errors.New("unmarshal can't deal with struct values. Use a pointer") - default: - return errors.New("unmarshal needs a map or a pointer to a struct") - } - return nil -} - -// Unmarshal deserializes raw into the out value. If the out value type -// is not compatible with raw, a *bson.TypeError is returned. -// -// See the Unmarshal function documentation for more details on the -// unmarshalling process. -func (raw Raw) Unmarshal(out interface{}) (err error) { - defer handleErr(&err) - v := reflect.ValueOf(out) - switch v.Kind() { - case reflect.Ptr: - v = v.Elem() - fallthrough - case reflect.Map: - d := newDecoder(raw.Data) - good := d.readElemTo(v, raw.Kind) - if !good { - return &TypeError{v.Type(), raw.Kind} - } - case reflect.Struct: - return errors.New("raw Unmarshal can't deal with struct values. Use a pointer") - default: - return errors.New("raw Unmarshal needs a map or a valid pointer") - } - return nil -} - -// TypeError store details for type error occuring -// during unmarshaling -type TypeError struct { - Type reflect.Type - Kind byte -} - -func (e *TypeError) Error() string { - return fmt.Sprintf("BSON kind 0x%02x isn't compatible with type %s", e.Kind, e.Type.String()) -} - -// -------------------------------------------------------------------------- -// Maintain a mapping of keys to structure field indexes - -type structInfo struct { - FieldsMap map[string]fieldInfo - FieldsList []fieldInfo - InlineMap int - Zero reflect.Value -} - -type fieldInfo struct { - Key string - Num int - OmitEmpty bool - MinSize bool - Inline []int -} - -var structMap = make(map[reflect.Type]*structInfo) -var structMapMutex sync.RWMutex - -type externalPanic string - -func (e externalPanic) String() string { - return string(e) -} - -func getStructInfo(st reflect.Type) (*structInfo, error) { - structMapMutex.RLock() - sinfo, found := structMap[st] - structMapMutex.RUnlock() - if found { - return sinfo, nil - } - n := st.NumField() - fieldsMap := make(map[string]fieldInfo) - fieldsList := make([]fieldInfo, 0, n) - inlineMap := -1 - for i := 0; i != n; i++ { - field := st.Field(i) - if field.PkgPath != "" && !field.Anonymous { - continue // Private field - } - - info := fieldInfo{Num: i} - - tag := field.Tag.Get("bson") - - // Fall-back to JSON struct tag, if feature flag is set. - if tag == "" && useJSONTagFallback { - tag = field.Tag.Get("json") - } - - // If there's no bson/json tag available. - if tag == "" { - // If there's no tag, and also no tag: value splits (i.e. no colon) - // then assume the entire tag is the value - if strings.Index(string(field.Tag), ":") < 0 { - tag = string(field.Tag) - } - } - - if tag == "-" { - continue - } - - inline := false - fields := strings.Split(tag, ",") - if len(fields) > 1 { - for _, flag := range fields[1:] { - switch flag { - case "omitempty": - info.OmitEmpty = true - case "minsize": - info.MinSize = true - case "inline": - inline = true - default: - msg := fmt.Sprintf("Unsupported flag %q in tag %q of type %s", flag, tag, st) - panic(externalPanic(msg)) - } - } - tag = fields[0] - } - - if inline { - switch field.Type.Kind() { - case reflect.Map: - if inlineMap >= 0 { - return nil, errors.New("Multiple ,inline maps in struct " + st.String()) - } - if field.Type.Key() != reflect.TypeOf("") { - return nil, errors.New("Option ,inline needs a map with string keys in struct " + st.String()) - } - inlineMap = info.Num - case reflect.Ptr: - // allow only pointer to struct - if kind := field.Type.Elem().Kind(); kind != reflect.Struct { - return nil, errors.New("Option ,inline allows a pointer only to a struct, was given pointer to " + kind.String()) - } - - field.Type = field.Type.Elem() - fallthrough - case reflect.Struct: - sinfo, err := getStructInfo(field.Type) - if err != nil { - return nil, err - } - for _, finfo := range sinfo.FieldsList { - if _, found := fieldsMap[finfo.Key]; found { - msg := "Duplicated key '" + finfo.Key + "' in struct " + st.String() - return nil, errors.New(msg) - } - if finfo.Inline == nil { - finfo.Inline = []int{i, finfo.Num} - } else { - finfo.Inline = append([]int{i}, finfo.Inline...) - } - fieldsMap[finfo.Key] = finfo - fieldsList = append(fieldsList, finfo) - } - default: - panic("Option ,inline needs a struct value or a pointer to a struct or map field") - } - continue - } - - if tag != "" { - info.Key = tag - } else { - info.Key = strings.ToLower(field.Name) - } - - if _, found = fieldsMap[info.Key]; found { - msg := "Duplicated key '" + info.Key + "' in struct " + st.String() - return nil, errors.New(msg) - } - - fieldsList = append(fieldsList, info) - fieldsMap[info.Key] = info - } - sinfo = &structInfo{ - fieldsMap, - fieldsList, - inlineMap, - reflect.New(st).Elem(), - } - structMapMutex.Lock() - structMap[st] = sinfo - structMapMutex.Unlock() - return sinfo, nil -} diff --git a/vendor/github.com/globalsign/mgo/bson/compatibility.go b/vendor/github.com/globalsign/mgo/bson/compatibility.go deleted file mode 100644 index 66efd465..00000000 --- a/vendor/github.com/globalsign/mgo/bson/compatibility.go +++ /dev/null @@ -1,29 +0,0 @@ -package bson - -// Current state of the JSON tag fallback option. -var useJSONTagFallback = false -var useRespectNilValues = false - -// SetJSONTagFallback enables or disables the JSON-tag fallback for structure tagging. When this is enabled, structures -// without BSON tags on a field will fall-back to using the JSON tag (if present). -func SetJSONTagFallback(state bool) { - useJSONTagFallback = state -} - -// JSONTagFallbackState returns the current status of the JSON tag fallback compatability option. See SetJSONTagFallback -// for more information. -func JSONTagFallbackState() bool { - return useJSONTagFallback -} - -// SetRespectNilValues enables or disables serializing nil slices or maps to `null` values. -// In other words it enables `encoding/json` compatible behaviour. -func SetRespectNilValues(state bool) { - useRespectNilValues = state -} - -// RespectNilValuesState returns the current status of the JSON nil slices and maps fallback compatibility option. -// See SetRespectNilValues for more information. -func RespectNilValuesState() bool { - return useRespectNilValues -} diff --git a/vendor/github.com/globalsign/mgo/bson/decimal.go b/vendor/github.com/globalsign/mgo/bson/decimal.go deleted file mode 100644 index 672ba182..00000000 --- a/vendor/github.com/globalsign/mgo/bson/decimal.go +++ /dev/null @@ -1,312 +0,0 @@ -// BSON library for Go -// -// Copyright (c) 2010-2012 - Gustavo Niemeyer -// -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are met: -// -// 1. Redistributions of source code must retain the above copyright notice, this -// list of conditions and the following disclaimer. -// 2. Redistributions in binary form must reproduce the above copyright notice, -// this list of conditions and the following disclaimer in the documentation -// and/or other materials provided with the distribution. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR -// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -package bson - -import ( - "fmt" - "strconv" - "strings" -) - -// Decimal128 holds decimal128 BSON values. -type Decimal128 struct { - h, l uint64 -} - -func (d Decimal128) String() string { - var pos int // positive sign - var e int // exponent - var h, l uint64 // significand high/low - - if d.h>>63&1 == 0 { - pos = 1 - } - - switch d.h >> 58 & (1<<5 - 1) { - case 0x1F: - return "NaN" - case 0x1E: - return "-Inf"[pos:] - } - - l = d.l - if d.h>>61&3 == 3 { - // Bits: 1*sign 2*ignored 14*exponent 111*significand. - // Implicit 0b100 prefix in significand. - e = int(d.h>>47&(1<<14-1)) - 6176 - //h = 4<<47 | d.h&(1<<47-1) - // Spec says all of these values are out of range. - h, l = 0, 0 - } else { - // Bits: 1*sign 14*exponent 113*significand - e = int(d.h>>49&(1<<14-1)) - 6176 - h = d.h & (1<<49 - 1) - } - - // Would be handled by the logic below, but that's trivial and common. - if h == 0 && l == 0 && e == 0 { - return "-0"[pos:] - } - - var repr [48]byte // Loop 5 times over 9 digits plus dot, negative sign, and leading zero. - var last = len(repr) - var i = len(repr) - var dot = len(repr) + e - var rem uint32 -Loop: - for d9 := 0; d9 < 5; d9++ { - h, l, rem = divmod(h, l, 1e9) - for d1 := 0; d1 < 9; d1++ { - // Handle "-0.0", "0.00123400", "-1.00E-6", "1.050E+3", etc. - if i < len(repr) && (dot == i || l == 0 && h == 0 && rem > 0 && rem < 10 && (dot < i-6 || e > 0)) { - e += len(repr) - i - i-- - repr[i] = '.' - last = i - 1 - dot = len(repr) // Unmark. - } - c := '0' + byte(rem%10) - rem /= 10 - i-- - repr[i] = c - // Handle "0E+3", "1E+3", etc. - if l == 0 && h == 0 && rem == 0 && i == len(repr)-1 && (dot < i-5 || e > 0) { - last = i - break Loop - } - if c != '0' { - last = i - } - // Break early. Works without it, but why. - if dot > i && l == 0 && h == 0 && rem == 0 { - break Loop - } - } - } - repr[last-1] = '-' - last-- - - if e > 0 { - return string(repr[last+pos:]) + "E+" + strconv.Itoa(e) - } - if e < 0 { - return string(repr[last+pos:]) + "E" + strconv.Itoa(e) - } - return string(repr[last+pos:]) -} - -func divmod(h, l uint64, div uint32) (qh, ql uint64, rem uint32) { - div64 := uint64(div) - a := h >> 32 - aq := a / div64 - ar := a % div64 - b := ar<<32 + h&(1<<32-1) - bq := b / div64 - br := b % div64 - c := br<<32 + l>>32 - cq := c / div64 - cr := c % div64 - d := cr<<32 + l&(1<<32-1) - dq := d / div64 - dr := d % div64 - return (aq<<32 | bq), (cq<<32 | dq), uint32(dr) -} - -var dNaN = Decimal128{0x1F << 58, 0} -var dPosInf = Decimal128{0x1E << 58, 0} -var dNegInf = Decimal128{0x3E << 58, 0} - -func dErr(s string) (Decimal128, error) { - return dNaN, fmt.Errorf("cannot parse %q as a decimal128", s) -} - -// ParseDecimal128 parse a string and return the corresponding value as -// a decimal128 -func ParseDecimal128(s string) (Decimal128, error) { - orig := s - if s == "" { - return dErr(orig) - } - neg := s[0] == '-' - if neg || s[0] == '+' { - s = s[1:] - } - - if (len(s) == 3 || len(s) == 8) && (s[0] == 'N' || s[0] == 'n' || s[0] == 'I' || s[0] == 'i') { - if s == "NaN" || s == "nan" || strings.EqualFold(s, "nan") { - return dNaN, nil - } - if s == "Inf" || s == "inf" || strings.EqualFold(s, "inf") || strings.EqualFold(s, "infinity") { - if neg { - return dNegInf, nil - } - return dPosInf, nil - } - return dErr(orig) - } - - var h, l uint64 - var e int - - var add, ovr uint32 - var mul uint32 = 1 - var dot = -1 - var digits = 0 - var i = 0 - for i < len(s) { - c := s[i] - if mul == 1e9 { - h, l, ovr = muladd(h, l, mul, add) - mul, add = 1, 0 - if ovr > 0 || h&((1<<15-1)<<49) > 0 { - return dErr(orig) - } - } - if c >= '0' && c <= '9' { - i++ - if c > '0' || digits > 0 { - digits++ - } - if digits > 34 { - if c == '0' { - // Exact rounding. - e++ - continue - } - return dErr(orig) - } - mul *= 10 - add *= 10 - add += uint32(c - '0') - continue - } - if c == '.' { - i++ - if dot >= 0 || i == 1 && len(s) == 1 { - return dErr(orig) - } - if i == len(s) { - break - } - if s[i] < '0' || s[i] > '9' || e > 0 { - return dErr(orig) - } - dot = i - continue - } - break - } - if i == 0 { - return dErr(orig) - } - if mul > 1 { - h, l, ovr = muladd(h, l, mul, add) - if ovr > 0 || h&((1<<15-1)<<49) > 0 { - return dErr(orig) - } - } - if dot >= 0 { - e += dot - i - } - if i+1 < len(s) && (s[i] == 'E' || s[i] == 'e') { - i++ - eneg := s[i] == '-' - if eneg || s[i] == '+' { - i++ - if i == len(s) { - return dErr(orig) - } - } - n := 0 - for i < len(s) && n < 1e4 { - c := s[i] - i++ - if c < '0' || c > '9' { - return dErr(orig) - } - n *= 10 - n += int(c - '0') - } - if eneg { - n = -n - } - e += n - for e < -6176 { - // Subnormal. - var div uint32 = 1 - for div < 1e9 && e < -6176 { - div *= 10 - e++ - } - var rem uint32 - h, l, rem = divmod(h, l, div) - if rem > 0 { - return dErr(orig) - } - } - for e > 6111 { - // Clamped. - var mul uint32 = 1 - for mul < 1e9 && e > 6111 { - mul *= 10 - e-- - } - h, l, ovr = muladd(h, l, mul, 0) - if ovr > 0 || h&((1<<15-1)<<49) > 0 { - return dErr(orig) - } - } - if e < -6176 || e > 6111 { - return dErr(orig) - } - } - - if i < len(s) { - return dErr(orig) - } - - h |= uint64(e+6176) & uint64(1<<14-1) << 49 - if neg { - h |= 1 << 63 - } - return Decimal128{h, l}, nil -} - -func muladd(h, l uint64, mul uint32, add uint32) (resh, resl uint64, overflow uint32) { - mul64 := uint64(mul) - a := mul64 * (l & (1<<32 - 1)) - b := a>>32 + mul64*(l>>32) - c := b>>32 + mul64*(h&(1<<32-1)) - d := c>>32 + mul64*(h>>32) - - a = a&(1<<32-1) + uint64(add) - b = b&(1<<32-1) + a>>32 - c = c&(1<<32-1) + b>>32 - d = d&(1<<32-1) + c>>32 - - return (d<<32 | c&(1<<32-1)), (b<<32 | a&(1<<32-1)), uint32(d >> 32) -} diff --git a/vendor/github.com/globalsign/mgo/bson/decode.go b/vendor/github.com/globalsign/mgo/bson/decode.go deleted file mode 100644 index 658856ad..00000000 --- a/vendor/github.com/globalsign/mgo/bson/decode.go +++ /dev/null @@ -1,1055 +0,0 @@ -// BSON library for Go -// -// Copyright (c) 2010-2012 - Gustavo Niemeyer -// -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are met: -// -// 1. Redistributions of source code must retain the above copyright notice, this -// list of conditions and the following disclaimer. -// 2. Redistributions in binary form must reproduce the above copyright notice, -// this list of conditions and the following disclaimer in the documentation -// and/or other materials provided with the distribution. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR -// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// gobson - BSON library for Go. - -package bson - -import ( - "errors" - "fmt" - "io" - "math" - "net/url" - "reflect" - "strconv" - "sync" - "time" -) - -type decoder struct { - in []byte - i int - docType reflect.Type -} - -var typeM = reflect.TypeOf(M{}) - -func newDecoder(in []byte) *decoder { - return &decoder{in, 0, typeM} -} - -// -------------------------------------------------------------------------- -// Some helper functions. - -func corrupted() { - panic("Document is corrupted") -} - -// -------------------------------------------------------------------------- -// Unmarshaling of documents. - -const ( - setterUnknown = iota - setterNone - setterType - setterAddr -) - -var setterStyles map[reflect.Type]int -var setterIface reflect.Type -var setterMutex sync.RWMutex - -func init() { - var iface Setter - setterIface = reflect.TypeOf(&iface).Elem() - setterStyles = make(map[reflect.Type]int) -} - -func setterStyle(outt reflect.Type) int { - setterMutex.RLock() - style := setterStyles[outt] - setterMutex.RUnlock() - if style != setterUnknown { - return style - } - - setterMutex.Lock() - defer setterMutex.Unlock() - if outt.Implements(setterIface) { - style = setterType - } else if reflect.PtrTo(outt).Implements(setterIface) { - style = setterAddr - } else { - style = setterNone - } - setterStyles[outt] = style - return style -} - -func getSetter(outt reflect.Type, out reflect.Value) Setter { - style := setterStyle(outt) - if style == setterNone { - return nil - } - if style == setterAddr { - if !out.CanAddr() { - return nil - } - out = out.Addr() - } else if outt.Kind() == reflect.Ptr && out.IsNil() { - out.Set(reflect.New(outt.Elem())) - } - return out.Interface().(Setter) -} - -func clearMap(m reflect.Value) { - var none reflect.Value - for _, k := range m.MapKeys() { - m.SetMapIndex(k, none) - } -} - -func (d *decoder) readDocTo(out reflect.Value) { - var elemType reflect.Type - outt := out.Type() - outk := outt.Kind() - - for { - if outk == reflect.Ptr && out.IsNil() { - out.Set(reflect.New(outt.Elem())) - } - if setter := getSetter(outt, out); setter != nil { - raw := d.readRaw(ElementDocument) - err := setter.SetBSON(raw) - if _, ok := err.(*TypeError); err != nil && !ok { - panic(err) - } - return - } - if outk == reflect.Ptr { - out = out.Elem() - outt = out.Type() - outk = out.Kind() - continue - } - break - } - - var fieldsMap map[string]fieldInfo - var inlineMap reflect.Value - if outt == typeRaw { - out.Set(reflect.ValueOf(d.readRaw(ElementDocument))) - return - } - - origout := out - if outk == reflect.Interface { - if d.docType.Kind() == reflect.Map { - mv := reflect.MakeMap(d.docType) - out.Set(mv) - out = mv - } else { - dv := reflect.New(d.docType).Elem() - out.Set(dv) - out = dv - } - outt = out.Type() - outk = outt.Kind() - } - - docType := d.docType - keyType := typeString - convertKey := false - switch outk { - case reflect.Map: - keyType = outt.Key() - if keyType != typeString { - convertKey = true - } - elemType = outt.Elem() - if elemType == typeIface { - d.docType = outt - } - if out.IsNil() { - out.Set(reflect.MakeMap(out.Type())) - } else if out.Len() > 0 { - clearMap(out) - } - case reflect.Struct: - sinfo, err := getStructInfo(out.Type()) - if err != nil { - panic(err) - } - fieldsMap = sinfo.FieldsMap - out.Set(sinfo.Zero) - if sinfo.InlineMap != -1 { - inlineMap = out.Field(sinfo.InlineMap) - if !inlineMap.IsNil() && inlineMap.Len() > 0 { - clearMap(inlineMap) - } - elemType = inlineMap.Type().Elem() - if elemType == typeIface { - d.docType = inlineMap.Type() - } - } - case reflect.Slice: - switch outt.Elem() { - case typeDocElem: - origout.Set(d.readDocElems(outt)) - return - case typeRawDocElem: - origout.Set(d.readRawDocElems(outt)) - return - } - fallthrough - default: - panic("Unsupported document type for unmarshalling: " + out.Type().String()) - } - - end := int(d.readInt32()) - end += d.i - 4 - if end <= d.i || end > len(d.in) || d.in[end-1] != '\x00' { - corrupted() - } - for d.in[d.i] != '\x00' { - kind := d.readByte() - name := d.readCStr() - if d.i >= end { - corrupted() - } - - switch outk { - case reflect.Map: - e := reflect.New(elemType).Elem() - if d.readElemTo(e, kind) { - k := reflect.ValueOf(name) - if convertKey { - mapKeyType := out.Type().Key() - mapKeyKind := mapKeyType.Kind() - - switch mapKeyKind { - case reflect.Int: - fallthrough - case reflect.Int8: - fallthrough - case reflect.Int16: - fallthrough - case reflect.Int32: - fallthrough - case reflect.Int64: - fallthrough - case reflect.Uint: - fallthrough - case reflect.Uint8: - fallthrough - case reflect.Uint16: - fallthrough - case reflect.Uint32: - fallthrough - case reflect.Uint64: - fallthrough - case reflect.Float32: - fallthrough - case reflect.Float64: - parsed := d.parseMapKeyAsFloat(k, mapKeyKind) - k = reflect.ValueOf(parsed) - case reflect.String: - mapKeyType = keyType - default: - panic("BSON map must have string or decimal keys. Got: " + outt.String()) - } - - k = k.Convert(mapKeyType) - } - out.SetMapIndex(k, e) - } - case reflect.Struct: - if info, ok := fieldsMap[name]; ok { - if info.Inline == nil { - d.readElemTo(out.Field(info.Num), kind) - } else { - d.readElemTo(out.FieldByIndex(info.Inline), kind) - } - } else if inlineMap.IsValid() { - if inlineMap.IsNil() { - inlineMap.Set(reflect.MakeMap(inlineMap.Type())) - } - e := reflect.New(elemType).Elem() - if d.readElemTo(e, kind) { - inlineMap.SetMapIndex(reflect.ValueOf(name), e) - } - } else { - d.dropElem(kind) - } - case reflect.Slice: - } - - if d.i >= end { - corrupted() - } - } - d.i++ // '\x00' - if d.i != end { - corrupted() - } - d.docType = docType -} - -func (decoder) parseMapKeyAsFloat(k reflect.Value, mapKeyKind reflect.Kind) float64 { - parsed, err := strconv.ParseFloat(k.String(), 64) - if err != nil { - panic("Map key is defined to be a decimal type (" + mapKeyKind.String() + ") but got error " + - err.Error()) - } - - return parsed -} - -func (d *decoder) readArrayDocTo(out reflect.Value) { - end := int(d.readInt32()) - end += d.i - 4 - if end <= d.i || end > len(d.in) || d.in[end-1] != '\x00' { - corrupted() - } - i := 0 - l := out.Len() - for d.in[d.i] != '\x00' { - if i >= l { - panic("Length mismatch on array field") - } - kind := d.readByte() - for d.i < end && d.in[d.i] != '\x00' { - d.i++ - } - if d.i >= end { - corrupted() - } - d.i++ - d.readElemTo(out.Index(i), kind) - if d.i >= end { - corrupted() - } - i++ - } - if i != l { - panic("Length mismatch on array field") - } - d.i++ // '\x00' - if d.i != end { - corrupted() - } -} - -func (d *decoder) readSliceDoc(t reflect.Type) interface{} { - tmp := make([]reflect.Value, 0, 8) - elemType := t.Elem() - if elemType == typeRawDocElem { - d.dropElem(ElementArray) - return reflect.Zero(t).Interface() - } - if elemType == typeRaw { - return d.readSliceOfRaw() - } - - end := int(d.readInt32()) - end += d.i - 4 - if end <= d.i || end > len(d.in) || d.in[end-1] != '\x00' { - corrupted() - } - for d.in[d.i] != '\x00' { - kind := d.readByte() - for d.i < end && d.in[d.i] != '\x00' { - d.i++ - } - if d.i >= end { - corrupted() - } - d.i++ - e := reflect.New(elemType).Elem() - if d.readElemTo(e, kind) { - tmp = append(tmp, e) - } - if d.i >= end { - corrupted() - } - } - d.i++ // '\x00' - if d.i != end { - corrupted() - } - - n := len(tmp) - slice := reflect.MakeSlice(t, n, n) - for i := 0; i != n; i++ { - slice.Index(i).Set(tmp[i]) - } - return slice.Interface() -} - -func BSONElementSize(kind byte, offset int, buffer []byte) (int, error) { - switch kind { - case ElementFloat64: // Float64 - return 8, nil - case ElementJavaScriptWithoutScope: // JavaScript without scope - fallthrough - case ElementSymbol: // Symbol - fallthrough - case ElementString: // UTF-8 string - size, err := getSize(offset, buffer) - if err != nil { - return 0, err - } - if size < 1 { - return 0, errors.New("String size can't be less then one byte") - } - size += 4 - if offset+size > len(buffer) { - return 0, io.ErrUnexpectedEOF - } - if buffer[offset+size-1] != 0 { - return 0, errors.New("Invalid string: non zero-terminated") - } - return size, nil - case ElementArray: // Array - fallthrough - case ElementDocument: // Document - size, err := getSize(offset, buffer) - if err != nil { - return 0, err - } - if size < 5 { - return 0, errors.New("Declared document size is too small") - } - return size, nil - case ElementBinary: // Binary - size, err := getSize(offset, buffer) - if err != nil { - return 0, err - } - if size < 0 { - return 0, errors.New("Binary data size can't be negative") - } - return size + 5, nil - case Element06: // Undefined (obsolete, but still seen in the wild) - return 0, nil - case ElementObjectId: // ObjectId - return 12, nil - case ElementBool: // Bool - return 1, nil - case ElementDatetime: // Timestamp - return 8, nil - case ElementNil: // Nil - return 0, nil - case ElementRegEx: // RegEx - end := offset - for i := 0; i < 2; i++ { - for end < len(buffer) && buffer[end] != '\x00' { - end++ - } - end++ - } - if end > len(buffer) { - return 0, io.ErrUnexpectedEOF - } - return end - offset, nil - case ElementDBPointer: // DBPointer - size, err := getSize(offset, buffer) - if err != nil { - return 0, err - } - if size < 1 { - return 0, errors.New("String size can't be less then one byte") - } - return size + 12 + 4, nil - case ElementJavaScriptWithScope: // JavaScript with scope - size, err := getSize(offset, buffer) - if err != nil { - return 0, err - } - if size < 4+5+5 { - return 0, errors.New("Declared document element is too small") - } - return size, nil - case ElementInt32: // Int32 - return 4, nil - case ElementTimestamp: // Mongo-specific timestamp - return 8, nil - case ElementInt64: // Int64 - return 8, nil - case ElementDecimal128: // Decimal128 - return 16, nil - case ElementMaxKey: // Max key - return 0, nil - case ElementMinKey: // Min key - return 0, nil - default: - return 0, errors.New(fmt.Sprintf("Unknown element kind (0x%02X)", kind)) - } -} - -func (d *decoder) readRaw(kind byte) Raw { - size, err := BSONElementSize(kind, d.i, d.in) - if err != nil { - corrupted() - } - if d.i+size > len(d.in) { - corrupted() - } - d.i += size - return Raw{ - Kind: kind, - Data: d.in[d.i-size : d.i], - } -} - -func (d *decoder) readSliceOfRaw() interface{} { - tmp := make([]Raw, 0, 8) - end := int(d.readInt32()) - end += d.i - 4 - if end <= d.i || end > len(d.in) || d.in[end-1] != '\x00' { - corrupted() - } - for d.in[d.i] != '\x00' { - kind := d.readByte() - for d.i < end && d.in[d.i] != '\x00' { - d.i++ - } - if d.i >= end { - corrupted() - } - d.i++ - e := d.readRaw(kind) - tmp = append(tmp, e) - if d.i >= end { - corrupted() - } - } - d.i++ // '\x00' - if d.i != end { - corrupted() - } - return tmp -} - -var typeSlice = reflect.TypeOf([]interface{}{}) -var typeIface = typeSlice.Elem() - -func (d *decoder) readDocElems(typ reflect.Type) reflect.Value { - docType := d.docType - d.docType = typ - slice := make([]DocElem, 0, 8) - d.readDocWith(func(kind byte, name string) { - e := DocElem{Name: name} - v := reflect.ValueOf(&e.Value) - if d.readElemTo(v.Elem(), kind) { - slice = append(slice, e) - } - }) - slicev := reflect.New(typ).Elem() - slicev.Set(reflect.ValueOf(slice)) - d.docType = docType - return slicev -} - -func (d *decoder) readRawDocElems(typ reflect.Type) reflect.Value { - docType := d.docType - d.docType = typ - slice := make([]RawDocElem, 0, 8) - d.readDocWith(func(kind byte, name string) { - e := RawDocElem{Name: name, Value: d.readRaw(kind)} - slice = append(slice, e) - }) - slicev := reflect.New(typ).Elem() - slicev.Set(reflect.ValueOf(slice)) - d.docType = docType - return slicev -} - -func (d *decoder) readDocWith(f func(kind byte, name string)) { - end := int(d.readInt32()) - end += d.i - 4 - if end <= d.i || end > len(d.in) || d.in[end-1] != '\x00' { - corrupted() - } - for d.in[d.i] != '\x00' { - kind := d.readByte() - name := d.readCStr() - if d.i >= end { - corrupted() - } - f(kind, name) - if d.i >= end { - corrupted() - } - } - d.i++ // '\x00' - if d.i != end { - corrupted() - } -} - -// -------------------------------------------------------------------------- -// Unmarshaling of individual elements within a document. -func (d *decoder) dropElem(kind byte) { - size, err := BSONElementSize(kind, d.i, d.in) - if err != nil { - corrupted() - } - if d.i+size > len(d.in) { - corrupted() - } - d.i += size -} - -// Attempt to decode an element from the document and put it into out. -// If the types are not compatible, the returned ok value will be -// false and out will be unchanged. -func (d *decoder) readElemTo(out reflect.Value, kind byte) (good bool) { - outt := out.Type() - - if outt == typeRaw { - out.Set(reflect.ValueOf(d.readRaw(kind))) - return true - } - - if outt == typeRawPtr { - raw := d.readRaw(kind) - out.Set(reflect.ValueOf(&raw)) - return true - } - - if kind == ElementDocument { - // Delegate unmarshaling of documents. - outt := out.Type() - outk := out.Kind() - switch outk { - case reflect.Interface, reflect.Ptr, reflect.Struct, reflect.Map: - d.readDocTo(out) - return true - } - if setterStyle(outt) != setterNone { - d.readDocTo(out) - return true - } - if outk == reflect.Slice { - switch outt.Elem() { - case typeDocElem: - out.Set(d.readDocElems(outt)) - case typeRawDocElem: - out.Set(d.readRawDocElems(outt)) - default: - d.dropElem(kind) - } - return true - } - d.dropElem(kind) - return true - } - - if setter := getSetter(outt, out); setter != nil { - err := setter.SetBSON(d.readRaw(kind)) - if err == ErrSetZero { - out.Set(reflect.Zero(outt)) - return true - } - if err == nil { - return true - } - if _, ok := err.(*TypeError); !ok { - panic(err) - } - return false - } - - var in interface{} - - switch kind { - case ElementFloat64: - in = d.readFloat64() - case ElementString: - in = d.readStr() - case ElementDocument: - panic("Can't happen. Handled above.") - case ElementArray: - outt := out.Type() - if setterStyle(outt) != setterNone { - // Skip the value so its data is handed to the setter below. - d.dropElem(kind) - break - } - for outt.Kind() == reflect.Ptr { - outt = outt.Elem() - } - switch outt.Kind() { - case reflect.Array: - d.readArrayDocTo(out) - return true - case reflect.Slice: - in = d.readSliceDoc(outt) - default: - in = d.readSliceDoc(typeSlice) - } - case ElementBinary: - b := d.readBinary() - if b.Kind == BinaryGeneric || b.Kind == BinaryBinaryOld { - in = b.Data - } else { - in = b - } - case Element06: // Undefined (obsolete, but still seen in the wild) - in = Undefined - case ElementObjectId: - in = ObjectId(d.readBytes(12)) - case ElementBool: - in = d.readBool() - case ElementDatetime: // Timestamp - // MongoDB handles timestamps as milliseconds. - i := d.readInt64() - if i == -62135596800000 { - in = time.Time{} // In UTC for convenience. - } else { - in = time.Unix(i/1e3, i%1e3*1e6).UTC() - } - case ElementNil: - in = nil - case ElementRegEx: - in = d.readRegEx() - case ElementDBPointer: - in = DBPointer{Namespace: d.readStr(), Id: ObjectId(d.readBytes(12))} - case ElementJavaScriptWithoutScope: - in = JavaScript{Code: d.readStr()} - case ElementSymbol: - in = Symbol(d.readStr()) - case ElementJavaScriptWithScope: - start := d.i - l := int(d.readInt32()) - js := JavaScript{d.readStr(), make(M)} - d.readDocTo(reflect.ValueOf(js.Scope)) - if d.i != start+l { - corrupted() - } - in = js - case ElementInt32: - in = int(d.readInt32()) - case ElementTimestamp: // Mongo-specific timestamp - in = MongoTimestamp(d.readInt64()) - case ElementInt64: - switch out.Type() { - case typeTimeDuration: - in = time.Duration(time.Duration(d.readInt64()) * time.Millisecond) - default: - in = d.readInt64() - } - case ElementDecimal128: - in = Decimal128{ - l: uint64(d.readInt64()), - h: uint64(d.readInt64()), - } - case ElementMaxKey: - in = MaxKey - case ElementMinKey: - in = MinKey - default: - panic(fmt.Sprintf("Unknown element kind (0x%02X)", kind)) - } - - if in == nil { - out.Set(reflect.Zero(outt)) - return true - } - - outk := outt.Kind() - - // Dereference and initialize pointer if necessary. - first := true - for outk == reflect.Ptr { - if !out.IsNil() { - out = out.Elem() - } else { - elem := reflect.New(outt.Elem()) - if first { - // Only set if value is compatible. - first = false - defer func(out, elem reflect.Value) { - if good { - out.Set(elem) - } - }(out, elem) - } else { - out.Set(elem) - } - out = elem - } - outt = out.Type() - outk = outt.Kind() - } - - inv := reflect.ValueOf(in) - if outt == inv.Type() { - out.Set(inv) - return true - } - - switch outk { - case reflect.Interface: - out.Set(inv) - return true - case reflect.String: - switch inv.Kind() { - case reflect.String: - out.SetString(inv.String()) - return true - case reflect.Slice: - if b, ok := in.([]byte); ok { - out.SetString(string(b)) - return true - } - case reflect.Int, reflect.Int64: - if outt == typeJSONNumber { - out.SetString(strconv.FormatInt(inv.Int(), 10)) - return true - } - case reflect.Float64: - if outt == typeJSONNumber { - out.SetString(strconv.FormatFloat(inv.Float(), 'f', -1, 64)) - return true - } - } - case reflect.Slice, reflect.Array: - // Remember, array (0x04) slices are built with the correct - // element type. If we are here, must be a cross BSON kind - // conversion (e.g. 0x05 unmarshalling on string). - if outt.Elem().Kind() != reflect.Uint8 { - break - } - switch inv.Kind() { - case reflect.String: - slice := []byte(inv.String()) - out.Set(reflect.ValueOf(slice)) - return true - case reflect.Slice: - switch outt.Kind() { - case reflect.Array: - reflect.Copy(out, inv) - case reflect.Slice: - out.SetBytes(inv.Bytes()) - } - return true - } - case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: - switch inv.Kind() { - case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: - out.SetInt(inv.Int()) - return true - case reflect.Float32, reflect.Float64: - out.SetInt(int64(inv.Float())) - return true - case reflect.Bool: - if inv.Bool() { - out.SetInt(1) - } else { - out.SetInt(0) - } - return true - case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: - panic("can't happen: no uint types in BSON (!?)") - } - case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: - switch inv.Kind() { - case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: - out.SetUint(uint64(inv.Int())) - return true - case reflect.Float32, reflect.Float64: - out.SetUint(uint64(inv.Float())) - return true - case reflect.Bool: - if inv.Bool() { - out.SetUint(1) - } else { - out.SetUint(0) - } - return true - case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: - panic("Can't happen. No uint types in BSON.") - } - case reflect.Float32, reflect.Float64: - switch inv.Kind() { - case reflect.Float32, reflect.Float64: - out.SetFloat(inv.Float()) - return true - case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: - out.SetFloat(float64(inv.Int())) - return true - case reflect.Bool: - if inv.Bool() { - out.SetFloat(1) - } else { - out.SetFloat(0) - } - return true - case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: - panic("Can't happen. No uint types in BSON?") - } - case reflect.Bool: - switch inv.Kind() { - case reflect.Bool: - out.SetBool(inv.Bool()) - return true - case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: - out.SetBool(inv.Int() != 0) - return true - case reflect.Float32, reflect.Float64: - out.SetBool(inv.Float() != 0) - return true - case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: - panic("Can't happen. No uint types in BSON?") - } - case reflect.Struct: - if outt == typeURL && inv.Kind() == reflect.String { - u, err := url.Parse(inv.String()) - if err != nil { - panic(err) - } - out.Set(reflect.ValueOf(u).Elem()) - return true - } - if outt == typeBinary { - if b, ok := in.([]byte); ok { - out.Set(reflect.ValueOf(Binary{Data: b})) - return true - } - } - } - - return false -} - -// -------------------------------------------------------------------------- -// Parsers of basic types. - -func (d *decoder) readRegEx() RegEx { - re := RegEx{} - re.Pattern = d.readCStr() - re.Options = d.readCStr() - return re -} - -func (d *decoder) readBinary() Binary { - l := d.readInt32() - b := Binary{} - b.Kind = d.readByte() - if b.Kind == BinaryBinaryOld && l > 4 { - // Weird obsolete format with redundant length. - rl := d.readInt32() - if rl != l-4 { - corrupted() - } - l = rl - } - b.Data = d.readBytes(l) - return b -} - -func (d *decoder) readStr() string { - l := d.readInt32() - b := d.readBytes(l - 1) - if d.readByte() != '\x00' { - corrupted() - } - return string(b) -} - -func (d *decoder) readCStr() string { - start := d.i - end := start - l := len(d.in) - for ; end != l; end++ { - if d.in[end] == '\x00' { - break - } - } - d.i = end + 1 - if d.i > l { - corrupted() - } - return string(d.in[start:end]) -} - -func (d *decoder) readBool() bool { - b := d.readByte() - if b == 0 { - return false - } - if b == 1 { - return true - } - panic(fmt.Sprintf("encoded boolean must be 1 or 0, found %d", b)) -} - -func (d *decoder) readFloat64() float64 { - return math.Float64frombits(uint64(d.readInt64())) -} - -func (d *decoder) readInt32() int32 { - b := d.readBytes(4) - return int32((uint32(b[0]) << 0) | - (uint32(b[1]) << 8) | - (uint32(b[2]) << 16) | - (uint32(b[3]) << 24)) -} - -func getSize(offset int, b []byte) (int, error) { - if offset+4 > len(b) { - return 0, io.ErrUnexpectedEOF - } - return int((uint32(b[offset]) << 0) | - (uint32(b[offset+1]) << 8) | - (uint32(b[offset+2]) << 16) | - (uint32(b[offset+3]) << 24)), nil -} - -func (d *decoder) readInt64() int64 { - b := d.readBytes(8) - return int64((uint64(b[0]) << 0) | - (uint64(b[1]) << 8) | - (uint64(b[2]) << 16) | - (uint64(b[3]) << 24) | - (uint64(b[4]) << 32) | - (uint64(b[5]) << 40) | - (uint64(b[6]) << 48) | - (uint64(b[7]) << 56)) -} - -func (d *decoder) readByte() byte { - i := d.i - d.i++ - if d.i > len(d.in) { - corrupted() - } - return d.in[i] -} - -func (d *decoder) readBytes(length int32) []byte { - if length < 0 { - corrupted() - } - start := d.i - d.i += int(length) - if d.i < start || d.i > len(d.in) { - corrupted() - } - return d.in[start : start+int(length)] -} diff --git a/vendor/github.com/globalsign/mgo/bson/encode.go b/vendor/github.com/globalsign/mgo/bson/encode.go deleted file mode 100644 index d0c6b2a8..00000000 --- a/vendor/github.com/globalsign/mgo/bson/encode.go +++ /dev/null @@ -1,645 +0,0 @@ -// BSON library for Go -// -// Copyright (c) 2010-2012 - Gustavo Niemeyer -// -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are met: -// -// 1. Redistributions of source code must retain the above copyright notice, this -// list of conditions and the following disclaimer. -// 2. Redistributions in binary form must reproduce the above copyright notice, -// this list of conditions and the following disclaimer in the documentation -// and/or other materials provided with the distribution. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR -// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// gobson - BSON library for Go. - -package bson - -import ( - "encoding/json" - "fmt" - "math" - "net/url" - "reflect" - "sort" - "strconv" - "sync" - "time" -) - -// -------------------------------------------------------------------------- -// Some internal infrastructure. - -var ( - typeBinary = reflect.TypeOf(Binary{}) - typeObjectId = reflect.TypeOf(ObjectId("")) - typeDBPointer = reflect.TypeOf(DBPointer{"", ObjectId("")}) - typeSymbol = reflect.TypeOf(Symbol("")) - typeMongoTimestamp = reflect.TypeOf(MongoTimestamp(0)) - typeOrderKey = reflect.TypeOf(MinKey) - typeDocElem = reflect.TypeOf(DocElem{}) - typeRawDocElem = reflect.TypeOf(RawDocElem{}) - typeRaw = reflect.TypeOf(Raw{}) - typeRawPtr = reflect.PtrTo(reflect.TypeOf(Raw{})) - typeURL = reflect.TypeOf(url.URL{}) - typeTime = reflect.TypeOf(time.Time{}) - typeString = reflect.TypeOf("") - typeJSONNumber = reflect.TypeOf(json.Number("")) - typeTimeDuration = reflect.TypeOf(time.Duration(0)) -) - -var ( - // spec for []uint8 or []byte encoding - arrayOps = map[string]bool{ - "$in": true, - "$nin": true, - "$all": true, - } -) - -const itoaCacheSize = 32 - -const ( - getterUnknown = iota - getterNone - getterTypeVal - getterTypePtr - getterAddr -) - -var itoaCache []string - -var getterStyles map[reflect.Type]int -var getterIface reflect.Type -var getterMutex sync.RWMutex - -func init() { - itoaCache = make([]string, itoaCacheSize) - for i := 0; i != itoaCacheSize; i++ { - itoaCache[i] = strconv.Itoa(i) - } - var iface Getter - getterIface = reflect.TypeOf(&iface).Elem() - getterStyles = make(map[reflect.Type]int) -} - -func itoa(i int) string { - if i < itoaCacheSize { - return itoaCache[i] - } - return strconv.Itoa(i) -} - -func getterStyle(outt reflect.Type) int { - getterMutex.RLock() - style := getterStyles[outt] - getterMutex.RUnlock() - if style != getterUnknown { - return style - } - - getterMutex.Lock() - defer getterMutex.Unlock() - if outt.Implements(getterIface) { - vt := outt - for vt.Kind() == reflect.Ptr { - vt = vt.Elem() - } - if vt.Implements(getterIface) { - style = getterTypeVal - } else { - style = getterTypePtr - } - } else if reflect.PtrTo(outt).Implements(getterIface) { - style = getterAddr - } else { - style = getterNone - } - getterStyles[outt] = style - return style -} - -func getGetter(outt reflect.Type, out reflect.Value) Getter { - style := getterStyle(outt) - if style == getterNone { - return nil - } - if style == getterAddr { - if !out.CanAddr() { - return nil - } - return out.Addr().Interface().(Getter) - } - if style == getterTypeVal && out.Kind() == reflect.Ptr && out.IsNil() { - return nil - } - return out.Interface().(Getter) -} - -// -------------------------------------------------------------------------- -// Marshaling of the document value itself. - -type encoder struct { - out []byte -} - -func (e *encoder) addDoc(v reflect.Value) { - for { - if vi, ok := v.Interface().(Getter); ok { - getv, err := vi.GetBSON() - if err != nil { - panic(err) - } - v = reflect.ValueOf(getv) - continue - } - if v.Kind() == reflect.Ptr { - v = v.Elem() - continue - } - break - } - - if v.Type() == typeRaw { - raw := v.Interface().(Raw) - if raw.Kind != 0x03 && raw.Kind != 0x00 { - panic("Attempted to marshal Raw kind " + strconv.Itoa(int(raw.Kind)) + " as a document") - } - if len(raw.Data) == 0 { - panic("Attempted to marshal empty Raw document") - } - e.addBytes(raw.Data...) - return - } - - start := e.reserveInt32() - - switch v.Kind() { - case reflect.Map: - e.addMap(v) - case reflect.Struct: - e.addStruct(v) - case reflect.Array, reflect.Slice: - e.addSlice(v) - default: - panic("Can't marshal " + v.Type().String() + " as a BSON document") - } - - e.addBytes(0) - e.setInt32(start, int32(len(e.out)-start)) -} - -func (e *encoder) addMap(v reflect.Value) { - for _, k := range v.MapKeys() { - e.addElem(fmt.Sprint(k), v.MapIndex(k), false) - } -} - -func (e *encoder) addStruct(v reflect.Value) { - sinfo, err := getStructInfo(v.Type()) - if err != nil { - panic(err) - } - var value reflect.Value - if sinfo.InlineMap >= 0 { - m := v.Field(sinfo.InlineMap) - if m.Len() > 0 { - for _, k := range m.MapKeys() { - ks := k.String() - if _, found := sinfo.FieldsMap[ks]; found { - panic(fmt.Sprintf("Can't have key %q in inlined map; conflicts with struct field", ks)) - } - e.addElem(ks, m.MapIndex(k), false) - } - } - } - for _, info := range sinfo.FieldsList { - if info.Inline == nil { - value = v.Field(info.Num) - } else { - // as pointers to struct are allowed here, - // there is no guarantee that pointer won't be nil. - // - // It is expected allowed behaviour - // so info.Inline MAY consist index to a nil pointer - // and that is why we safely call v.FieldByIndex and just continue on panic - field, errField := safeFieldByIndex(v, info.Inline) - if errField != nil { - continue - } - - value = field - } - if info.OmitEmpty && isZero(value) { - continue - } - if useRespectNilValues && - (value.Kind() == reflect.Slice || value.Kind() == reflect.Map) && - value.IsNil() { - e.addElem(info.Key, reflect.ValueOf(nil), info.MinSize) - continue - } - e.addElem(info.Key, value, info.MinSize) - } -} - -func safeFieldByIndex(v reflect.Value, index []int) (result reflect.Value, err error) { - defer func() { - if recovered := recover(); recovered != nil { - switch r := recovered.(type) { - case string: - err = fmt.Errorf("%s", r) - case error: - err = r - } - } - }() - - result = v.FieldByIndex(index) - return -} - -func isZero(v reflect.Value) bool { - switch v.Kind() { - case reflect.String: - return len(v.String()) == 0 - case reflect.Ptr, reflect.Interface: - return v.IsNil() - case reflect.Slice: - return v.Len() == 0 - case reflect.Map: - return v.Len() == 0 - case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: - return v.Int() == 0 - case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: - return v.Uint() == 0 - case reflect.Float32, reflect.Float64: - return v.Float() == 0 - case reflect.Bool: - return !v.Bool() - case reflect.Struct: - vt := v.Type() - if vt == typeTime { - return v.Interface().(time.Time).IsZero() - } - for i := 0; i < v.NumField(); i++ { - if vt.Field(i).PkgPath != "" && !vt.Field(i).Anonymous { - continue // Private field - } - if !isZero(v.Field(i)) { - return false - } - } - return true - } - return false -} - -func (e *encoder) addSlice(v reflect.Value) { - vi := v.Interface() - if d, ok := vi.(D); ok { - for _, elem := range d { - e.addElem(elem.Name, reflect.ValueOf(elem.Value), false) - } - return - } - if d, ok := vi.(RawD); ok { - for _, elem := range d { - e.addElem(elem.Name, reflect.ValueOf(elem.Value), false) - } - return - } - l := v.Len() - et := v.Type().Elem() - if et == typeDocElem { - for i := 0; i < l; i++ { - elem := v.Index(i).Interface().(DocElem) - e.addElem(elem.Name, reflect.ValueOf(elem.Value), false) - } - return - } - if et == typeRawDocElem { - for i := 0; i < l; i++ { - elem := v.Index(i).Interface().(RawDocElem) - e.addElem(elem.Name, reflect.ValueOf(elem.Value), false) - } - return - } - for i := 0; i < l; i++ { - e.addElem(itoa(i), v.Index(i), false) - } -} - -// -------------------------------------------------------------------------- -// Marshaling of elements in a document. - -func (e *encoder) addElemName(kind byte, name string) { - e.addBytes(kind) - e.addBytes([]byte(name)...) - e.addBytes(0) -} - -func (e *encoder) addElem(name string, v reflect.Value, minSize bool) { - - if !v.IsValid() { - e.addElemName(0x0A, name) - return - } - - if getter := getGetter(v.Type(), v); getter != nil { - getv, err := getter.GetBSON() - if err != nil { - panic(err) - } - e.addElem(name, reflect.ValueOf(getv), minSize) - return - } - - switch v.Kind() { - - case reflect.Interface: - e.addElem(name, v.Elem(), minSize) - - case reflect.Ptr: - e.addElem(name, v.Elem(), minSize) - - case reflect.String: - s := v.String() - switch v.Type() { - case typeObjectId: - if len(s) != 12 { - panic("ObjectIDs must be exactly 12 bytes long (got " + - strconv.Itoa(len(s)) + ")") - } - e.addElemName(0x07, name) - e.addBytes([]byte(s)...) - case typeSymbol: - e.addElemName(0x0E, name) - e.addStr(s) - case typeJSONNumber: - n := v.Interface().(json.Number) - if i, err := n.Int64(); err == nil { - e.addElemName(0x12, name) - e.addInt64(i) - } else if f, err := n.Float64(); err == nil { - e.addElemName(0x01, name) - e.addFloat64(f) - } else { - panic("failed to convert json.Number to a number: " + s) - } - default: - e.addElemName(0x02, name) - e.addStr(s) - } - - case reflect.Float32, reflect.Float64: - e.addElemName(0x01, name) - e.addFloat64(v.Float()) - - case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: - u := v.Uint() - if int64(u) < 0 { - panic("BSON has no uint64 type, and value is too large to fit correctly in an int64") - } else if u <= math.MaxInt32 && (minSize || v.Kind() <= reflect.Uint32) { - e.addElemName(0x10, name) - e.addInt32(int32(u)) - } else { - e.addElemName(0x12, name) - e.addInt64(int64(u)) - } - - case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: - switch v.Type() { - case typeMongoTimestamp: - e.addElemName(0x11, name) - e.addInt64(v.Int()) - - case typeOrderKey: - if v.Int() == int64(MaxKey) { - e.addElemName(0x7F, name) - } else { - e.addElemName(0xFF, name) - } - case typeTimeDuration: - // Stored as int64 - e.addElemName(0x12, name) - - e.addInt64(int64(v.Int() / 1e6)) - default: - i := v.Int() - if (minSize || v.Type().Kind() != reflect.Int64) && i >= math.MinInt32 && i <= math.MaxInt32 { - // It fits into an int32, encode as such. - e.addElemName(0x10, name) - e.addInt32(int32(i)) - } else { - e.addElemName(0x12, name) - e.addInt64(i) - } - } - - case reflect.Bool: - e.addElemName(0x08, name) - if v.Bool() { - e.addBytes(1) - } else { - e.addBytes(0) - } - - case reflect.Map: - e.addElemName(0x03, name) - e.addDoc(v) - - case reflect.Slice: - vt := v.Type() - et := vt.Elem() - if et.Kind() == reflect.Uint8 { - if arrayOps[name] { - e.addElemName(0x04, name) - e.addDoc(v) - } else { - e.addElemName(0x05, name) - e.addBinary(0x00, v.Bytes()) - } - } else if et == typeDocElem || et == typeRawDocElem { - e.addElemName(0x03, name) - e.addDoc(v) - } else { - e.addElemName(0x04, name) - e.addDoc(v) - } - - case reflect.Array: - et := v.Type().Elem() - if et.Kind() == reflect.Uint8 { - if arrayOps[name] { - e.addElemName(0x04, name) - e.addDoc(v) - } else { - e.addElemName(0x05, name) - if v.CanAddr() { - e.addBinary(0x00, v.Slice(0, v.Len()).Interface().([]byte)) - } else { - n := v.Len() - e.addInt32(int32(n)) - e.addBytes(0x00) - for i := 0; i < n; i++ { - el := v.Index(i) - e.addBytes(byte(el.Uint())) - } - } - } - } else { - e.addElemName(0x04, name) - e.addDoc(v) - } - - case reflect.Struct: - switch s := v.Interface().(type) { - - case Raw: - kind := s.Kind - if kind == 0x00 { - kind = 0x03 - } - if len(s.Data) == 0 && kind != 0x06 && kind != 0x0A && kind != 0xFF && kind != 0x7F { - panic("Attempted to marshal empty Raw document") - } - e.addElemName(kind, name) - e.addBytes(s.Data...) - - case Binary: - e.addElemName(0x05, name) - e.addBinary(s.Kind, s.Data) - - case Decimal128: - e.addElemName(0x13, name) - e.addInt64(int64(s.l)) - e.addInt64(int64(s.h)) - - case DBPointer: - e.addElemName(0x0C, name) - e.addStr(s.Namespace) - if len(s.Id) != 12 { - panic("ObjectIDs must be exactly 12 bytes long (got " + - strconv.Itoa(len(s.Id)) + ")") - } - e.addBytes([]byte(s.Id)...) - - case RegEx: - e.addElemName(0x0B, name) - e.addCStr(s.Pattern) - options := runes(s.Options) - sort.Sort(options) - e.addCStr(string(options)) - - case JavaScript: - if s.Scope == nil { - e.addElemName(0x0D, name) - e.addStr(s.Code) - } else { - e.addElemName(0x0F, name) - start := e.reserveInt32() - e.addStr(s.Code) - e.addDoc(reflect.ValueOf(s.Scope)) - e.setInt32(start, int32(len(e.out)-start)) - } - - case time.Time: - // MongoDB handles timestamps as milliseconds. - e.addElemName(0x09, name) - e.addInt64(s.Unix()*1000 + int64(s.Nanosecond()/1e6)) - - case url.URL: - e.addElemName(0x02, name) - e.addStr(s.String()) - - case undefined: - e.addElemName(0x06, name) - - default: - e.addElemName(0x03, name) - e.addDoc(v) - } - - default: - panic("Can't marshal " + v.Type().String() + " in a BSON document") - } -} - -// ------------- -// Helper method for sorting regex options -type runes []rune - -func (a runes) Len() int { return len(a) } -func (a runes) Swap(i, j int) { a[i], a[j] = a[j], a[i] } -func (a runes) Less(i, j int) bool { return a[i] < a[j] } - -// -------------------------------------------------------------------------- -// Marshaling of base types. - -func (e *encoder) addBinary(subtype byte, v []byte) { - if subtype == 0x02 { - // Wonder how that brilliant idea came to life. Obsolete, luckily. - e.addInt32(int32(len(v) + 4)) - e.addBytes(subtype) - e.addInt32(int32(len(v))) - } else { - e.addInt32(int32(len(v))) - e.addBytes(subtype) - } - e.addBytes(v...) -} - -func (e *encoder) addStr(v string) { - e.addInt32(int32(len(v) + 1)) - e.addCStr(v) -} - -func (e *encoder) addCStr(v string) { - e.addBytes([]byte(v)...) - e.addBytes(0) -} - -func (e *encoder) reserveInt32() (pos int) { - pos = len(e.out) - e.addBytes(0, 0, 0, 0) - return pos -} - -func (e *encoder) setInt32(pos int, v int32) { - e.out[pos+0] = byte(v) - e.out[pos+1] = byte(v >> 8) - e.out[pos+2] = byte(v >> 16) - e.out[pos+3] = byte(v >> 24) -} - -func (e *encoder) addInt32(v int32) { - u := uint32(v) - e.addBytes(byte(u), byte(u>>8), byte(u>>16), byte(u>>24)) -} - -func (e *encoder) addInt64(v int64) { - u := uint64(v) - e.addBytes(byte(u), byte(u>>8), byte(u>>16), byte(u>>24), - byte(u>>32), byte(u>>40), byte(u>>48), byte(u>>56)) -} - -func (e *encoder) addFloat64(v float64) { - e.addInt64(int64(math.Float64bits(v))) -} - -func (e *encoder) addBytes(v ...byte) { - e.out = append(e.out, v...) -} diff --git a/vendor/github.com/globalsign/mgo/bson/json.go b/vendor/github.com/globalsign/mgo/bson/json.go deleted file mode 100644 index 045c7130..00000000 --- a/vendor/github.com/globalsign/mgo/bson/json.go +++ /dev/null @@ -1,384 +0,0 @@ -package bson - -import ( - "bytes" - "encoding/base64" - "fmt" - "strconv" - "strings" - "time" - - "github.com/globalsign/mgo/internal/json" -) - -// UnmarshalJSON unmarshals a JSON value that may hold non-standard -// syntax as defined in BSON's extended JSON specification. -func UnmarshalJSON(data []byte, value interface{}) error { - d := json.NewDecoder(bytes.NewBuffer(data)) - d.Extend(&jsonExt) - return d.Decode(value) -} - -// MarshalJSON marshals a JSON value that may hold non-standard -// syntax as defined in BSON's extended JSON specification. -func MarshalJSON(value interface{}) ([]byte, error) { - var buf bytes.Buffer - e := json.NewEncoder(&buf) - e.Extend(&jsonExt) - err := e.Encode(value) - if err != nil { - return nil, err - } - return buf.Bytes(), nil -} - -// jdec is used internally by the JSON decoding functions -// so they may unmarshal functions without getting into endless -// recursion due to keyed objects. -func jdec(data []byte, value interface{}) error { - d := json.NewDecoder(bytes.NewBuffer(data)) - d.Extend(&funcExt) - return d.Decode(value) -} - -var jsonExt json.Extension -var funcExt json.Extension - -// TODO -// - Shell regular expressions ("/regexp/opts") - -func init() { - jsonExt.DecodeUnquotedKeys(true) - jsonExt.DecodeTrailingCommas(true) - - funcExt.DecodeFunc("BinData", "$binaryFunc", "$type", "$binary") - jsonExt.DecodeKeyed("$binary", jdecBinary) - jsonExt.DecodeKeyed("$binaryFunc", jdecBinary) - jsonExt.EncodeType([]byte(nil), jencBinarySlice) - jsonExt.EncodeType(Binary{}, jencBinaryType) - - funcExt.DecodeFunc("ISODate", "$dateFunc", "S") - funcExt.DecodeFunc("new Date", "$dateFunc", "S") - jsonExt.DecodeKeyed("$date", jdecDate) - jsonExt.DecodeKeyed("$dateFunc", jdecDate) - jsonExt.EncodeType(time.Time{}, jencDate) - - funcExt.DecodeFunc("Timestamp", "$timestamp", "t", "i") - jsonExt.DecodeKeyed("$timestamp", jdecTimestamp) - jsonExt.EncodeType(MongoTimestamp(0), jencTimestamp) - - funcExt.DecodeConst("undefined", Undefined) - - jsonExt.DecodeKeyed("$regex", jdecRegEx) - jsonExt.EncodeType(RegEx{}, jencRegEx) - - funcExt.DecodeFunc("ObjectId", "$oidFunc", "Id") - jsonExt.DecodeKeyed("$oid", jdecObjectId) - jsonExt.DecodeKeyed("$oidFunc", jdecObjectId) - jsonExt.EncodeType(ObjectId(""), jencObjectId) - - funcExt.DecodeFunc("DBRef", "$dbrefFunc", "$ref", "$id") - jsonExt.DecodeKeyed("$dbrefFunc", jdecDBRef) - - funcExt.DecodeFunc("NumberLong", "$numberLongFunc", "N") - jsonExt.DecodeKeyed("$numberLong", jdecNumberLong) - jsonExt.DecodeKeyed("$numberLongFunc", jdecNumberLong) - jsonExt.EncodeType(int64(0), jencNumberLong) - jsonExt.EncodeType(int(0), jencInt) - - funcExt.DecodeConst("MinKey", MinKey) - funcExt.DecodeConst("MaxKey", MaxKey) - jsonExt.DecodeKeyed("$minKey", jdecMinKey) - jsonExt.DecodeKeyed("$maxKey", jdecMaxKey) - jsonExt.EncodeType(orderKey(0), jencMinMaxKey) - - jsonExt.DecodeKeyed("$undefined", jdecUndefined) - jsonExt.EncodeType(Undefined, jencUndefined) - - jsonExt.Extend(&funcExt) -} - -func fbytes(format string, args ...interface{}) []byte { - var buf bytes.Buffer - fmt.Fprintf(&buf, format, args...) - return buf.Bytes() -} - -func jdecBinary(data []byte) (interface{}, error) { - var v struct { - Binary []byte `json:"$binary"` - Type string `json:"$type"` - Func struct { - Binary []byte `json:"$binary"` - Type int64 `json:"$type"` - } `json:"$binaryFunc"` - } - err := jdec(data, &v) - if err != nil { - return nil, err - } - - var binData []byte - var binKind int64 - if v.Type == "" && v.Binary == nil { - binData = v.Func.Binary - binKind = v.Func.Type - } else if v.Type == "" { - return v.Binary, nil - } else { - binData = v.Binary - binKind, err = strconv.ParseInt(v.Type, 0, 64) - if err != nil { - binKind = -1 - } - } - - if binKind == 0 { - return binData, nil - } - if binKind < 0 || binKind > 255 { - return nil, fmt.Errorf("invalid type in binary object: %s", data) - } - - return Binary{Kind: byte(binKind), Data: binData}, nil -} - -func jencBinarySlice(v interface{}) ([]byte, error) { - in := v.([]byte) - out := make([]byte, base64.StdEncoding.EncodedLen(len(in))) - base64.StdEncoding.Encode(out, in) - return fbytes(`{"$binary":"%s","$type":"0x0"}`, out), nil -} - -func jencBinaryType(v interface{}) ([]byte, error) { - in := v.(Binary) - out := make([]byte, base64.StdEncoding.EncodedLen(len(in.Data))) - base64.StdEncoding.Encode(out, in.Data) - return fbytes(`{"$binary":"%s","$type":"0x%x"}`, out, in.Kind), nil -} - -const jdateFormat = "2006-01-02T15:04:05.999Z07:00" - -func jdecDate(data []byte) (interface{}, error) { - var v struct { - S string `json:"$date"` - Func struct { - S string - } `json:"$dateFunc"` - } - _ = jdec(data, &v) - if v.S == "" { - v.S = v.Func.S - } - if v.S != "" { - var errs []string - for _, format := range []string{jdateFormat, "2006-01-02"} { - t, err := time.Parse(format, v.S) - if err == nil { - return t, nil - } - errs = append(errs, err.Error()) - } - return nil, fmt.Errorf("cannot parse date: %q [%s]", v.S, strings.Join(errs, ", ")) - } - - var vn struct { - Date struct { - N int64 `json:"$numberLong,string"` - } `json:"$date"` - Func struct { - S int64 - } `json:"$dateFunc"` - } - err := jdec(data, &vn) - if err != nil { - return nil, fmt.Errorf("cannot parse date: %q", data) - } - n := vn.Date.N - if n == 0 { - n = vn.Func.S - } - return time.Unix(n/1000, n%1000*1e6).UTC(), nil -} - -func jencDate(v interface{}) ([]byte, error) { - t := v.(time.Time) - return fbytes(`{"$date":%q}`, t.Format(jdateFormat)), nil -} - -func jdecTimestamp(data []byte) (interface{}, error) { - var v struct { - Func struct { - T int32 `json:"t"` - I int32 `json:"i"` - } `json:"$timestamp"` - } - err := jdec(data, &v) - if err != nil { - return nil, err - } - return MongoTimestamp(uint64(v.Func.T)<<32 | uint64(uint32(v.Func.I))), nil -} - -func jencTimestamp(v interface{}) ([]byte, error) { - ts := uint64(v.(MongoTimestamp)) - return fbytes(`{"$timestamp":{"t":%d,"i":%d}}`, ts>>32, uint32(ts)), nil -} - -func jdecRegEx(data []byte) (interface{}, error) { - var v struct { - Regex string `json:"$regex"` - Options string `json:"$options"` - } - err := jdec(data, &v) - if err != nil { - return nil, err - } - return RegEx{v.Regex, v.Options}, nil -} - -func jencRegEx(v interface{}) ([]byte, error) { - re := v.(RegEx) - type regex struct { - Regex string `json:"$regex"` - Options string `json:"$options"` - } - return json.Marshal(regex{re.Pattern, re.Options}) -} - -func jdecObjectId(data []byte) (interface{}, error) { - var v struct { - Id string `json:"$oid"` - Func struct { - Id string - } `json:"$oidFunc"` - } - err := jdec(data, &v) - if err != nil { - return nil, err - } - if v.Id == "" { - v.Id = v.Func.Id - } - return ObjectIdHex(v.Id), nil -} - -func jencObjectId(v interface{}) ([]byte, error) { - return fbytes(`{"$oid":"%s"}`, v.(ObjectId).Hex()), nil -} - -func jdecDBRef(data []byte) (interface{}, error) { - // TODO Support unmarshaling $ref and $id into the input value. - var v struct { - Obj map[string]interface{} `json:"$dbrefFunc"` - } - // TODO Fix this. Must not be required. - v.Obj = make(map[string]interface{}) - err := jdec(data, &v) - if err != nil { - return nil, err - } - return v.Obj, nil -} - -func jdecNumberLong(data []byte) (interface{}, error) { - var v struct { - N int64 `json:"$numberLong,string"` - Func struct { - N int64 `json:",string"` - } `json:"$numberLongFunc"` - } - var vn struct { - N int64 `json:"$numberLong"` - Func struct { - N int64 - } `json:"$numberLongFunc"` - } - err := jdec(data, &v) - if err != nil { - err = jdec(data, &vn) - v.N = vn.N - v.Func.N = vn.Func.N - } - if err != nil { - return nil, err - } - if v.N != 0 { - return v.N, nil - } - return v.Func.N, nil -} - -func jencNumberLong(v interface{}) ([]byte, error) { - n := v.(int64) - f := `{"$numberLong":"%d"}` - if n <= 1<<53 { - f = `{"$numberLong":%d}` - } - return fbytes(f, n), nil -} - -func jencInt(v interface{}) ([]byte, error) { - n := v.(int) - f := `{"$numberLong":"%d"}` - if int64(n) <= 1<<53 { - f = `%d` - } - return fbytes(f, n), nil -} - -func jdecMinKey(data []byte) (interface{}, error) { - var v struct { - N int64 `json:"$minKey"` - } - err := jdec(data, &v) - if err != nil { - return nil, err - } - if v.N != 1 { - return nil, fmt.Errorf("invalid $minKey object: %s", data) - } - return MinKey, nil -} - -func jdecMaxKey(data []byte) (interface{}, error) { - var v struct { - N int64 `json:"$maxKey"` - } - err := jdec(data, &v) - if err != nil { - return nil, err - } - if v.N != 1 { - return nil, fmt.Errorf("invalid $maxKey object: %s", data) - } - return MaxKey, nil -} - -func jencMinMaxKey(v interface{}) ([]byte, error) { - switch v.(orderKey) { - case MinKey: - return []byte(`{"$minKey":1}`), nil - case MaxKey: - return []byte(`{"$maxKey":1}`), nil - } - panic(fmt.Sprintf("invalid $minKey/$maxKey value: %d", v)) -} - -func jdecUndefined(data []byte) (interface{}, error) { - var v struct { - B bool `json:"$undefined"` - } - err := jdec(data, &v) - if err != nil { - return nil, err - } - if !v.B { - return nil, fmt.Errorf("invalid $undefined object: %s", data) - } - return Undefined, nil -} - -func jencUndefined(v interface{}) ([]byte, error) { - return []byte(`{"$undefined":true}`), nil -} diff --git a/vendor/github.com/globalsign/mgo/bson/stream.go b/vendor/github.com/globalsign/mgo/bson/stream.go deleted file mode 100644 index 46652845..00000000 --- a/vendor/github.com/globalsign/mgo/bson/stream.go +++ /dev/null @@ -1,90 +0,0 @@ -package bson - -import ( - "bytes" - "encoding/binary" - "fmt" - "io" -) - -const ( - // MinDocumentSize is the size of the smallest possible valid BSON document: - // an int32 size header + 0x00 (end of document). - MinDocumentSize = 5 - - // MaxDocumentSize is the largest possible size for a BSON document allowed by MongoDB, - // that is, 16 MiB (see https://docs.mongodb.com/manual/reference/limits/). - MaxDocumentSize = 16777216 -) - -// ErrInvalidDocumentSize is an error returned when a BSON document's header -// contains a size smaller than MinDocumentSize or greater than MaxDocumentSize. -type ErrInvalidDocumentSize struct { - DocumentSize int32 -} - -func (e ErrInvalidDocumentSize) Error() string { - return fmt.Sprintf("invalid document size %d", e.DocumentSize) -} - -// A Decoder reads and decodes BSON values from an input stream. -type Decoder struct { - source io.Reader -} - -// NewDecoder returns a new Decoder that reads from source. -// It does not add any extra buffering, and may not read data from source beyond the BSON values requested. -func NewDecoder(source io.Reader) *Decoder { - return &Decoder{source: source} -} - -// Decode reads the next BSON-encoded value from its input and stores it in the value pointed to by v. -// See the documentation for Unmarshal for details about the conversion of BSON into a Go value. -func (dec *Decoder) Decode(v interface{}) (err error) { - // BSON documents start with their size as a *signed* int32. - var docSize int32 - if err = binary.Read(dec.source, binary.LittleEndian, &docSize); err != nil { - return - } - - if docSize < MinDocumentSize || docSize > MaxDocumentSize { - return ErrInvalidDocumentSize{DocumentSize: docSize} - } - - docBuffer := bytes.NewBuffer(make([]byte, 0, docSize)) - if err = binary.Write(docBuffer, binary.LittleEndian, docSize); err != nil { - return - } - - // docSize is the *full* document's size (including the 4-byte size header, - // which has already been read). - if _, err = io.CopyN(docBuffer, dec.source, int64(docSize-4)); err != nil { - return - } - - // Let Unmarshal handle the rest. - defer handleErr(&err) - return Unmarshal(docBuffer.Bytes(), v) -} - -// An Encoder encodes and writes BSON values to an output stream. -type Encoder struct { - target io.Writer -} - -// NewEncoder returns a new Encoder that writes to target. -func NewEncoder(target io.Writer) *Encoder { - return &Encoder{target: target} -} - -// Encode encodes v to BSON, and if successful writes it to the Encoder's output stream. -// See the documentation for Marshal for details about the conversion of Go values to BSON. -func (enc *Encoder) Encode(v interface{}) error { - data, err := Marshal(v) - if err != nil { - return err - } - - _, err = enc.target.Write(data) - return err -} diff --git a/vendor/github.com/globalsign/mgo/bulk.go b/vendor/github.com/globalsign/mgo/bulk.go deleted file mode 100644 index c234fcce..00000000 --- a/vendor/github.com/globalsign/mgo/bulk.go +++ /dev/null @@ -1,366 +0,0 @@ -package mgo - -import ( - "bytes" - "sort" - "sync" - - "github.com/globalsign/mgo/bson" -) - -// Bulk represents an operation that can be prepared with several -// orthogonal changes before being delivered to the server. -// -// MongoDB servers older than version 2.6 do not have proper support for bulk -// operations, so the driver attempts to map its API as much as possible into -// the functionality that works. In particular, in those releases updates and -// removals are sent individually, and inserts are sent in bulk but have -// suboptimal error reporting compared to more recent versions of the server. -// See the documentation of BulkErrorCase for details on that. -// -// Relevant documentation: -// -// http://blog.mongodb.org/post/84922794768/mongodbs-new-bulk-api -// -type Bulk struct { - c *Collection - opcount int - actions []bulkAction - ordered bool -} - -type bulkOp int - -const ( - bulkInsert bulkOp = iota + 1 - bulkUpdate - bulkUpdateAll - bulkRemove -) - -type bulkAction struct { - op bulkOp - docs []interface{} - idxs []int -} - -type bulkUpdateOp []interface{} -type bulkDeleteOp []interface{} - -// BulkResult holds the results for a bulk operation. -type BulkResult struct { - Matched int - Modified int // Available only for MongoDB 2.6+ - - // Be conservative while we understand exactly how to report these - // results in a useful and convenient way, and also how to emulate - // them with prior servers. - private bool -} - -// BulkError holds an error returned from running a Bulk operation. -// Individual errors may be obtained and inspected via the Cases method. -type BulkError struct { - ecases []BulkErrorCase -} - -func (e *BulkError) Error() string { - if len(e.ecases) == 0 { - return "invalid BulkError instance: no errors" - } - if len(e.ecases) == 1 { - return e.ecases[0].Err.Error() - } - msgs := make([]string, 0, len(e.ecases)) - seen := make(map[string]bool) - for _, ecase := range e.ecases { - msg := ecase.Err.Error() - if !seen[msg] { - seen[msg] = true - msgs = append(msgs, msg) - } - } - if len(msgs) == 1 { - return msgs[0] - } - var buf bytes.Buffer - buf.WriteString("multiple errors in bulk operation:\n") - for _, msg := range msgs { - buf.WriteString(" - ") - buf.WriteString(msg) - buf.WriteByte('\n') - } - return buf.String() -} - -type bulkErrorCases []BulkErrorCase - -func (slice bulkErrorCases) Len() int { return len(slice) } -func (slice bulkErrorCases) Less(i, j int) bool { return slice[i].Index < slice[j].Index } -func (slice bulkErrorCases) Swap(i, j int) { slice[i], slice[j] = slice[j], slice[i] } - -// BulkErrorCase holds an individual error found while attempting a single change -// within a bulk operation, and the position in which it was enqueued. -// -// MongoDB servers older than version 2.6 do not have proper support for bulk -// operations, so the driver attempts to map its API as much as possible into -// the functionality that works. In particular, only the last error is reported -// for bulk inserts and without any positional information, so the Index -// field is set to -1 in these cases. -type BulkErrorCase struct { - Index int // Position of operation that failed, or -1 if unknown. - Err error -} - -// Cases returns all individual errors found while attempting the requested changes. -// -// See the documentation of BulkErrorCase for limitations in older MongoDB releases. -func (e *BulkError) Cases() []BulkErrorCase { - return e.ecases -} - -var actionPool = sync.Pool{ - New: func() interface{} { - return &bulkAction{ - docs: make([]interface{}, 0), - idxs: make([]int, 0), - } - }, -} - -// Bulk returns a value to prepare the execution of a bulk operation. -func (c *Collection) Bulk() *Bulk { - return &Bulk{c: c, ordered: true} -} - -// Unordered puts the bulk operation in unordered mode. -// -// In unordered mode the indvidual operations may be sent -// out of order, which means latter operations may proceed -// even if prior ones have failed. -func (b *Bulk) Unordered() { - b.ordered = false -} - -func (b *Bulk) action(op bulkOp, opcount int) *bulkAction { - var action *bulkAction - if len(b.actions) > 0 && b.actions[len(b.actions)-1].op == op { - action = &b.actions[len(b.actions)-1] - } else if !b.ordered { - for i := range b.actions { - if b.actions[i].op == op { - action = &b.actions[i] - break - } - } - } - if action == nil { - a := actionPool.Get().(*bulkAction) - a.op = op - b.actions = append(b.actions, *a) - action = &b.actions[len(b.actions)-1] - } - for i := 0; i < opcount; i++ { - action.idxs = append(action.idxs, b.opcount) - b.opcount++ - } - return action -} - -// Insert queues up the provided documents for insertion. -func (b *Bulk) Insert(docs ...interface{}) { - action := b.action(bulkInsert, len(docs)) - action.docs = append(action.docs, docs...) -} - -// Remove queues up the provided selectors for removing matching documents. -// Each selector will remove only a single matching document. -func (b *Bulk) Remove(selectors ...interface{}) { - action := b.action(bulkRemove, len(selectors)) - for _, selector := range selectors { - if selector == nil { - selector = bson.D{} - } - action.docs = append(action.docs, &deleteOp{ - Collection: b.c.FullName, - Selector: selector, - Flags: 1, - Limit: 1, - }) - } -} - -// RemoveAll queues up the provided selectors for removing all matching documents. -// Each selector will remove all matching documents. -func (b *Bulk) RemoveAll(selectors ...interface{}) { - action := b.action(bulkRemove, len(selectors)) - for _, selector := range selectors { - if selector == nil { - selector = bson.D{} - } - action.docs = append(action.docs, &deleteOp{ - Collection: b.c.FullName, - Selector: selector, - Flags: 0, - Limit: 0, - }) - } -} - -// Update queues up the provided pairs of updating instructions. -// The first element of each pair selects which documents must be -// updated, and the second element defines how to update it. -// Each pair matches exactly one document for updating at most. -func (b *Bulk) Update(pairs ...interface{}) { - if len(pairs)%2 != 0 { - panic("Bulk.Update requires an even number of parameters") - } - action := b.action(bulkUpdate, len(pairs)/2) - for i := 0; i < len(pairs); i += 2 { - selector := pairs[i] - if selector == nil { - selector = bson.D{} - } - action.docs = append(action.docs, &updateOp{ - Collection: b.c.FullName, - Selector: selector, - Update: pairs[i+1], - }) - } -} - -// UpdateAll queues up the provided pairs of updating instructions. -// The first element of each pair selects which documents must be -// updated, and the second element defines how to update it. -// Each pair updates all documents matching the selector. -func (b *Bulk) UpdateAll(pairs ...interface{}) { - if len(pairs)%2 != 0 { - panic("Bulk.UpdateAll requires an even number of parameters") - } - action := b.action(bulkUpdate, len(pairs)/2) - for i := 0; i < len(pairs); i += 2 { - selector := pairs[i] - if selector == nil { - selector = bson.D{} - } - action.docs = append(action.docs, &updateOp{ - Collection: b.c.FullName, - Selector: selector, - Update: pairs[i+1], - Flags: 2, - Multi: true, - }) - } -} - -// Upsert queues up the provided pairs of upserting instructions. -// The first element of each pair selects which documents must be -// updated, and the second element defines how to update it. -// Each pair matches exactly one document for updating at most. -func (b *Bulk) Upsert(pairs ...interface{}) { - if len(pairs)%2 != 0 { - panic("Bulk.Update requires an even number of parameters") - } - action := b.action(bulkUpdate, len(pairs)/2) - for i := 0; i < len(pairs); i += 2 { - selector := pairs[i] - if selector == nil { - selector = bson.D{} - } - action.docs = append(action.docs, &updateOp{ - Collection: b.c.FullName, - Selector: selector, - Update: pairs[i+1], - Flags: 1, - Upsert: true, - }) - } -} - -// Run runs all the operations queued up. -// -// If an error is reported on an unordered bulk operation, the error value may -// be an aggregation of all issues observed. As an exception to that, Insert -// operations running on MongoDB versions prior to 2.6 will report the last -// error only due to a limitation in the wire protocol. -func (b *Bulk) Run() (*BulkResult, error) { - var result BulkResult - var berr BulkError - var failed bool - for i := range b.actions { - action := &b.actions[i] - var ok bool - switch action.op { - case bulkInsert: - ok = b.runInsert(action, &result, &berr) - case bulkUpdate: - ok = b.runUpdate(action, &result, &berr) - case bulkRemove: - ok = b.runRemove(action, &result, &berr) - default: - panic("unknown bulk operation") - } - action.idxs = action.idxs[0:0] - action.docs = action.docs[0:0] - actionPool.Put(action) - if !ok { - failed = true - if b.ordered { - break - } - } - } - if failed { - sort.Sort(bulkErrorCases(berr.ecases)) - return nil, &berr - } - return &result, nil -} - -func (b *Bulk) runInsert(action *bulkAction, result *BulkResult, berr *BulkError) bool { - op := &insertOp{b.c.FullName, action.docs, 0} - if !b.ordered { - op.flags = 1 // ContinueOnError - } - lerr, err := b.c.writeOp(op, b.ordered) - return b.checkSuccess(action, berr, lerr, err) -} - -func (b *Bulk) runUpdate(action *bulkAction, result *BulkResult, berr *BulkError) bool { - lerr, err := b.c.writeOp(bulkUpdateOp(action.docs), b.ordered) - if lerr != nil { - result.Matched += lerr.N - result.Modified += lerr.modified - } - return b.checkSuccess(action, berr, lerr, err) -} - -func (b *Bulk) runRemove(action *bulkAction, result *BulkResult, berr *BulkError) bool { - lerr, err := b.c.writeOp(bulkDeleteOp(action.docs), b.ordered) - if lerr != nil { - result.Matched += lerr.N - result.Modified += lerr.modified - } - return b.checkSuccess(action, berr, lerr, err) -} - -func (b *Bulk) checkSuccess(action *bulkAction, berr *BulkError, lerr *LastError, err error) bool { - if lerr != nil && len(lerr.ecases) > 0 { - for i := 0; i < len(lerr.ecases); i++ { - // Map back from the local error index into the visible one. - ecase := lerr.ecases[i] - idx := ecase.Index - if idx >= 0 { - idx = action.idxs[idx] - } - berr.ecases = append(berr.ecases, BulkErrorCase{idx, ecase.Err}) - } - return false - } else if err != nil { - for i := 0; i < len(action.idxs); i++ { - berr.ecases = append(berr.ecases, BulkErrorCase{action.idxs[i], err}) - } - return false - } - return true -} diff --git a/vendor/github.com/globalsign/mgo/changestreams.go b/vendor/github.com/globalsign/mgo/changestreams.go deleted file mode 100644 index 5c2279c6..00000000 --- a/vendor/github.com/globalsign/mgo/changestreams.go +++ /dev/null @@ -1,357 +0,0 @@ -package mgo - -import ( - "errors" - "fmt" - "reflect" - "sync" - "time" - - "github.com/globalsign/mgo/bson" -) - -type FullDocument string - -const ( - Default = "default" - UpdateLookup = "updateLookup" -) - -type ChangeStream struct { - iter *Iter - isClosed bool - options ChangeStreamOptions - pipeline interface{} - resumeToken *bson.Raw - collection *Collection - readPreference *ReadPreference - err error - m sync.Mutex - sessionCopied bool -} - -type ChangeStreamOptions struct { - - // FullDocument controls the amount of data that the server will return when - // returning a changes document. - FullDocument FullDocument - - // ResumeAfter specifies the logical starting point for the new change stream. - ResumeAfter *bson.Raw - - // MaxAwaitTimeMS specifies the maximum amount of time for the server to wait - // on new documents to satisfy a change stream query. - MaxAwaitTimeMS time.Duration - - // BatchSize specifies the number of documents to return per batch. - BatchSize int - - // Collation specifies the way the server should collate returned data. - //TODO Collation *Collation -} - -var errMissingResumeToken = errors.New("resume token missing from result") - -// Watch constructs a new ChangeStream capable of receiving continuing data -// from the database. -func (coll *Collection) Watch(pipeline interface{}, - options ChangeStreamOptions) (*ChangeStream, error) { - - if pipeline == nil { - pipeline = []bson.M{} - } - - csPipe := constructChangeStreamPipeline(pipeline, options) - pipe := coll.Pipe(&csPipe) - if options.MaxAwaitTimeMS > 0 { - pipe.SetMaxTime(options.MaxAwaitTimeMS) - } - if options.BatchSize > 0 { - pipe.Batch(options.BatchSize) - } - pIter := pipe.Iter() - - // check that there was no issue creating the iterator. - // this will fail immediately with an error from the server if running against - // a standalone. - if err := pIter.Err(); err != nil { - return nil, err - } - - pIter.isChangeStream = true - return &ChangeStream{ - iter: pIter, - collection: coll, - resumeToken: nil, - options: options, - pipeline: pipeline, - }, nil -} - -// Next retrieves the next document from the change stream, blocking if necessary. -// Next returns true if a document was successfully unmarshalled into result, -// and false if an error occured. When Next returns false, the Err method should -// be called to check what error occurred during iteration. If there were no events -// available (ErrNotFound), the Err method returns nil so the user can retry the invocaton. -// -// For example: -// -// pipeline := []bson.M{} -// -// changeStream := collection.Watch(pipeline, ChangeStreamOptions{}) -// for changeStream.Next(&changeDoc) { -// fmt.Printf("Change: %v\n", changeDoc) -// } -// -// if err := changeStream.Close(); err != nil { -// return err -// } -// -// If the pipeline used removes the _id field from the result, Next will error -// because the _id field is needed to resume iteration when an error occurs. -// -func (changeStream *ChangeStream) Next(result interface{}) bool { - // the err field is being constantly overwritten and we don't want the user to - // attempt to read it at this point so we lock. - changeStream.m.Lock() - - defer changeStream.m.Unlock() - - // if we are in a state of error, then don't continue. - if changeStream.err != nil { - return false - } - - if changeStream.isClosed { - changeStream.err = fmt.Errorf("illegal use of a closed ChangeStream") - return false - } - - var err error - - // attempt to fetch the change stream result. - err = changeStream.fetchResultSet(result) - if err == nil { - return true - } - - // if we get no results we return false with no errors so the user can call Next - // again, resuming is not needed as the iterator is simply timed out as no events happened. - // The user will call Timeout in order to understand if this was the case. - if err == ErrNotFound { - return false - } - - // check if the error is resumable - if !isResumableError(err) { - // error is not resumable, give up and return it to the user. - changeStream.err = err - return false - } - - // try to resume. - err = changeStream.resume() - if err != nil { - // we've not been able to successfully resume and should only try once, - // so we give up. - changeStream.err = err - return false - } - - // we've successfully resumed the changestream. - // try to fetch the next result. - err = changeStream.fetchResultSet(result) - if err != nil { - changeStream.err = err - return false - } - - return true -} - -// Err returns nil if no errors happened during iteration, or the actual -// error otherwise. -func (changeStream *ChangeStream) Err() error { - changeStream.m.Lock() - defer changeStream.m.Unlock() - return changeStream.err -} - -// Close kills the server cursor used by the iterator, if any, and returns -// nil if no errors happened during iteration, or the actual error otherwise. -func (changeStream *ChangeStream) Close() error { - changeStream.m.Lock() - defer changeStream.m.Unlock() - changeStream.isClosed = true - err := changeStream.iter.Close() - if err != nil { - changeStream.err = err - } - if changeStream.sessionCopied { - changeStream.iter.session.Close() - changeStream.sessionCopied = false - } - return err -} - -// ResumeToken returns a copy of the current resume token held by the change stream. -// This token should be treated as an opaque token that can be provided to instantiate -// a new change stream. -func (changeStream *ChangeStream) ResumeToken() *bson.Raw { - changeStream.m.Lock() - defer changeStream.m.Unlock() - if changeStream.resumeToken == nil { - return nil - } - var tokenCopy = *changeStream.resumeToken - return &tokenCopy -} - -// Timeout returns true if the last call of Next returned false because of an iterator timeout. -func (changeStream *ChangeStream) Timeout() bool { - return changeStream.iter.Timeout() -} - -func constructChangeStreamPipeline(pipeline interface{}, - options ChangeStreamOptions) interface{} { - pipelinev := reflect.ValueOf(pipeline) - - // ensure that the pipeline passed in is a slice. - if pipelinev.Kind() != reflect.Slice { - panic("pipeline argument must be a slice") - } - - // construct the options to be used by the change notification - // pipeline stage. - changeStreamStageOptions := bson.M{} - - if options.FullDocument != "" { - changeStreamStageOptions["fullDocument"] = options.FullDocument - } - if options.ResumeAfter != nil { - changeStreamStageOptions["resumeAfter"] = options.ResumeAfter - } - - changeStreamStage := bson.M{"$changeStream": changeStreamStageOptions} - - pipeOfInterfaces := make([]interface{}, pipelinev.Len()+1) - - // insert the change notification pipeline stage at the beginning of the - // aggregation. - pipeOfInterfaces[0] = changeStreamStage - - // convert the passed in slice to a slice of interfaces. - for i := 0; i < pipelinev.Len(); i++ { - pipeOfInterfaces[1+i] = pipelinev.Index(i).Addr().Interface() - } - var pipelineAsInterface interface{} = pipeOfInterfaces - return pipelineAsInterface -} - -func (changeStream *ChangeStream) resume() error { - // copy the information for the new socket. - - // Thanks to Copy() future uses will acquire a new socket against the newly selected DB. - newSession := changeStream.iter.session.Copy() - - // fetch the cursor from the iterator and use it to run a killCursors - // on the connection. - cursorId := changeStream.iter.op.cursorId - err := runKillCursorsOnSession(newSession, cursorId) - if err != nil { - return err - } - - // change out the old connection to the database with the new connection. - if changeStream.sessionCopied { - changeStream.collection.Database.Session.Close() - } - changeStream.collection.Database.Session = newSession - changeStream.sessionCopied = true - - opts := changeStream.options - if changeStream.resumeToken != nil { - opts.ResumeAfter = changeStream.resumeToken - } - // make a new pipeline containing the resume token. - changeStreamPipeline := constructChangeStreamPipeline(changeStream.pipeline, opts) - - // generate the new iterator with the new connection. - newPipe := changeStream.collection.Pipe(changeStreamPipeline) - changeStream.iter = newPipe.Iter() - if err := changeStream.iter.Err(); err != nil { - return err - } - changeStream.iter.isChangeStream = true - return nil -} - -// fetchResumeToken unmarshals the _id field from the document, setting an error -// on the changeStream if it is unable to. -func (changeStream *ChangeStream) fetchResumeToken(rawResult *bson.Raw) error { - changeStreamResult := struct { - ResumeToken *bson.Raw `bson:"_id,omitempty"` - }{} - - err := rawResult.Unmarshal(&changeStreamResult) - if err != nil { - return err - } - - if changeStreamResult.ResumeToken == nil { - return errMissingResumeToken - } - - changeStream.resumeToken = changeStreamResult.ResumeToken - return nil -} - -func (changeStream *ChangeStream) fetchResultSet(result interface{}) error { - rawResult := bson.Raw{} - - // fetch the next set of documents from the cursor. - gotNext := changeStream.iter.Next(&rawResult) - err := changeStream.iter.Err() - if err != nil { - return err - } - - if !gotNext && err == nil { - // If the iter.Err() method returns nil despite us not getting a next batch, - // it is becuase iter.Err() silences this case. - return ErrNotFound - } - - // grab the resumeToken from the results - if err := changeStream.fetchResumeToken(&rawResult); err != nil { - return err - } - - // put the raw results into the data structure the user provided. - if err := rawResult.Unmarshal(result); err != nil { - return err - } - return nil -} - -func isResumableError(err error) bool { - _, isQueryError := err.(*QueryError) - // if it is not a database error OR it is a database error, - // but the error is a notMaster error - //and is not a missingResumeToken error (caused by the user provided pipeline) - return (!isQueryError || isNotMasterError(err)) && (err != errMissingResumeToken) -} - -func runKillCursorsOnSession(session *Session, cursorId int64) error { - socket, err := session.acquireSocket(true) - if err != nil { - return err - } - err = socket.Query(&killCursorsOp{[]int64{cursorId}}) - if err != nil { - return err - } - socket.Release() - - return nil -} diff --git a/vendor/github.com/globalsign/mgo/cluster.go b/vendor/github.com/globalsign/mgo/cluster.go deleted file mode 100644 index ff431cac..00000000 --- a/vendor/github.com/globalsign/mgo/cluster.go +++ /dev/null @@ -1,704 +0,0 @@ -// mgo - MongoDB driver for Go -// -// Copyright (c) 2010-2012 - Gustavo Niemeyer -// -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are met: -// -// 1. Redistributions of source code must retain the above copyright notice, this -// list of conditions and the following disclaimer. -// 2. Redistributions in binary form must reproduce the above copyright notice, -// this list of conditions and the following disclaimer in the documentation -// and/or other materials provided with the distribution. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR -// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -package mgo - -import ( - "errors" - "fmt" - "net" - "runtime" - "strconv" - "strings" - "sync" - "time" - - "github.com/globalsign/mgo/bson" -) - -// --------------------------------------------------------------------------- -// Mongo cluster encapsulation. -// -// A cluster enables the communication with one or more servers participating -// in a mongo cluster. This works with individual servers, a replica set, -// a replica pair, one or multiple mongos routers, etc. - -type mongoCluster struct { - sync.RWMutex - serverSynced sync.Cond - userSeeds []string - dynaSeeds []string - servers mongoServers - masters mongoServers - references int - syncing bool - syncCount uint - cachedIndex map[string]bool - sync chan bool - dial dialer - dialInfo *DialInfo -} - -func newCluster(userSeeds []string, info *DialInfo) *mongoCluster { - cluster := &mongoCluster{ - userSeeds: userSeeds, - references: 1, - dial: dialer{info.Dial, info.DialServer}, - dialInfo: info, - } - cluster.serverSynced.L = cluster.RWMutex.RLocker() - cluster.sync = make(chan bool, 1) - stats.cluster(+1) - go cluster.syncServersLoop() - return cluster -} - -// Acquire increases the reference count for the cluster. -func (cluster *mongoCluster) Acquire() { - cluster.Lock() - cluster.references++ - debugf("Cluster %p acquired (refs=%d)", cluster, cluster.references) - cluster.Unlock() -} - -// Release decreases the reference count for the cluster. Once -// it reaches zero, all servers will be closed. -func (cluster *mongoCluster) Release() { - cluster.Lock() - if cluster.references == 0 { - panic("cluster.Release() with references == 0") - } - cluster.references-- - debugf("Cluster %p released (refs=%d)", cluster, cluster.references) - if cluster.references == 0 { - for _, server := range cluster.servers.Slice() { - server.Close() - } - // Wake up the sync loop so it can die. - cluster.syncServers() - stats.cluster(-1) - } - cluster.Unlock() -} - -func (cluster *mongoCluster) LiveServers() (servers []string) { - cluster.RLock() - for _, serv := range cluster.servers.Slice() { - servers = append(servers, serv.Addr) - } - cluster.RUnlock() - return servers -} - -func (cluster *mongoCluster) removeServer(server *mongoServer) { - cluster.Lock() - cluster.masters.Remove(server) - other := cluster.servers.Remove(server) - cluster.Unlock() - if other != nil { - other.CloseIdle() - log("Removed server ", server.Addr, " from cluster.") - } - server.CloseIdle() -} - -type isMasterResult struct { - IsMaster bool - Secondary bool - Primary string - Hosts []string - Passives []string - Tags bson.D - Msg string - SetName string `bson:"setName"` - MaxWireVersion int `bson:"maxWireVersion"` -} - -func (cluster *mongoCluster) isMaster(socket *mongoSocket, result *isMasterResult) error { - // Monotonic let's it talk to a slave and still hold the socket. - session := newSession(Monotonic, cluster, cluster.dialInfo) - session.setSocket(socket) - - var cmd = bson.D{{Name: "isMaster", Value: 1}} - - // Send client metadata to the server to identify this socket if this is - // the first isMaster call only. - // - // isMaster commands issued after the initial connection handshake MUST NOT contain handshake arguments - // https://github.com/mongodb/specifications/blob/master/source/mongodb-handshake/handshake.rst#connection-handshake - // - socket.sendMeta.Do(func() { - var meta = bson.M{ - "driver": bson.M{ - "name": "mgo", - "version": "globalsign", - }, - "os": bson.M{ - "type": runtime.GOOS, - "architecture": runtime.GOARCH, - }, - } - - // Include the application name if set - if cluster.dialInfo.AppName != "" { - meta["application"] = bson.M{"name": cluster.dialInfo.AppName} - } - - cmd = append(cmd, bson.DocElem{ - Name: "client", - Value: meta, - }) - }) - - err := session.runOnSocket(socket, cmd, result) - session.Close() - return err -} - -type possibleTimeout interface { - Timeout() bool -} - -func (cluster *mongoCluster) syncServer(server *mongoServer) (info *mongoServerInfo, hosts []string, err error) { - addr := server.Addr - log("SYNC Processing ", addr, "...") - - // Retry a few times to avoid knocking a server down for a hiccup. - var result isMasterResult - var tryerr error - for retry := 0; ; retry++ { - if retry == 3 || retry == 1 && cluster.dialInfo.FailFast { - return nil, nil, tryerr - } - if retry > 0 { - // Don't abuse the server needlessly if there's something actually wrong. - if err, ok := tryerr.(possibleTimeout); ok && err.Timeout() { - // Give a chance for waiters to timeout as well. - cluster.serverSynced.Broadcast() - } - time.Sleep(syncShortDelay) - } - - // Don't ever hit the pool limit for syncing - config := cluster.dialInfo.Copy() - config.PoolLimit = 0 - - socket, _, err := server.AcquireSocket(config) - if err != nil { - tryerr = err - logf("SYNC Failed to get socket to %s: %v", addr, err) - continue - } - err = cluster.isMaster(socket, &result) - - // Restore the correct dial config before returning it to the pool - socket.dialInfo = cluster.dialInfo - socket.Release() - - if err != nil { - tryerr = err - logf("SYNC Command 'ismaster' to %s failed: %v", addr, err) - continue - } - debugf("SYNC Result of 'ismaster' from %s: %#v", addr, result) - break - } - - if cluster.dialInfo.ReplicaSetName != "" && result.SetName != cluster.dialInfo.ReplicaSetName { - logf("SYNC Server %s is not a member of replica set %q", addr, cluster.dialInfo.ReplicaSetName) - return nil, nil, fmt.Errorf("server %s is not a member of replica set %q", addr, cluster.dialInfo.ReplicaSetName) - } - - if result.IsMaster { - debugf("SYNC %s is a master.", addr) - if !server.info.Master { - // Made an incorrect assumption above, so fix stats. - stats.conn(-1, false) - stats.conn(+1, true) - } - } else if result.Secondary { - debugf("SYNC %s is a slave.", addr) - } else if cluster.dialInfo.Direct { - logf("SYNC %s in unknown state. Pretending it's a slave due to direct connection.", addr) - } else { - logf("SYNC %s is neither a master nor a slave.", addr) - // Let stats track it as whatever was known before. - return nil, nil, errors.New(addr + " is not a master nor slave") - } - - info = &mongoServerInfo{ - Master: result.IsMaster, - Mongos: result.Msg == "isdbgrid", - Tags: result.Tags, - SetName: result.SetName, - MaxWireVersion: result.MaxWireVersion, - } - - hosts = make([]string, 0, 1+len(result.Hosts)+len(result.Passives)) - if result.Primary != "" { - // First in the list to speed up master discovery. - hosts = append(hosts, result.Primary) - } - hosts = append(hosts, result.Hosts...) - hosts = append(hosts, result.Passives...) - - debugf("SYNC %s knows about the following peers: %#v", addr, hosts) - return info, hosts, nil -} - -type syncKind bool - -const ( - completeSync syncKind = true - partialSync syncKind = false -) - -func (cluster *mongoCluster) addServer(server *mongoServer, info *mongoServerInfo, syncKind syncKind) { - cluster.Lock() - current := cluster.servers.Search(server.ResolvedAddr) - if current == nil { - if syncKind == partialSync { - cluster.Unlock() - server.Close() - log("SYNC Discarding unknown server ", server.Addr, " due to partial sync.") - return - } - cluster.servers.Add(server) - if info.Master { - cluster.masters.Add(server) - log("SYNC Adding ", server.Addr, " to cluster as a master.") - } else { - log("SYNC Adding ", server.Addr, " to cluster as a slave.") - } - } else { - if server != current { - panic("addServer attempting to add duplicated server") - } - if server.Info().Master != info.Master { - if info.Master { - log("SYNC Server ", server.Addr, " is now a master.") - cluster.masters.Add(server) - } else { - log("SYNC Server ", server.Addr, " is now a slave.") - cluster.masters.Remove(server) - } - } - } - server.SetInfo(info) - debugf("SYNC Broadcasting availability of server %s", server.Addr) - cluster.serverSynced.Broadcast() - cluster.Unlock() -} - -func (cluster *mongoCluster) getKnownAddrs() []string { - cluster.RLock() - max := len(cluster.userSeeds) + len(cluster.dynaSeeds) + cluster.servers.Len() - seen := make(map[string]bool, max) - known := make([]string, 0, max) - - add := func(addr string) { - if _, found := seen[addr]; !found { - seen[addr] = true - known = append(known, addr) - } - } - - for _, addr := range cluster.userSeeds { - add(addr) - } - for _, addr := range cluster.dynaSeeds { - add(addr) - } - for _, serv := range cluster.servers.Slice() { - add(serv.Addr) - } - cluster.RUnlock() - - return known -} - -// syncServers injects a value into the cluster.sync channel to force -// an iteration of the syncServersLoop function. -func (cluster *mongoCluster) syncServers() { - select { - case cluster.sync <- true: - default: - } -} - -// How long to wait for a checkup of the cluster topology if nothing -// else kicks a synchronization before that. -const syncServersDelay = 30 * time.Second -const syncShortDelay = 500 * time.Millisecond - -// syncServersLoop loops while the cluster is alive to keep its idea of -// the server topology up-to-date. It must be called just once from -// newCluster. The loop iterates once syncServersDelay has passed, or -// if somebody injects a value into the cluster.sync channel to force a -// synchronization. A loop iteration will contact all servers in -// parallel, ask them about known peers and their own role within the -// cluster, and then attempt to do the same with all the peers -// retrieved. -func (cluster *mongoCluster) syncServersLoop() { - for { - debugf("SYNC Cluster %p is starting a sync loop iteration.", cluster) - - cluster.Lock() - if cluster.references == 0 { - cluster.Unlock() - break - } - cluster.references++ // Keep alive while syncing. - direct := cluster.dialInfo.Direct - cluster.Unlock() - - cluster.syncServersIteration(direct) - - // We just synchronized, so consume any outstanding requests. - select { - case <-cluster.sync: - default: - } - - cluster.Release() - - // Hold off before allowing another sync. No point in - // burning CPU looking for down servers. - if !cluster.dialInfo.FailFast { - time.Sleep(syncShortDelay) - } - - cluster.Lock() - if cluster.references == 0 { - cluster.Unlock() - break - } - cluster.syncCount++ - // Poke all waiters so they have a chance to timeout or - // restart syncing if they wish to. - cluster.serverSynced.Broadcast() - // Check if we have to restart immediately either way. - restart := !direct && cluster.masters.Empty() || cluster.servers.Empty() - cluster.Unlock() - - if restart { - log("SYNC No masters found. Will synchronize again.") - time.Sleep(syncShortDelay) - continue - } - - debugf("SYNC Cluster %p waiting for next requested or scheduled sync.", cluster) - - // Hold off until somebody explicitly requests a synchronization - // or it's time to check for a cluster topology change again. - select { - case <-cluster.sync: - case <-time.After(syncServersDelay): - } - } - debugf("SYNC Cluster %p is stopping its sync loop.", cluster) -} - -func (cluster *mongoCluster) server(addr string, tcpaddr *net.TCPAddr) *mongoServer { - cluster.RLock() - server := cluster.servers.Search(tcpaddr.String()) - cluster.RUnlock() - if server != nil { - return server - } - return newServer(addr, tcpaddr, cluster.sync, cluster.dial, cluster.dialInfo) -} - -func resolveAddr(addr string) (*net.TCPAddr, error) { - // Simple cases that do not need actual resolution. Works with IPv4 and v6. - if host, port, err := net.SplitHostPort(addr); err == nil { - if port, _ := strconv.Atoi(port); port > 0 { - zone := "" - if i := strings.LastIndex(host, "%"); i >= 0 { - zone = host[i+1:] - host = host[:i] - } - ip := net.ParseIP(host) - if ip != nil { - return &net.TCPAddr{IP: ip, Port: port, Zone: zone}, nil - } - } - } - - // Attempt to resolve IPv4 and v6 concurrently. - addrChan := make(chan *net.TCPAddr, 2) - for _, network := range []string{"udp4", "udp6"} { - network := network - go func() { - // The unfortunate UDP dialing hack allows having a timeout on address resolution. - conn, err := net.DialTimeout(network, addr, 10*time.Second) - if err != nil { - addrChan <- nil - } else { - addrChan <- (*net.TCPAddr)(conn.RemoteAddr().(*net.UDPAddr)) - conn.Close() - } - }() - } - - // Wait for the result of IPv4 and v6 resolution. Use IPv4 if available. - tcpaddr := <-addrChan - if tcpaddr == nil || len(tcpaddr.IP) != 4 { - var timeout <-chan time.Time - if tcpaddr != nil { - // Don't wait too long if an IPv6 address is known. - timeout = time.After(50 * time.Millisecond) - } - select { - case <-timeout: - case tcpaddr2 := <-addrChan: - if tcpaddr == nil || tcpaddr2 != nil { - // It's an IPv4 address or the only known address. Use it. - tcpaddr = tcpaddr2 - } - } - } - - if tcpaddr == nil { - log("SYNC Failed to resolve server address: ", addr) - return nil, errors.New("failed to resolve server address: " + addr) - } - if tcpaddr.String() != addr { - debug("SYNC Address ", addr, " resolved as ", tcpaddr.String()) - } - return tcpaddr, nil -} - -type pendingAdd struct { - server *mongoServer - info *mongoServerInfo -} - -func (cluster *mongoCluster) syncServersIteration(direct bool) { - log("SYNC Starting full topology synchronization...") - - var wg sync.WaitGroup - var m sync.Mutex - notYetAdded := make(map[string]pendingAdd) - addIfFound := make(map[string]bool) - seen := make(map[string]bool) - syncKind := partialSync - - var spawnSync func(addr string, byMaster bool) - spawnSync = func(addr string, byMaster bool) { - wg.Add(1) - go func() { - defer wg.Done() - - tcpaddr, err := resolveAddr(addr) - if err != nil { - log("SYNC Failed to start sync of ", addr, ": ", err.Error()) - return - } - resolvedAddr := tcpaddr.String() - - m.Lock() - if byMaster { - if pending, ok := notYetAdded[resolvedAddr]; ok { - delete(notYetAdded, resolvedAddr) - m.Unlock() - cluster.addServer(pending.server, pending.info, completeSync) - return - } - addIfFound[resolvedAddr] = true - } - if seen[resolvedAddr] { - m.Unlock() - return - } - seen[resolvedAddr] = true - m.Unlock() - - server := cluster.server(addr, tcpaddr) - info, hosts, err := cluster.syncServer(server) - if err != nil { - cluster.removeServer(server) - return - } - - m.Lock() - add := direct || info.Master || addIfFound[resolvedAddr] - if add { - syncKind = completeSync - } else { - notYetAdded[resolvedAddr] = pendingAdd{server, info} - } - m.Unlock() - if add { - cluster.addServer(server, info, completeSync) - } - if !direct { - for _, addr := range hosts { - spawnSync(addr, info.Master) - } - } - }() - } - - knownAddrs := cluster.getKnownAddrs() - for _, addr := range knownAddrs { - spawnSync(addr, false) - } - wg.Wait() - - if syncKind == completeSync { - logf("SYNC Synchronization was complete (got data from primary).") - for _, pending := range notYetAdded { - cluster.removeServer(pending.server) - } - } else { - logf("SYNC Synchronization was partial (cannot talk to primary).") - for _, pending := range notYetAdded { - cluster.addServer(pending.server, pending.info, partialSync) - } - } - - cluster.Lock() - mastersLen := cluster.masters.Len() - logf("SYNC Synchronization completed: %d master(s) and %d slave(s) alive.", mastersLen, cluster.servers.Len()-mastersLen) - - // Update dynamic seeds, but only if we have any good servers. Otherwise, - // leave them alone for better chances of a successful sync in the future. - if syncKind == completeSync { - dynaSeeds := make([]string, cluster.servers.Len()) - for i, server := range cluster.servers.Slice() { - dynaSeeds[i] = server.Addr - } - cluster.dynaSeeds = dynaSeeds - debugf("SYNC New dynamic seeds: %#v\n", dynaSeeds) - } - cluster.Unlock() -} - -// AcquireSocketWithPoolTimeout returns a socket to a server in the cluster. If slaveOk is -// true, it will attempt to return a socket to a slave server. If it is -// false, the socket will necessarily be to a master server. -func (cluster *mongoCluster) AcquireSocketWithPoolTimeout(mode Mode, slaveOk bool, syncTimeout time.Duration, serverTags []bson.D, info *DialInfo) (s *mongoSocket, err error) { - var started time.Time - var syncCount uint - for { - cluster.RLock() - for { - mastersLen := cluster.masters.Len() - slavesLen := cluster.servers.Len() - mastersLen - debugf("Cluster has %d known masters and %d known slaves.", mastersLen, slavesLen) - if mastersLen > 0 && !(slaveOk && mode == Secondary) || slavesLen > 0 && slaveOk { - break - } - if mastersLen > 0 && mode == Secondary && cluster.masters.HasMongos() { - break - } - if started.IsZero() { - // Initialize after fast path above. - started = time.Now() - syncCount = cluster.syncCount - } else if syncTimeout != 0 && started.Before(time.Now().Add(-syncTimeout)) || cluster.dialInfo.FailFast && cluster.syncCount != syncCount { - cluster.RUnlock() - return nil, errors.New("no reachable servers") - } - log("Waiting for servers to synchronize...") - cluster.syncServers() - - // Remember: this will release and reacquire the lock. - cluster.serverSynced.Wait() - } - - var server *mongoServer - if slaveOk { - server = cluster.servers.BestFit(mode, serverTags) - } else { - server = cluster.masters.BestFit(mode, nil) - } - cluster.RUnlock() - - if server == nil { - // Must have failed the requested tags. Sleep to avoid spinning. - time.Sleep(1e8) - continue - } - - s, abended, err := server.AcquireSocketWithBlocking(info) - if err == errPoolTimeout { - // No need to remove servers from the topology if acquiring a socket fails for this reason. - return nil, err - } - if err != nil { - cluster.removeServer(server) - cluster.syncServers() - continue - } - if abended && !slaveOk { - var result isMasterResult - err := cluster.isMaster(s, &result) - if err != nil || !result.IsMaster { - logf("Cannot confirm server %s as master (%v)", server.Addr, err) - s.Release() - cluster.syncServers() - time.Sleep(100 * time.Millisecond) - continue - } else { - // We've managed to successfully reconnect to the master, we are no longer abnormaly ended - server.Lock() - server.abended = false - server.Unlock() - } - } - return s, nil - } -} - -func (cluster *mongoCluster) CacheIndex(cacheKey string, exists bool) { - cluster.Lock() - if cluster.cachedIndex == nil { - cluster.cachedIndex = make(map[string]bool) - } - if exists { - cluster.cachedIndex[cacheKey] = true - } else { - delete(cluster.cachedIndex, cacheKey) - } - cluster.Unlock() -} - -func (cluster *mongoCluster) HasCachedIndex(cacheKey string) (result bool) { - cluster.RLock() - if cluster.cachedIndex != nil { - result = cluster.cachedIndex[cacheKey] - } - cluster.RUnlock() - return -} - -func (cluster *mongoCluster) ResetIndexCache() { - cluster.Lock() - cluster.cachedIndex = make(map[string]bool) - cluster.Unlock() -} diff --git a/vendor/github.com/globalsign/mgo/coarse_time.go b/vendor/github.com/globalsign/mgo/coarse_time.go deleted file mode 100644 index e54dd17c..00000000 --- a/vendor/github.com/globalsign/mgo/coarse_time.go +++ /dev/null @@ -1,62 +0,0 @@ -package mgo - -import ( - "sync" - "sync/atomic" - "time" -) - -// coarseTimeProvider provides a periodically updated (approximate) time value to -// amortise the cost of frequent calls to time.Now. -// -// A read throughput increase of ~6% was measured when using coarseTimeProvider with the -// high-precision event timer (HPET) on FreeBSD 11.1 and Go 1.10.1 after merging -// #116. -// -// Calling Now returns a time.Time that is updated at the configured interval, -// however due to scheduling the value may be marginally older than expected. -// -// coarseTimeProvider is safe for concurrent use. -type coarseTimeProvider struct { - once sync.Once - stop chan struct{} - last atomic.Value -} - -// Now returns the most recently acquired time.Time value. -func (t *coarseTimeProvider) Now() time.Time { - return t.last.Load().(time.Time) -} - -// Close stops the periodic update of t. -// -// Any subsequent calls to Now will return the same value forever. -func (t *coarseTimeProvider) Close() { - t.once.Do(func() { - close(t.stop) - }) -} - -// newcoarseTimeProvider returns a coarseTimeProvider configured to update at granularity. -func newcoarseTimeProvider(granularity time.Duration) *coarseTimeProvider { - t := &coarseTimeProvider{ - stop: make(chan struct{}), - } - - t.last.Store(time.Now()) - - go func() { - ticker := time.NewTicker(granularity) - for { - select { - case <-t.stop: - ticker.Stop() - return - case <-ticker.C: - t.last.Store(time.Now()) - } - } - }() - - return t -} diff --git a/vendor/github.com/globalsign/mgo/doc.go b/vendor/github.com/globalsign/mgo/doc.go deleted file mode 100644 index f3f373bf..00000000 --- a/vendor/github.com/globalsign/mgo/doc.go +++ /dev/null @@ -1,35 +0,0 @@ -// Package mgo (pronounced as "mango") offers a rich MongoDB driver for Go. -// -// Detailed documentation of the API is available at GoDoc: -// -// https://godoc.org/github.com/globalsign/mgo -// -// Usage of the driver revolves around the concept of sessions. To -// get started, obtain a session using the Dial function: -// -// session, err := mgo.Dial(url) -// -// This will establish one or more connections with the cluster of -// servers defined by the url parameter. From then on, the cluster -// may be queried with multiple consistency rules (see SetMode) and -// documents retrieved with statements such as: -// -// c := session.DB(database).C(collection) -// err := c.Find(query).One(&result) -// -// New sessions are typically created by calling session.Copy on the -// initial session obtained at dial time. These new sessions will share -// the same cluster information and connection pool, and may be easily -// handed into other methods and functions for organizing logic. -// Every session created must have its Close method called at the end -// of its life time, so its resources may be put back in the pool or -// collected, depending on the case. -// -// There is a sub-package that provides support for BSON, which can be -// used by itself as well: -// -// https://godoc.org/github.com/globalsign/mgo/bson -// -// For more details, see the documentation for the types and methods. -// -package mgo diff --git a/vendor/github.com/globalsign/mgo/gridfs.go b/vendor/github.com/globalsign/mgo/gridfs.go deleted file mode 100644 index 0954b166..00000000 --- a/vendor/github.com/globalsign/mgo/gridfs.go +++ /dev/null @@ -1,782 +0,0 @@ -// mgo - MongoDB driver for Go -// -// Copyright (c) 2010-2012 - Gustavo Niemeyer -// -// All rights reserved. -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are met: -// -// 1. Redistributions of source code must retain the above copyright notice, this -// list of conditions and the following disclaimer. -// 2. Redistributions in binary form must reproduce the above copyright notice, -// this list of conditions and the following disclaimer in the documentation -// and/or other materials provided with the distribution. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR -// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND -// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -package mgo - -import ( - "crypto/md5" - "encoding/hex" - "errors" - "hash" - "io" - "os" - "sync" - "time" - - "github.com/globalsign/mgo/bson" -) - -// GridFS stores files in two collections: -// -// - chunks stores the binary chunks. For details, see the chunks Collection. -// - files stores the file’s metadata. For details, see the files Collection. -// -// GridFS places the collections in a common bucket by prefixing each with the bucket name. -// By default, GridFS uses two collections with a bucket named fs: -// -// - fs.files -// - fs.chunks -// -// You can choose a different bucket name, as well as create multiple buckets in a single database. -// The full collection name, which includes the bucket name, is subject to the namespace length limit. -// -// Relevant documentation: -// -// https://docs.mongodb.com/manual/core/gridfs/ -// https://docs.mongodb.com/manual/core/gridfs/#gridfs-chunks-collection -// https://docs.mongodb.com/manual/core/gridfs/#gridfs-files-collection -// -type GridFS struct { - Files *Collection - Chunks *Collection -} - -type gfsFileMode int - -const ( - gfsClosed gfsFileMode = 0 - gfsReading gfsFileMode = 1 - gfsWriting gfsFileMode = 2 -) - -// GridFile document in files collection -type GridFile struct { - m sync.Mutex - c sync.Cond - gfs *GridFS - mode gfsFileMode - err error - - chunk int - offset int64 - - wpending int - wbuf []byte - wsum hash.Hash - - rbuf []byte - rcache *gfsCachedChunk - - doc gfsFile -} - -type gfsFile struct { - Id interface{} `bson:"_id"` - ChunkSize int `bson:"chunkSize"` - UploadDate time.Time `bson:"uploadDate"` - Length int64 `bson:",minsize"` - MD5 string - Filename string `bson:",omitempty"` - ContentType string `bson:"contentType,omitempty"` - Metadata *bson.Raw `bson:",omitempty"` -} - -type gfsChunk struct { - Id interface{} `bson:"_id"` - FilesId interface{} `bson:"files_id"` - N int - Data []byte -} - -type gfsCachedChunk struct { - wait sync.Mutex - n int - data []byte - err error -} - -func newGridFS(db *Database, prefix string) *GridFS { - return &GridFS{db.C(prefix + ".files"), db.C(prefix + ".chunks")} -} - -func (gfs *GridFS) newFile() *GridFile { - file := &GridFile{gfs: gfs} - file.c.L = &file.m - //runtime.SetFinalizer(file, finalizeFile) - return file -} - -func finalizeFile(file *GridFile) { - file.Close() -} - -// Create creates a new file with the provided name in the GridFS. If the file -// name already exists, a new version will be inserted with an up-to-date -// uploadDate that will cause it to be atomically visible to the Open and -// OpenId methods. If the file name is not important, an empty name may be -// provided and the file Id used instead. -// -// It's important to Close files whether they are being written to -// or read from, and to check the err result to ensure the operation -// completed successfully. -// -// A simple example inserting a new file: -// -// func check(err error) { -// if err != nil { -// panic(err.String()) -// } -// } -// file, err := db.GridFS("fs").Create("myfile.txt") -// check(err) -// n, err := file.Write([]byte("Hello world!")) -// check(err) -// err = file.Close() -// check(err) -// fmt.Printf("%d bytes written\n", n) -// -// The io.Writer interface is implemented by *GridFile and may be used to -// help on the file creation. For example: -// -// file, err := db.GridFS("fs").Create("myfile.txt") -// check(err) -// messages, err := os.Open("/var/log/messages") -// check(err) -// defer messages.Close() -// err = io.Copy(file, messages) -// check(err) -// err = file.Close() -// check(err) -// -func (gfs *GridFS) Create(name string) (file *GridFile, err error) { - file = gfs.newFile() - file.mode = gfsWriting - file.wsum = md5.New() - file.doc = gfsFile{Id: bson.NewObjectId(), ChunkSize: 255 * 1024, Filename: name} - return -} - -// OpenId returns the file with the provided id, for reading. -// If the file isn't found, err will be set to mgo.ErrNotFound. -// -// It's important to Close files whether they are being written to -// or read from, and to check the err result to ensure the operation -// completed successfully. -// -// The following example will print the first 8192 bytes from the file: -// -// func check(err error) { -// if err != nil { -// panic(err.String()) -// } -// } -// file, err := db.GridFS("fs").OpenId(objid) -// check(err) -// b := make([]byte, 8192) -// n, err := file.Read(b) -// check(err) -// fmt.Println(string(b)) -// check(err) -// err = file.Close() -// check(err) -// fmt.Printf("%d bytes read\n", n) -// -// The io.Reader interface is implemented by *GridFile and may be used to -// deal with it. As an example, the following snippet will dump the whole -// file into the standard output: -// -// file, err := db.GridFS("fs").OpenId(objid) -// check(err) -// err = io.Copy(os.Stdout, file) -// check(err) -// err = file.Close() -// check(err) -// -func (gfs *GridFS) OpenId(id interface{}) (file *GridFile, err error) { - var doc gfsFile - err = gfs.Files.Find(bson.M{"_id": id}).One(&doc) - if err != nil { - return - } - file = gfs.newFile() - file.mode = gfsReading - file.doc = doc - return -} - -// Open returns the most recently uploaded file with the provided -// name, for reading. If the file isn't found, err will be set -// to mgo.ErrNotFound. -// -// It's important to Close files whether they are being written to -// or read from, and to check the err result to ensure the operation -// completed successfully. -// -// The following example will print the first 8192 bytes from the file: -// -// file, err := db.GridFS("fs").Open("myfile.txt") -// check(err) -// b := make([]byte, 8192) -// n, err := file.Read(b) -// check(err) -// fmt.Println(string(b)) -// check(err) -// err = file.Close() -// check(err) -// fmt.Printf("%d bytes read\n", n) -// -// The io.Reader interface is implemented by *GridFile and may be used to -// deal with it. As an example, the following snippet will dump the whole -// file into the standard output: -// -// file, err := db.GridFS("fs").Open("myfile.txt") -// check(err) -// err = io.Copy(os.Stdout, file) -// check(err) -// err = file.Close() -// check(err) -// -func (gfs *GridFS) Open(name string) (file *GridFile, err error) { - var doc gfsFile - err = gfs.Files.Find(bson.M{"filename": name}).Sort("-uploadDate").One(&doc) - if err != nil { - return - } - file = gfs.newFile() - file.mode = gfsReading - file.doc = doc - return -} - -// OpenNext opens the next file from iter for reading, sets *file to it, -// and returns true on the success case. If no more documents are available -// on iter or an error occurred, *file is set to nil and the result is false. -// Errors will be available via iter.Err(). -// -// The iter parameter must be an iterator on the GridFS files collection. -// Using the GridFS.Find method is an easy way to obtain such an iterator, -// but any iterator on the collection will work. -// -// If the provided *file is non-nil, OpenNext will close it before attempting -// to iterate to the next element. This means that in a loop one only -// has to worry about closing files when breaking out of the loop early -// (break, return, or panic). -// -// For example: -// -// gfs := db.GridFS("fs") -// query := gfs.Find(nil).Sort("filename") -// iter := query.Iter() -// var f *mgo.GridFile -// for gfs.OpenNext(iter, &f) { -// fmt.Printf("Filename: %s\n", f.Name()) -// } -// if iter.Close() != nil { -// panic(iter.Close()) -// } -// -func (gfs *GridFS) OpenNext(iter *Iter, file **GridFile) bool { - if *file != nil { - // Ignoring the error here shouldn't be a big deal - // as we're reading the file and the loop iteration - // for this file is finished. - _ = (*file).Close() - } - var doc gfsFile - if !iter.Next(&doc) { - *file = nil - return false - } - f := gfs.newFile() - f.mode = gfsReading - f.doc = doc - *file = f - return true -} - -// Find runs query on GridFS's files collection and returns -// the resulting Query. -// -// This logic: -// -// gfs := db.GridFS("fs") -// iter := gfs.Find(nil).Iter() -// -// Is equivalent to: -// -// files := db.C("fs" + ".files") -// iter := files.Find(nil).Iter() -// -func (gfs *GridFS) Find(query interface{}) *Query { - return gfs.Files.Find(query) -} - -// RemoveId deletes the file with the provided id from the GridFS. -func (gfs *GridFS) RemoveId(id interface{}) error { - err := gfs.Files.Remove(bson.M{"_id": id}) - if err != nil { - return err - } - _, err = gfs.Chunks.RemoveAll(bson.D{{Name: "files_id", Value: id}}) - return err -} - -type gfsDocId struct { - Id interface{} `bson:"_id"` -} - -// Remove deletes all files with the provided name from the GridFS. -func (gfs *GridFS) Remove(name string) (err error) { - iter := gfs.Files.Find(bson.M{"filename": name}).Select(bson.M{"_id": 1}).Iter() - var doc gfsDocId - for iter.Next(&doc) { - if e := gfs.RemoveId(doc.Id); e != nil { - err = e - } - } - if err == nil { - err = iter.Close() - } - return err -} - -func (file *GridFile) assertMode(mode gfsFileMode) { - switch file.mode { - case mode: - return - case gfsWriting: - panic("GridFile is open for writing") - case gfsReading: - panic("GridFile is open for reading") - case gfsClosed: - panic("GridFile is closed") - default: - panic("internal error: missing GridFile mode") - } -} - -// SetChunkSize sets size of saved chunks. Once the file is written to, it -// will be split in blocks of that size and each block saved into an -// independent chunk document. The default chunk size is 255kb. -// -// It is a runtime error to call this function once the file has started -// being written to. -func (file *GridFile) SetChunkSize(bytes int) { - file.assertMode(gfsWriting) - debugf("GridFile %p: setting chunk size to %d", file, bytes) - file.m.Lock() - file.doc.ChunkSize = bytes - file.m.Unlock() -} - -// Id returns the current file Id. -func (file *GridFile) Id() interface{} { - return file.doc.Id -} - -// SetId changes the current file Id. -// -// It is a runtime error to call this function once the file has started -// being written to, or when the file is not open for writing. -func (file *GridFile) SetId(id interface{}) { - file.assertMode(gfsWriting) - file.m.Lock() - file.doc.Id = id - file.m.Unlock() -} - -// Name returns the optional file name. An empty string will be returned -// in case it is unset. -func (file *GridFile) Name() string { - return file.doc.Filename -} - -// SetName changes the optional file name. An empty string may be used to -// unset it. -// -// It is a runtime error to call this function when the file is not open -// for writing. -func (file *GridFile) SetName(name string) { - file.assertMode(gfsWriting) - file.m.Lock() - file.doc.Filename = name - file.m.Unlock() -} - -// ContentType returns the optional file content type. An empty string will be -// returned in case it is unset. -func (file *GridFile) ContentType() string { - return file.doc.ContentType -} - -// SetContentType changes the optional file content type. An empty string may be -// used to unset it. -// -// It is a runtime error to call this function when the file is not open -// for writing. -func (file *GridFile) SetContentType(ctype string) { - file.assertMode(gfsWriting) - file.m.Lock() - file.doc.ContentType = ctype - file.m.Unlock() -} - -// GetMeta unmarshals the optional "metadata" field associated with the -// file into the result parameter. The meaning of keys under that field -// is user-defined. For example: -// -// result := struct{ INode int }{} -// err = file.GetMeta(&result) -// if err != nil { -// panic(err.String()) -// } -// fmt.Printf("inode: %d\n", result.INode) -// -func (file *GridFile) GetMeta(result interface{}) (err error) { - file.m.Lock() - if file.doc.Metadata != nil { - err = bson.Unmarshal(file.doc.Metadata.Data, result) - } - file.m.Unlock() - return -} - -// SetMeta changes the optional "metadata" field associated with the -// file. The meaning of keys under that field is user-defined. -// For example: -// -// file.SetMeta(bson.M{"inode": inode}) -// -// It is a runtime error to call this function when the file is not open -// for writing. -func (file *GridFile) SetMeta(metadata interface{}) { - file.assertMode(gfsWriting) - data, err := bson.Marshal(metadata) - file.m.Lock() - if err != nil && file.err == nil { - file.err = err - } else { - file.doc.Metadata = &bson.Raw{Data: data} - } - file.m.Unlock() -} - -// Size returns the file size in bytes. -func (file *GridFile) Size() (bytes int64) { - file.m.Lock() - bytes = file.doc.Length - file.m.Unlock() - return -} - -// MD5 returns the file MD5 as a hex-encoded string. -func (file *GridFile) MD5() (md5 string) { - return file.doc.MD5 -} - -// UploadDate returns the file upload time. -func (file *GridFile) UploadDate() time.Time { - return file.doc.UploadDate -} - -// SetUploadDate changes the file upload time. -// -// It is a runtime error to call this function when the file is not open -// for writing. -func (file *GridFile) SetUploadDate(t time.Time) { - file.assertMode(gfsWriting) - file.m.Lock() - file.doc.UploadDate = t - file.m.Unlock() -} - -// Close flushes any pending changes in case the file is being written -// to, waits for any background operations to finish, and closes the file. -// -// It's important to Close files whether they are being written to -// or read from, and to check the err result to ensure the operation -// completed successfully. -func (file *GridFile) Close() (err error) { - file.m.Lock() - defer file.m.Unlock() - if file.mode == gfsWriting { - if len(file.wbuf) > 0 && file.err == nil { - file.insertChunk(file.wbuf) - file.wbuf = file.wbuf[0:0] - } - file.completeWrite() - } else if file.mode == gfsReading && file.rcache != nil { - file.rcache.wait.Lock() - file.rcache = nil - } - file.mode = gfsClosed - debugf("GridFile %p: closed", file) - return file.err -} - -func (file *GridFile) completeWrite() { - for file.wpending > 0 { - debugf("GridFile %p: waiting for %d pending chunks to complete file write", file, file.wpending) - file.c.Wait() - } - if file.err == nil { - hexsum := hex.EncodeToString(file.wsum.Sum(nil)) - if file.doc.UploadDate.IsZero() { - file.doc.UploadDate = bson.Now() - } - file.doc.MD5 = hexsum - file.err = file.gfs.Files.Insert(file.doc) - } - if file.err != nil { - file.gfs.Chunks.RemoveAll(bson.D{{Name: "files_id", Value: file.doc.Id}}) - } - if file.err == nil { - index := Index{ - Key: []string{"files_id", "n"}, - Unique: true, - } - file.err = file.gfs.Chunks.EnsureIndex(index) - } -} - -// Abort cancels an in-progress write, preventing the file from being -// automically created and ensuring previously written chunks are -// removed when the file is closed. -// -// It is a runtime error to call Abort when the file was not opened -// for writing. -func (file *GridFile) Abort() { - if file.mode != gfsWriting { - panic("file.Abort must be called on file opened for writing") - } - file.err = errors.New("write aborted") -} - -// Write writes the provided data to the file and returns the -// number of bytes written and an error in case something -// wrong happened. -// -// The file will internally cache the data so that all but the last -// chunk sent to the database have the size defined by SetChunkSize. -// This also means that errors may be deferred until a future call -// to Write or Close. -// -// The parameters and behavior of this function turn the file -// into an io.Writer. -func (file *GridFile) Write(data []byte) (n int, err error) { - file.assertMode(gfsWriting) - file.m.Lock() - debugf("GridFile %p: writing %d bytes", file, len(data)) - defer file.m.Unlock() - - if file.err != nil { - return 0, file.err - } - - n = len(data) - file.doc.Length += int64(n) - chunkSize := file.doc.ChunkSize - - if len(file.wbuf)+len(data) < chunkSize { - file.wbuf = append(file.wbuf, data...) - return - } - - // First, flush file.wbuf complementing with data. - if len(file.wbuf) > 0 { - missing := chunkSize - len(file.wbuf) - if missing > len(data) { - missing = len(data) - } - file.wbuf = append(file.wbuf, data[:missing]...) - data = data[missing:] - file.insertChunk(file.wbuf) - file.wbuf = file.wbuf[0:0] - } - - // Then, flush all chunks from data without copying. - for len(data) > chunkSize { - size := chunkSize - if size > len(data) { - size = len(data) - } - file.insertChunk(data[:size]) - data = data[size:] - } - - // And append the rest for a future call. - file.wbuf = append(file.wbuf, data...) - - return n, file.err -} - -func (file *GridFile) insertChunk(data []byte) { - n := file.chunk - file.chunk++ - debugf("GridFile %p: adding to checksum: %q", file, string(data)) - file.wsum.Write(data) - - for file.doc.ChunkSize*file.wpending >= 1024*1024 { - // Hold on.. we got a MB pending. - file.c.Wait() - if file.err != nil { - return - } - } - - file.wpending++ - - debugf("GridFile %p: inserting chunk %d with %d bytes", file, n, len(data)) - - // We may not own the memory of data, so rather than - // simply copying it, we'll marshal the document ahead of time. - data, err := bson.Marshal(gfsChunk{bson.NewObjectId(), file.doc.Id, n, data}) - if err != nil { - file.err = err - return - } - - go func() { - err := file.gfs.Chunks.Insert(bson.Raw{Data: data}) - file.m.Lock() - file.wpending-- - if err != nil && file.err == nil { - file.err = err - } - file.c.Broadcast() - file.m.Unlock() - }() -} - -// Seek sets the offset for the next Read or Write on file to -// offset, interpreted according to whence: 0 means relative to -// the origin of the file, 1 means relative to the current offset, -// and 2 means relative to the end. It returns the new offset and -// an error, if any. -func (file *GridFile) Seek(offset int64, whence int) (pos int64, err error) { - file.m.Lock() - debugf("GridFile %p: seeking for %s (whence=%d)", file, offset, whence) - defer file.m.Unlock() - switch whence { - case os.SEEK_SET: - case os.SEEK_CUR: - offset += file.offset - case os.SEEK_END: - offset += file.doc.Length - default: - panic("unsupported whence value") - } - if offset > file.doc.Length { - return file.offset, errors.New("seek past end of file") - } - if offset == file.doc.Length { - // If we're seeking to the end of the file, - // no need to read anything. This enables - // a client to find the size of the file using only the - // io.ReadSeeker interface with low overhead. - file.offset = offset - return file.offset, nil - } - chunk := int(offset / int64(file.doc.ChunkSize)) - if chunk+1 == file.chunk && offset >= file.offset { - file.rbuf = file.rbuf[int(offset-file.offset):] - file.offset = offset - return file.offset, nil - } - file.offset = offset - file.chunk = chunk - file.rbuf = nil - file.rbuf, err = file.getChunk() - if err == nil { - file.rbuf = file.rbuf[int(file.offset-int64(chunk)*int64(file.doc.ChunkSize)):] - } - return file.offset, err -} - -// Read reads into b the next available data from the file and -// returns the number of bytes written and an error in case -// something wrong happened. At the end of the file, n will -// be zero and err will be set to io.EOF. -// -// The parameters and behavior of this function turn the file -// into an io.Reader. -func (file *GridFile) Read(b []byte) (n int, err error) { - file.assertMode(gfsReading) - file.m.Lock() - debugf("GridFile %p: reading at offset %d into buffer of length %d", file, file.offset, len(b)) - defer file.m.Unlock() - if file.offset == file.doc.Length { - return 0, io.EOF - } - for err == nil { - i := copy(b, file.rbuf) - n += i - file.offset += int64(i) - file.rbuf = file.rbuf[i:] - if i == len(b) || file.offset == file.doc.Length { - break - } - b = b[i:] - file.rbuf, err = file.getChunk() - } - return n, err -} - -func (file *GridFile) getChunk() (data []byte, err error) { - cache := file.rcache - file.rcache = nil - if cache != nil && cache.n == file.chunk { - debugf("GridFile %p: Getting chunk %d from cache", file, file.chunk) - cache.wait.Lock() - data, err = cache.data, cache.err - } else { - debugf("GridFile %p: Fetching chunk %d", file, file.chunk) - var doc gfsChunk - err = file.gfs.Chunks.Find(bson.D{{Name: "files_id", Value: file.doc.Id}, {Name: "n", Value: file.chunk}}).One(&doc) - data = doc.Data - } - file.chunk++ - if int64(file.chunk)*int64(file.doc.ChunkSize) < file.doc.Length { - // Read the next one in background. - cache = &gfsCachedChunk{n: file.chunk} - cache.wait.Lock() - debugf("GridFile %p: Scheduling chunk %d for background caching", file, file.chunk) - // Clone the session to avoid having it closed in between. - chunks := file.gfs.Chunks - session := chunks.Database.Session.Clone() - go func(id interface{}, n int) { - defer session.Close() - chunks = chunks.With(session) - var doc gfsChunk - cache.err = chunks.Find(bson.D{{Name: "files_id", Value: id}, {Name: "n", Value: n}}).One(&doc) - cache.data = doc.Data - cache.wait.Unlock() - }(file.doc.Id, file.chunk) - file.rcache = cache - } - debugf("Returning err: %#v", err) - return -} diff --git a/vendor/github.com/globalsign/mgo/internal/json/LICENSE b/vendor/github.com/globalsign/mgo/internal/json/LICENSE deleted file mode 100644 index 74487567..00000000 --- a/vendor/github.com/globalsign/mgo/internal/json/LICENSE +++ /dev/null @@ -1,27 +0,0 @@ -Copyright (c) 2012 The Go Authors. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/globalsign/mgo/internal/json/decode.go b/vendor/github.com/globalsign/mgo/internal/json/decode.go deleted file mode 100644 index d5ca1f9a..00000000 --- a/vendor/github.com/globalsign/mgo/internal/json/decode.go +++ /dev/null @@ -1,1685 +0,0 @@ -// Copyright 2010 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Represents JSON data structure using native Go types: booleans, floats, -// strings, arrays, and maps. - -package json - -import ( - "bytes" - "encoding" - "encoding/base64" - "errors" - "fmt" - "reflect" - "runtime" - "strconv" - "unicode" - "unicode/utf16" - "unicode/utf8" -) - -// Unmarshal parses the JSON-encoded data and stores the result -// in the value pointed to by v. -// -// Unmarshal uses the inverse of the encodings that -// Marshal uses, allocating maps, slices, and pointers as necessary, -// with the following additional rules: -// -// To unmarshal JSON into a pointer, Unmarshal first handles the case of -// the JSON being the JSON literal null. In that case, Unmarshal sets -// the pointer to nil. Otherwise, Unmarshal unmarshals the JSON into -// the value pointed at by the pointer. If the pointer is nil, Unmarshal -// allocates a new value for it to point to. -// -// To unmarshal JSON into a struct, Unmarshal matches incoming object -// keys to the keys used by Marshal (either the struct field name or its tag), -// preferring an exact match but also accepting a case-insensitive match. -// Unmarshal will only set exported fields of the struct. -// -// To unmarshal JSON into an interface value, -// Unmarshal stores one of these in the interface value: -// -// bool, for JSON booleans -// float64, for JSON numbers -// string, for JSON strings -// []interface{}, for JSON arrays -// map[string]interface{}, for JSON objects -// nil for JSON null -// -// To unmarshal a JSON array into a slice, Unmarshal resets the slice length -// to zero and then appends each element to the slice. -// As a special case, to unmarshal an empty JSON array into a slice, -// Unmarshal replaces the slice with a new empty slice. -// -// To unmarshal a JSON array into a Go array, Unmarshal decodes -// JSON array elements into corresponding Go array elements. -// If the Go array is smaller than the JSON array, -// the additional JSON array elements are discarded. -// If the JSON array is smaller than the Go array, -// the additional Go array elements are set to zero values. -// -// To unmarshal a JSON object into a map, Unmarshal first establishes a map to -// use, If the map is nil, Unmarshal allocates a new map. Otherwise Unmarshal -// reuses the existing map, keeping existing entries. Unmarshal then stores key- -// value pairs from the JSON object into the map. The map's key type must -// either be a string or implement encoding.TextUnmarshaler. -// -// If a JSON value is not appropriate for a given target type, -// or if a JSON number overflows the target type, Unmarshal -// skips that field and completes the unmarshaling as best it can. -// If no more serious errors are encountered, Unmarshal returns -// an UnmarshalTypeError describing the earliest such error. -// -// The JSON null value unmarshals into an interface, map, pointer, or slice -// by setting that Go value to nil. Because null is often used in JSON to mean -// ``not present,'' unmarshaling a JSON null into any other Go type has no effect -// on the value and produces no error. -// -// When unmarshaling quoted strings, invalid UTF-8 or -// invalid UTF-16 surrogate pairs are not treated as an error. -// Instead, they are replaced by the Unicode replacement -// character U+FFFD. -// -func Unmarshal(data []byte, v interface{}) error { - // Check for well-formedness. - // Avoids filling out half a data structure - // before discovering a JSON syntax error. - var d decodeState - err := checkValid(data, &d.scan) - if err != nil { - return err - } - - d.init(data) - return d.unmarshal(v) -} - -// Unmarshaler is the interface implemented by types -// that can unmarshal a JSON description of themselves. -// The input can be assumed to be a valid encoding of -// a JSON value. UnmarshalJSON must copy the JSON data -// if it wishes to retain the data after returning. -type Unmarshaler interface { - UnmarshalJSON([]byte) error -} - -// An UnmarshalTypeError describes a JSON value that was -// not appropriate for a value of a specific Go type. -type UnmarshalTypeError struct { - Value string // description of JSON value - "bool", "array", "number -5" - Type reflect.Type // type of Go value it could not be assigned to - Offset int64 // error occurred after reading Offset bytes -} - -func (e *UnmarshalTypeError) Error() string { - return "json: cannot unmarshal " + e.Value + " into Go value of type " + e.Type.String() -} - -// An UnmarshalFieldError describes a JSON object key that -// led to an unexported (and therefore unwritable) struct field. -// (No longer used; kept for compatibility.) -type UnmarshalFieldError struct { - Key string - Type reflect.Type - Field reflect.StructField -} - -func (e *UnmarshalFieldError) Error() string { - return "json: cannot unmarshal object key " + strconv.Quote(e.Key) + " into unexported field " + e.Field.Name + " of type " + e.Type.String() -} - -// An InvalidUnmarshalError describes an invalid argument passed to Unmarshal. -// (The argument to Unmarshal must be a non-nil pointer.) -type InvalidUnmarshalError struct { - Type reflect.Type -} - -func (e *InvalidUnmarshalError) Error() string { - if e.Type == nil { - return "json: Unmarshal(nil)" - } - - if e.Type.Kind() != reflect.Ptr { - return "json: Unmarshal(non-pointer " + e.Type.String() + ")" - } - return "json: Unmarshal(nil " + e.Type.String() + ")" -} - -func (d *decodeState) unmarshal(v interface{}) (err error) { - defer func() { - if r := recover(); r != nil { - if _, ok := r.(runtime.Error); ok { - panic(r) - } - err = r.(error) - } - }() - - rv := reflect.ValueOf(v) - if rv.Kind() != reflect.Ptr || rv.IsNil() { - return &InvalidUnmarshalError{reflect.TypeOf(v)} - } - - d.scan.reset() - // We decode rv not rv.Elem because the Unmarshaler interface - // test must be applied at the top level of the value. - d.value(rv) - return d.savedError -} - -// A Number represents a JSON number literal. -type Number string - -// String returns the literal text of the number. -func (n Number) String() string { return string(n) } - -// Float64 returns the number as a float64. -func (n Number) Float64() (float64, error) { - return strconv.ParseFloat(string(n), 64) -} - -// Int64 returns the number as an int64. -func (n Number) Int64() (int64, error) { - return strconv.ParseInt(string(n), 10, 64) -} - -// isValidNumber reports whether s is a valid JSON number literal. -func isValidNumber(s string) bool { - // This function implements the JSON numbers grammar. - // See https://tools.ietf.org/html/rfc7159#section-6 - // and http://json.org/number.gif - - if s == "" { - return false - } - - // Optional - - if s[0] == '-' { - s = s[1:] - if s == "" { - return false - } - } - - // Digits - switch { - default: - return false - - case s[0] == '0': - s = s[1:] - - case '1' <= s[0] && s[0] <= '9': - s = s[1:] - for len(s) > 0 && '0' <= s[0] && s[0] <= '9' { - s = s[1:] - } - } - - // . followed by 1 or more digits. - if len(s) >= 2 && s[0] == '.' && '0' <= s[1] && s[1] <= '9' { - s = s[2:] - for len(s) > 0 && '0' <= s[0] && s[0] <= '9' { - s = s[1:] - } - } - - // e or E followed by an optional - or + and - // 1 or more digits. - if len(s) >= 2 && (s[0] == 'e' || s[0] == 'E') { - s = s[1:] - if s[0] == '+' || s[0] == '-' { - s = s[1:] - if s == "" { - return false - } - } - for len(s) > 0 && '0' <= s[0] && s[0] <= '9' { - s = s[1:] - } - } - - // Make sure we are at the end. - return s == "" -} - -// decodeState represents the state while decoding a JSON value. -type decodeState struct { - data []byte - off int // read offset in data - scan scanner - nextscan scanner // for calls to nextValue - savedError error - useNumber bool - ext Extension -} - -// errPhase is used for errors that should not happen unless -// there is a bug in the JSON decoder or something is editing -// the data slice while the decoder executes. -var errPhase = errors.New("JSON decoder out of sync - data changing underfoot?") - -func (d *decodeState) init(data []byte) *decodeState { - d.data = data - d.off = 0 - d.savedError = nil - return d -} - -// error aborts the decoding by panicking with err. -func (d *decodeState) error(err error) { - panic(err) -} - -// saveError saves the first err it is called with, -// for reporting at the end of the unmarshal. -func (d *decodeState) saveError(err error) { - if d.savedError == nil { - d.savedError = err - } -} - -// next cuts off and returns the next full JSON value in d.data[d.off:]. -// The next value is known to be an object or array, not a literal. -func (d *decodeState) next() []byte { - c := d.data[d.off] - item, rest, err := nextValue(d.data[d.off:], &d.nextscan) - if err != nil { - d.error(err) - } - d.off = len(d.data) - len(rest) - - // Our scanner has seen the opening brace/bracket - // and thinks we're still in the middle of the object. - // invent a closing brace/bracket to get it out. - if c == '{' { - d.scan.step(&d.scan, '}') - } else if c == '[' { - d.scan.step(&d.scan, ']') - } else { - // Was inside a function name. Get out of it. - d.scan.step(&d.scan, '(') - d.scan.step(&d.scan, ')') - } - - return item -} - -// scanWhile processes bytes in d.data[d.off:] until it -// receives a scan code not equal to op. -// It updates d.off and returns the new scan code. -func (d *decodeState) scanWhile(op int) int { - var newOp int - for { - if d.off >= len(d.data) { - newOp = d.scan.eof() - d.off = len(d.data) + 1 // mark processed EOF with len+1 - } else { - c := d.data[d.off] - d.off++ - newOp = d.scan.step(&d.scan, c) - } - if newOp != op { - break - } - } - return newOp -} - -// value decodes a JSON value from d.data[d.off:] into the value. -// it updates d.off to point past the decoded value. -func (d *decodeState) value(v reflect.Value) { - if !v.IsValid() { - _, rest, err := nextValue(d.data[d.off:], &d.nextscan) - if err != nil { - d.error(err) - } - d.off = len(d.data) - len(rest) - - // d.scan thinks we're still at the beginning of the item. - // Feed in an empty string - the shortest, simplest value - - // so that it knows we got to the end of the value. - if d.scan.redo { - // rewind. - d.scan.redo = false - d.scan.step = stateBeginValue - } - d.scan.step(&d.scan, '"') - d.scan.step(&d.scan, '"') - - n := len(d.scan.parseState) - if n > 0 && d.scan.parseState[n-1] == parseObjectKey { - // d.scan thinks we just read an object key; finish the object - d.scan.step(&d.scan, ':') - d.scan.step(&d.scan, '"') - d.scan.step(&d.scan, '"') - d.scan.step(&d.scan, '}') - } - - return - } - - switch op := d.scanWhile(scanSkipSpace); op { - default: - d.error(errPhase) - - case scanBeginArray: - d.array(v) - - case scanBeginObject: - d.object(v) - - case scanBeginLiteral: - d.literal(v) - - case scanBeginName: - d.name(v) - } -} - -type unquotedValue struct{} - -// valueQuoted is like value but decodes a -// quoted string literal or literal null into an interface value. -// If it finds anything other than a quoted string literal or null, -// valueQuoted returns unquotedValue{}. -func (d *decodeState) valueQuoted() interface{} { - switch op := d.scanWhile(scanSkipSpace); op { - default: - d.error(errPhase) - - case scanBeginArray: - d.array(reflect.Value{}) - - case scanBeginObject: - d.object(reflect.Value{}) - - case scanBeginName: - switch v := d.nameInterface().(type) { - case nil, string: - return v - } - - case scanBeginLiteral: - switch v := d.literalInterface().(type) { - case nil, string: - return v - } - } - return unquotedValue{} -} - -// indirect walks down v allocating pointers as needed, -// until it gets to a non-pointer. -// if it encounters an Unmarshaler, indirect stops and returns that. -// if decodingNull is true, indirect stops at the last pointer so it can be set to nil. -func (d *decodeState) indirect(v reflect.Value, decodingNull bool) (Unmarshaler, encoding.TextUnmarshaler, reflect.Value) { - // If v is a named type and is addressable, - // start with its address, so that if the type has pointer methods, - // we find them. - if v.Kind() != reflect.Ptr && v.Type().Name() != "" && v.CanAddr() { - v = v.Addr() - } - for { - // Load value from interface, but only if the result will be - // usefully addressable. - if v.Kind() == reflect.Interface && !v.IsNil() { - e := v.Elem() - if e.Kind() == reflect.Ptr && !e.IsNil() && (!decodingNull || e.Elem().Kind() == reflect.Ptr) { - v = e - continue - } - } - - if v.Kind() != reflect.Ptr { - break - } - - if v.Elem().Kind() != reflect.Ptr && decodingNull && v.CanSet() { - break - } - if v.IsNil() { - v.Set(reflect.New(v.Type().Elem())) - } - if v.Type().NumMethod() > 0 { - if u, ok := v.Interface().(Unmarshaler); ok { - return u, nil, v - } - if u, ok := v.Interface().(encoding.TextUnmarshaler); ok { - return nil, u, v - } - } - v = v.Elem() - } - return nil, nil, v -} - -// array consumes an array from d.data[d.off-1:], decoding into the value v. -// the first byte of the array ('[') has been read already. -func (d *decodeState) array(v reflect.Value) { - // Check for unmarshaler. - u, ut, pv := d.indirect(v, false) - if u != nil { - d.off-- - err := u.UnmarshalJSON(d.next()) - if err != nil { - d.error(err) - } - return - } - if ut != nil { - d.saveError(&UnmarshalTypeError{"array", v.Type(), int64(d.off)}) - d.off-- - d.next() - return - } - - v = pv - - // Check type of target. - switch v.Kind() { - case reflect.Interface: - if v.NumMethod() == 0 { - // Decoding into nil interface? Switch to non-reflect code. - v.Set(reflect.ValueOf(d.arrayInterface())) - return - } - // Otherwise it's invalid. - fallthrough - default: - d.saveError(&UnmarshalTypeError{"array", v.Type(), int64(d.off)}) - d.off-- - d.next() - return - case reflect.Array: - case reflect.Slice: - break - } - - i := 0 - for { - // Look ahead for ] - can only happen on first iteration. - op := d.scanWhile(scanSkipSpace) - if op == scanEndArray { - break - } - - // Back up so d.value can have the byte we just read. - d.off-- - d.scan.undo(op) - - // Get element of array, growing if necessary. - if v.Kind() == reflect.Slice { - // Grow slice if necessary - if i >= v.Cap() { - newcap := v.Cap() + v.Cap()/2 - if newcap < 4 { - newcap = 4 - } - newv := reflect.MakeSlice(v.Type(), v.Len(), newcap) - reflect.Copy(newv, v) - v.Set(newv) - } - if i >= v.Len() { - v.SetLen(i + 1) - } - } - - if i < v.Len() { - // Decode into element. - d.value(v.Index(i)) - } else { - // Ran out of fixed array: skip. - d.value(reflect.Value{}) - } - i++ - - // Next token must be , or ]. - op = d.scanWhile(scanSkipSpace) - if op == scanEndArray { - break - } - if op != scanArrayValue { - d.error(errPhase) - } - } - - if i < v.Len() { - if v.Kind() == reflect.Array { - // Array. Zero the rest. - z := reflect.Zero(v.Type().Elem()) - for ; i < v.Len(); i++ { - v.Index(i).Set(z) - } - } else { - v.SetLen(i) - } - } - if i == 0 && v.Kind() == reflect.Slice { - v.Set(reflect.MakeSlice(v.Type(), 0, 0)) - } -} - -var nullLiteral = []byte("null") -var textUnmarshalerType = reflect.TypeOf(new(encoding.TextUnmarshaler)).Elem() - -// object consumes an object from d.data[d.off-1:], decoding into the value v. -// the first byte ('{') of the object has been read already. -func (d *decodeState) object(v reflect.Value) { - // Check for unmarshaler. - u, ut, pv := d.indirect(v, false) - if d.storeKeyed(pv) { - return - } - if u != nil { - d.off-- - err := u.UnmarshalJSON(d.next()) - if err != nil { - d.error(err) - } - return - } - if ut != nil { - d.saveError(&UnmarshalTypeError{"object", v.Type(), int64(d.off)}) - d.off-- - d.next() // skip over { } in input - return - } - v = pv - - // Decoding into nil interface? Switch to non-reflect code. - if v.Kind() == reflect.Interface && v.NumMethod() == 0 { - v.Set(reflect.ValueOf(d.objectInterface())) - return - } - - // Check type of target: - // struct or - // map[string]T or map[encoding.TextUnmarshaler]T - switch v.Kind() { - case reflect.Map: - // Map key must either have string kind or be an encoding.TextUnmarshaler. - t := v.Type() - if t.Key().Kind() != reflect.String && - !reflect.PtrTo(t.Key()).Implements(textUnmarshalerType) { - d.saveError(&UnmarshalTypeError{"object", v.Type(), int64(d.off)}) - d.off-- - d.next() // skip over { } in input - return - } - if v.IsNil() { - v.Set(reflect.MakeMap(t)) - } - case reflect.Struct: - - default: - d.saveError(&UnmarshalTypeError{"object", v.Type(), int64(d.off)}) - d.off-- - d.next() // skip over { } in input - return - } - - var mapElem reflect.Value - - empty := true - for { - // Read opening " of string key or closing }. - op := d.scanWhile(scanSkipSpace) - if op == scanEndObject { - if !empty && !d.ext.trailingCommas { - d.syntaxError("beginning of object key string") - } - break - } - empty = false - if op == scanBeginName { - if !d.ext.unquotedKeys { - d.syntaxError("beginning of object key string") - } - } else if op != scanBeginLiteral { - d.error(errPhase) - } - unquotedKey := op == scanBeginName - - // Read key. - start := d.off - 1 - op = d.scanWhile(scanContinue) - item := d.data[start : d.off-1] - var key []byte - if unquotedKey { - key = item - // TODO Fix code below to quote item when necessary. - } else { - var ok bool - key, ok = unquoteBytes(item) - if !ok { - d.error(errPhase) - } - } - - // Figure out field corresponding to key. - var subv reflect.Value - destring := false // whether the value is wrapped in a string to be decoded first - - if v.Kind() == reflect.Map { - elemType := v.Type().Elem() - if !mapElem.IsValid() { - mapElem = reflect.New(elemType).Elem() - } else { - mapElem.Set(reflect.Zero(elemType)) - } - subv = mapElem - } else { - var f *field - fields := cachedTypeFields(v.Type()) - for i := range fields { - ff := &fields[i] - if bytes.Equal(ff.nameBytes, key) { - f = ff - break - } - if f == nil && ff.equalFold(ff.nameBytes, key) { - f = ff - } - } - if f != nil { - subv = v - destring = f.quoted - for _, i := range f.index { - if subv.Kind() == reflect.Ptr { - if subv.IsNil() { - subv.Set(reflect.New(subv.Type().Elem())) - } - subv = subv.Elem() - } - subv = subv.Field(i) - } - } - } - - // Read : before value. - if op == scanSkipSpace { - op = d.scanWhile(scanSkipSpace) - } - if op != scanObjectKey { - d.error(errPhase) - } - - // Read value. - if destring { - switch qv := d.valueQuoted().(type) { - case nil: - d.literalStore(nullLiteral, subv, false) - case string: - d.literalStore([]byte(qv), subv, true) - default: - d.saveError(fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal unquoted value into %v", subv.Type())) - } - } else { - d.value(subv) - } - - // Write value back to map; - // if using struct, subv points into struct already. - if v.Kind() == reflect.Map { - kt := v.Type().Key() - var kv reflect.Value - switch { - case kt.Kind() == reflect.String: - kv = reflect.ValueOf(key).Convert(v.Type().Key()) - case reflect.PtrTo(kt).Implements(textUnmarshalerType): - kv = reflect.New(v.Type().Key()) - d.literalStore(item, kv, true) - kv = kv.Elem() - default: - panic("json: Unexpected key type") // should never occur - } - v.SetMapIndex(kv, subv) - } - - // Next token must be , or }. - op = d.scanWhile(scanSkipSpace) - if op == scanEndObject { - break - } - if op != scanObjectValue { - d.error(errPhase) - } - } -} - -// isNull returns whether there's a null literal at the provided offset. -func (d *decodeState) isNull(off int) bool { - if off+4 >= len(d.data) || d.data[off] != 'n' || d.data[off+1] != 'u' || d.data[off+2] != 'l' || d.data[off+3] != 'l' { - return false - } - d.nextscan.reset() - for i, c := range d.data[off:] { - if i > 4 { - return false - } - switch d.nextscan.step(&d.nextscan, c) { - case scanContinue, scanBeginName: - continue - } - break - } - return true -} - -// name consumes a const or function from d.data[d.off-1:], decoding into the value v. -// the first byte of the function name has been read already. -func (d *decodeState) name(v reflect.Value) { - if d.isNull(d.off - 1) { - d.literal(v) - return - } - - // Check for unmarshaler. - u, ut, pv := d.indirect(v, false) - if d.storeKeyed(pv) { - return - } - if u != nil { - d.off-- - err := u.UnmarshalJSON(d.next()) - if err != nil { - d.error(err) - } - return - } - if ut != nil { - d.saveError(&UnmarshalTypeError{"object", v.Type(), int64(d.off)}) - d.off-- - d.next() // skip over function in input - return - } - v = pv - - // Decoding into nil interface? Switch to non-reflect code. - if v.Kind() == reflect.Interface && v.NumMethod() == 0 { - out := d.nameInterface() - if out == nil { - v.Set(reflect.Zero(v.Type())) - } else { - v.Set(reflect.ValueOf(out)) - } - return - } - - nameStart := d.off - 1 - - op := d.scanWhile(scanContinue) - - name := d.data[nameStart : d.off-1] - if op != scanParam { - // Back up so the byte just read is consumed next. - d.off-- - d.scan.undo(op) - if l, ok := d.convertLiteral(name); ok { - d.storeValue(v, l) - return - } - d.error(&SyntaxError{fmt.Sprintf("json: unknown constant %q", name), int64(d.off)}) - } - - funcName := string(name) - funcData := d.ext.funcs[funcName] - if funcData.key == "" { - d.error(fmt.Errorf("json: unknown function %q", funcName)) - } - - // Check type of target: - // struct or - // map[string]T or map[encoding.TextUnmarshaler]T - switch v.Kind() { - case reflect.Map: - // Map key must either have string kind or be an encoding.TextUnmarshaler. - t := v.Type() - if t.Key().Kind() != reflect.String && - !reflect.PtrTo(t.Key()).Implements(textUnmarshalerType) { - d.saveError(&UnmarshalTypeError{"object", v.Type(), int64(d.off)}) - d.off-- - d.next() // skip over { } in input - return - } - if v.IsNil() { - v.Set(reflect.MakeMap(t)) - } - case reflect.Struct: - - default: - d.saveError(&UnmarshalTypeError{"object", v.Type(), int64(d.off)}) - d.off-- - d.next() // skip over { } in input - return - } - - // TODO Fix case of func field as map. - //topv := v - - // Figure out field corresponding to function. - key := []byte(funcData.key) - if v.Kind() == reflect.Map { - elemType := v.Type().Elem() - v = reflect.New(elemType).Elem() - } else { - var f *field - fields := cachedTypeFields(v.Type()) - for i := range fields { - ff := &fields[i] - if bytes.Equal(ff.nameBytes, key) { - f = ff - break - } - if f == nil && ff.equalFold(ff.nameBytes, key) { - f = ff - } - } - if f != nil { - for _, i := range f.index { - if v.Kind() == reflect.Ptr { - if v.IsNil() { - v.Set(reflect.New(v.Type().Elem())) - } - v = v.Elem() - } - v = v.Field(i) - } - if v.Kind() == reflect.Ptr { - if v.IsNil() { - v.Set(reflect.New(v.Type().Elem())) - } - v = v.Elem() - } - } - } - - // Check for unmarshaler on func field itself. - u, _, _ = d.indirect(v, false) - if u != nil { - d.off = nameStart - err := u.UnmarshalJSON(d.next()) - if err != nil { - d.error(err) - } - return - } - - var mapElem reflect.Value - - // Parse function arguments. - for i := 0; ; i++ { - // closing ) - can only happen on first iteration. - op := d.scanWhile(scanSkipSpace) - if op == scanEndParams { - break - } - - // Back up so d.value can have the byte we just read. - d.off-- - d.scan.undo(op) - - if i >= len(funcData.args) { - d.error(fmt.Errorf("json: too many arguments for function %s", funcName)) - } - key := []byte(funcData.args[i]) - - // Figure out field corresponding to key. - var subv reflect.Value - destring := false // whether the value is wrapped in a string to be decoded first - - if v.Kind() == reflect.Map { - elemType := v.Type().Elem() - if !mapElem.IsValid() { - mapElem = reflect.New(elemType).Elem() - } else { - mapElem.Set(reflect.Zero(elemType)) - } - subv = mapElem - } else { - var f *field - fields := cachedTypeFields(v.Type()) - for i := range fields { - ff := &fields[i] - if bytes.Equal(ff.nameBytes, key) { - f = ff - break - } - if f == nil && ff.equalFold(ff.nameBytes, key) { - f = ff - } - } - if f != nil { - subv = v - destring = f.quoted - for _, i := range f.index { - if subv.Kind() == reflect.Ptr { - if subv.IsNil() { - subv.Set(reflect.New(subv.Type().Elem())) - } - subv = subv.Elem() - } - subv = subv.Field(i) - } - } - } - - // Read value. - if destring { - switch qv := d.valueQuoted().(type) { - case nil: - d.literalStore(nullLiteral, subv, false) - case string: - d.literalStore([]byte(qv), subv, true) - default: - d.saveError(fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal unquoted value into %v", subv.Type())) - } - } else { - d.value(subv) - } - - // Write value back to map; - // if using struct, subv points into struct already. - if v.Kind() == reflect.Map { - kt := v.Type().Key() - var kv reflect.Value - switch { - case kt.Kind() == reflect.String: - kv = reflect.ValueOf(key).Convert(v.Type().Key()) - case reflect.PtrTo(kt).Implements(textUnmarshalerType): - kv = reflect.New(v.Type().Key()) - d.literalStore(key, kv, true) - kv = kv.Elem() - default: - panic("json: Unexpected key type") // should never occur - } - v.SetMapIndex(kv, subv) - } - - // Next token must be , or ). - op = d.scanWhile(scanSkipSpace) - if op == scanEndParams { - break - } - if op != scanParam { - d.error(errPhase) - } - } -} - -// keyed attempts to decode an object or function using a keyed doc extension, -// and returns the value and true on success, or nil and false otherwise. -func (d *decodeState) keyed() (interface{}, bool) { - if len(d.ext.keyed) == 0 { - return nil, false - } - - unquote := false - - // Look-ahead first key to check for a keyed document extension. - d.nextscan.reset() - var start, end int - for i, c := range d.data[d.off-1:] { - switch op := d.nextscan.step(&d.nextscan, c); op { - case scanSkipSpace, scanContinue, scanBeginObject: - continue - case scanBeginLiteral, scanBeginName: - unquote = op == scanBeginLiteral - start = i - continue - } - end = i - break - } - - name := bytes.Trim(d.data[d.off-1+start:d.off-1+end], " \n\t") - - var key []byte - var ok bool - if unquote { - key, ok = unquoteBytes(name) - if !ok { - d.error(errPhase) - } - } else { - funcData, ok := d.ext.funcs[string(name)] - if !ok { - return nil, false - } - key = []byte(funcData.key) - } - - decode, ok := d.ext.keyed[string(key)] - if !ok { - return nil, false - } - - d.off-- - out, err := decode(d.next()) - if err != nil { - d.error(err) - } - return out, true -} - -func (d *decodeState) storeKeyed(v reflect.Value) bool { - keyed, ok := d.keyed() - if !ok { - return false - } - d.storeValue(v, keyed) - return true -} - -var ( - trueBytes = []byte("true") - falseBytes = []byte("false") - nullBytes = []byte("null") -) - -func (d *decodeState) storeValue(v reflect.Value, from interface{}) { - switch from { - case nil: - d.literalStore(nullBytes, v, false) - return - case true: - d.literalStore(trueBytes, v, false) - return - case false: - d.literalStore(falseBytes, v, false) - return - } - fromv := reflect.ValueOf(from) - for fromv.Kind() == reflect.Ptr && !fromv.IsNil() { - fromv = fromv.Elem() - } - fromt := fromv.Type() - for v.Kind() == reflect.Ptr && !v.IsNil() { - v = v.Elem() - } - vt := v.Type() - if fromt.AssignableTo(vt) { - v.Set(fromv) - } else if fromt.ConvertibleTo(vt) { - v.Set(fromv.Convert(vt)) - } else { - d.saveError(&UnmarshalTypeError{"object", v.Type(), int64(d.off)}) - } -} - -func (d *decodeState) convertLiteral(name []byte) (interface{}, bool) { - if len(name) == 0 { - return nil, false - } - switch name[0] { - case 't': - if bytes.Equal(name, trueBytes) { - return true, true - } - case 'f': - if bytes.Equal(name, falseBytes) { - return false, true - } - case 'n': - if bytes.Equal(name, nullBytes) { - return nil, true - } - } - if l, ok := d.ext.consts[string(name)]; ok { - return l, true - } - return nil, false -} - -// literal consumes a literal from d.data[d.off-1:], decoding into the value v. -// The first byte of the literal has been read already -// (that's how the caller knows it's a literal). -func (d *decodeState) literal(v reflect.Value) { - // All bytes inside literal return scanContinue op code. - start := d.off - 1 - op := d.scanWhile(scanContinue) - - // Scan read one byte too far; back up. - d.off-- - d.scan.undo(op) - - d.literalStore(d.data[start:d.off], v, false) -} - -// convertNumber converts the number literal s to a float64 or a Number -// depending on the setting of d.useNumber. -func (d *decodeState) convertNumber(s string) (interface{}, error) { - if d.useNumber { - return Number(s), nil - } - f, err := strconv.ParseFloat(s, 64) - if err != nil { - return nil, &UnmarshalTypeError{"number " + s, reflect.TypeOf(0.0), int64(d.off)} - } - return f, nil -} - -var numberType = reflect.TypeOf(Number("")) - -// literalStore decodes a literal stored in item into v. -// -// fromQuoted indicates whether this literal came from unwrapping a -// string from the ",string" struct tag option. this is used only to -// produce more helpful error messages. -func (d *decodeState) literalStore(item []byte, v reflect.Value, fromQuoted bool) { - // Check for unmarshaler. - if len(item) == 0 { - //Empty string given - d.saveError(fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type())) - return - } - wantptr := item[0] == 'n' // null - u, ut, pv := d.indirect(v, wantptr) - if u != nil { - err := u.UnmarshalJSON(item) - if err != nil { - d.error(err) - } - return - } - if ut != nil { - if item[0] != '"' { - if fromQuoted { - d.saveError(fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type())) - } else { - d.saveError(&UnmarshalTypeError{"string", v.Type(), int64(d.off)}) - } - return - } - s, ok := unquoteBytes(item) - if !ok { - if fromQuoted { - d.error(fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type())) - } else { - d.error(errPhase) - } - } - err := ut.UnmarshalText(s) - if err != nil { - d.error(err) - } - return - } - - v = pv - - switch c := item[0]; c { - case 'n': // null - switch v.Kind() { - case reflect.Interface, reflect.Ptr, reflect.Map, reflect.Slice: - v.Set(reflect.Zero(v.Type())) - // otherwise, ignore null for primitives/string - } - case 't', 'f': // true, false - value := c == 't' - switch v.Kind() { - default: - if fromQuoted { - d.saveError(fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type())) - } else { - d.saveError(&UnmarshalTypeError{"bool", v.Type(), int64(d.off)}) - } - case reflect.Bool: - v.SetBool(value) - case reflect.Interface: - if v.NumMethod() == 0 { - v.Set(reflect.ValueOf(value)) - } else { - d.saveError(&UnmarshalTypeError{"bool", v.Type(), int64(d.off)}) - } - } - - case '"': // string - s, ok := unquoteBytes(item) - if !ok { - if fromQuoted { - d.error(fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type())) - } else { - d.error(errPhase) - } - } - switch v.Kind() { - default: - d.saveError(&UnmarshalTypeError{"string", v.Type(), int64(d.off)}) - case reflect.Slice: - if v.Type().Elem().Kind() != reflect.Uint8 { - d.saveError(&UnmarshalTypeError{"string", v.Type(), int64(d.off)}) - break - } - b := make([]byte, base64.StdEncoding.DecodedLen(len(s))) - n, err := base64.StdEncoding.Decode(b, s) - if err != nil { - d.saveError(err) - break - } - v.SetBytes(b[:n]) - case reflect.String: - v.SetString(string(s)) - case reflect.Interface: - if v.NumMethod() == 0 { - v.Set(reflect.ValueOf(string(s))) - } else { - d.saveError(&UnmarshalTypeError{"string", v.Type(), int64(d.off)}) - } - } - - default: // number - if c != '-' && (c < '0' || c > '9') { - if fromQuoted { - d.error(fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type())) - } else { - d.error(errPhase) - } - } - s := string(item) - switch v.Kind() { - default: - if v.Kind() == reflect.String && v.Type() == numberType { - v.SetString(s) - if !isValidNumber(s) { - d.error(fmt.Errorf("json: invalid number literal, trying to unmarshal %q into Number", item)) - } - break - } - if fromQuoted { - d.error(fmt.Errorf("json: invalid use of ,string struct tag, trying to unmarshal %q into %v", item, v.Type())) - } else { - d.error(&UnmarshalTypeError{"number", v.Type(), int64(d.off)}) - } - case reflect.Interface: - n, err := d.convertNumber(s) - if err != nil { - d.saveError(err) - break - } - if v.NumMethod() != 0 { - d.saveError(&UnmarshalTypeError{"number", v.Type(), int64(d.off)}) - break - } - v.Set(reflect.ValueOf(n)) - - case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: - n, err := strconv.ParseInt(s, 10, 64) - if err != nil || v.OverflowInt(n) { - d.saveError(&UnmarshalTypeError{"number " + s, v.Type(), int64(d.off)}) - break - } - v.SetInt(n) - - case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: - n, err := strconv.ParseUint(s, 10, 64) - if err != nil || v.OverflowUint(n) { - d.saveError(&UnmarshalTypeError{"number " + s, v.Type(), int64(d.off)}) - break - } - v.SetUint(n) - - case reflect.Float32, reflect.Float64: - n, err := strconv.ParseFloat(s, v.Type().Bits()) - if err != nil || v.OverflowFloat(n) { - d.saveError(&UnmarshalTypeError{"number " + s, v.Type(), int64(d.off)}) - break - } - v.SetFloat(n) - } - } -} - -// The xxxInterface routines build up a value to be stored -// in an empty interface. They are not strictly necessary, -// but they avoid the weight of reflection in this common case. - -// valueInterface is like value but returns interface{} -func (d *decodeState) valueInterface() interface{} { - switch d.scanWhile(scanSkipSpace) { - default: - d.error(errPhase) - panic("unreachable") - case scanBeginArray: - return d.arrayInterface() - case scanBeginObject: - return d.objectInterface() - case scanBeginLiteral: - return d.literalInterface() - case scanBeginName: - return d.nameInterface() - } -} - -func (d *decodeState) syntaxError(expected string) { - msg := fmt.Sprintf("invalid character '%c' looking for %s", d.data[d.off-1], expected) - d.error(&SyntaxError{msg, int64(d.off)}) -} - -// arrayInterface is like array but returns []interface{}. -func (d *decodeState) arrayInterface() []interface{} { - var v = make([]interface{}, 0) - for { - // Look ahead for ] - can only happen on first iteration. - op := d.scanWhile(scanSkipSpace) - if op == scanEndArray { - if len(v) > 0 && !d.ext.trailingCommas { - d.syntaxError("beginning of value") - } - break - } - - // Back up so d.value can have the byte we just read. - d.off-- - d.scan.undo(op) - - v = append(v, d.valueInterface()) - - // Next token must be , or ]. - op = d.scanWhile(scanSkipSpace) - if op == scanEndArray { - break - } - if op != scanArrayValue { - d.error(errPhase) - } - } - return v -} - -// objectInterface is like object but returns map[string]interface{}. -func (d *decodeState) objectInterface() interface{} { - v, ok := d.keyed() - if ok { - return v - } - - m := make(map[string]interface{}) - for { - // Read opening " of string key or closing }. - op := d.scanWhile(scanSkipSpace) - if op == scanEndObject { - if len(m) > 0 && !d.ext.trailingCommas { - d.syntaxError("beginning of object key string") - } - break - } - if op == scanBeginName { - if !d.ext.unquotedKeys { - d.syntaxError("beginning of object key string") - } - } else if op != scanBeginLiteral { - d.error(errPhase) - } - unquotedKey := op == scanBeginName - - // Read string key. - start := d.off - 1 - op = d.scanWhile(scanContinue) - item := d.data[start : d.off-1] - var key string - if unquotedKey { - key = string(item) - } else { - var ok bool - key, ok = unquote(item) - if !ok { - d.error(errPhase) - } - } - - // Read : before value. - if op == scanSkipSpace { - op = d.scanWhile(scanSkipSpace) - } - if op != scanObjectKey { - d.error(errPhase) - } - - // Read value. - m[key] = d.valueInterface() - - // Next token must be , or }. - op = d.scanWhile(scanSkipSpace) - if op == scanEndObject { - break - } - if op != scanObjectValue { - d.error(errPhase) - } - } - return m -} - -// literalInterface is like literal but returns an interface value. -func (d *decodeState) literalInterface() interface{} { - // All bytes inside literal return scanContinue op code. - start := d.off - 1 - op := d.scanWhile(scanContinue) - - // Scan read one byte too far; back up. - d.off-- - d.scan.undo(op) - item := d.data[start:d.off] - - switch c := item[0]; c { - case 'n': // null - return nil - - case 't', 'f': // true, false - return c == 't' - - case '"': // string - s, ok := unquote(item) - if !ok { - d.error(errPhase) - } - return s - - default: // number - if c != '-' && (c < '0' || c > '9') { - d.error(errPhase) - } - n, err := d.convertNumber(string(item)) - if err != nil { - d.saveError(err) - } - return n - } -} - -// nameInterface is like function but returns map[string]interface{}. -func (d *decodeState) nameInterface() interface{} { - v, ok := d.keyed() - if ok { - return v - } - - nameStart := d.off - 1 - - op := d.scanWhile(scanContinue) - - name := d.data[nameStart : d.off-1] - if op != scanParam { - // Back up so the byte just read is consumed next. - d.off-- - d.scan.undo(op) - if l, ok := d.convertLiteral(name); ok { - return l - } - d.error(&SyntaxError{fmt.Sprintf("json: unknown constant %q", name), int64(d.off)}) - } - - funcName := string(name) - funcData := d.ext.funcs[funcName] - if funcData.key == "" { - d.error(fmt.Errorf("json: unknown function %q", funcName)) - } - - m := make(map[string]interface{}) - for i := 0; ; i++ { - // Look ahead for ) - can only happen on first iteration. - op := d.scanWhile(scanSkipSpace) - if op == scanEndParams { - break - } - - // Back up so d.value can have the byte we just read. - d.off-- - d.scan.undo(op) - - if i >= len(funcData.args) { - d.error(fmt.Errorf("json: too many arguments for function %s", funcName)) - } - m[funcData.args[i]] = d.valueInterface() - - // Next token must be , or ). - op = d.scanWhile(scanSkipSpace) - if op == scanEndParams { - break - } - if op != scanParam { - d.error(errPhase) - } - } - return map[string]interface{}{funcData.key: m} -} - -// getu4 decodes \uXXXX from the beginning of s, returning the hex value, -// or it returns -1. -func getu4(s []byte) rune { - if len(s) < 6 || s[0] != '\\' || s[1] != 'u' { - return -1 - } - r, err := strconv.ParseUint(string(s[2:6]), 16, 64) - if err != nil { - return -1 - } - return rune(r) -} - -// unquote converts a quoted JSON string literal s into an actual string t. -// The rules are different than for Go, so cannot use strconv.Unquote. -func unquote(s []byte) (t string, ok bool) { - s, ok = unquoteBytes(s) - t = string(s) - return -} - -func unquoteBytes(s []byte) (t []byte, ok bool) { - if len(s) < 2 || s[0] != '"' || s[len(s)-1] != '"' { - return - } - s = s[1 : len(s)-1] - - // Check for unusual characters. If there are none, - // then no unquoting is needed, so return a slice of the - // original bytes. - r := 0 - for r < len(s) { - c := s[r] - if c == '\\' || c == '"' || c < ' ' { - break - } - if c < utf8.RuneSelf { - r++ - continue - } - rr, size := utf8.DecodeRune(s[r:]) - if rr == utf8.RuneError && size == 1 { - break - } - r += size - } - if r == len(s) { - return s, true - } - - b := make([]byte, len(s)+2*utf8.UTFMax) - w := copy(b, s[0:r]) - for r < len(s) { - // Out of room? Can only happen if s is full of - // malformed UTF-8 and we're replacing each - // byte with RuneError. - if w >= len(b)-2*utf8.UTFMax { - nb := make([]byte, (len(b)+utf8.UTFMax)*2) - copy(nb, b[0:w]) - b = nb - } - switch c := s[r]; { - case c == '\\': - r++ - if r >= len(s) { - return - } - switch s[r] { - default: - return - case '"', '\\', '/', '\'': - b[w] = s[r] - r++ - w++ - case 'b': - b[w] = '\b' - r++ - w++ - case 'f': - b[w] = '\f' - r++ - w++ - case 'n': - b[w] = '\n' - r++ - w++ - case 'r': - b[w] = '\r' - r++ - w++ - case 't': - b[w] = '\t' - r++ - w++ - case 'u': - r-- - rr := getu4(s[r:]) - if rr < 0 { - return - } - r += 6 - if utf16.IsSurrogate(rr) { - rr1 := getu4(s[r:]) - if dec := utf16.DecodeRune(rr, rr1); dec != unicode.ReplacementChar { - // A valid pair; consume. - r += 6 - w += utf8.EncodeRune(b[w:], dec) - break - } - // Invalid surrogate; fall back to replacement rune. - rr = unicode.ReplacementChar - } - w += utf8.EncodeRune(b[w:], rr) - } - - // Quote, control characters are invalid. - case c == '"', c < ' ': - return - - // ASCII - case c < utf8.RuneSelf: - b[w] = c - r++ - w++ - - // Coerce to well-formed UTF-8. - default: - rr, size := utf8.DecodeRune(s[r:]) - r += size - w += utf8.EncodeRune(b[w:], rr) - } - } - return b[0:w], true -} diff --git a/vendor/github.com/globalsign/mgo/internal/json/encode.go b/vendor/github.com/globalsign/mgo/internal/json/encode.go deleted file mode 100644 index e4b8f864..00000000 --- a/vendor/github.com/globalsign/mgo/internal/json/encode.go +++ /dev/null @@ -1,1260 +0,0 @@ -// Copyright 2010 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Package json implements encoding and decoding of JSON as defined in -// RFC 4627. The mapping between JSON and Go values is described -// in the documentation for the Marshal and Unmarshal functions. -// -// See "JSON and Go" for an introduction to this package: -// https://golang.org/doc/articles/json_and_go.html -package json - -import ( - "bytes" - "encoding" - "encoding/base64" - "fmt" - "math" - "reflect" - "runtime" - "sort" - "strconv" - "strings" - "sync" - "unicode" - "unicode/utf8" -) - -// Marshal returns the JSON encoding of v. -// -// Marshal traverses the value v recursively. -// If an encountered value implements the Marshaler interface -// and is not a nil pointer, Marshal calls its MarshalJSON method -// to produce JSON. If no MarshalJSON method is present but the -// value implements encoding.TextMarshaler instead, Marshal calls -// its MarshalText method. -// The nil pointer exception is not strictly necessary -// but mimics a similar, necessary exception in the behavior of -// UnmarshalJSON. -// -// Otherwise, Marshal uses the following type-dependent default encodings: -// -// Boolean values encode as JSON booleans. -// -// Floating point, integer, and Number values encode as JSON numbers. -// -// String values encode as JSON strings coerced to valid UTF-8, -// replacing invalid bytes with the Unicode replacement rune. -// The angle brackets "<" and ">" are escaped to "\u003c" and "\u003e" -// to keep some browsers from misinterpreting JSON output as HTML. -// Ampersand "&" is also escaped to "\u0026" for the same reason. -// This escaping can be disabled using an Encoder with DisableHTMLEscaping. -// -// Array and slice values encode as JSON arrays, except that -// []byte encodes as a base64-encoded string, and a nil slice -// encodes as the null JSON value. -// -// Struct values encode as JSON objects. Each exported struct field -// becomes a member of the object unless -// - the field's tag is "-", or -// - the field is empty and its tag specifies the "omitempty" option. -// The empty values are false, 0, any -// nil pointer or interface value, and any array, slice, map, or string of -// length zero. The object's default key string is the struct field name -// but can be specified in the struct field's tag value. The "json" key in -// the struct field's tag value is the key name, followed by an optional comma -// and options. Examples: -// -// // Field is ignored by this package. -// Field int `json:"-"` -// -// // Field appears in JSON as key "myName". -// Field int `json:"myName"` -// -// // Field appears in JSON as key "myName" and -// // the field is omitted from the object if its value is empty, -// // as defined above. -// Field int `json:"myName,omitempty"` -// -// // Field appears in JSON as key "Field" (the default), but -// // the field is skipped if empty. -// // Note the leading comma. -// Field int `json:",omitempty"` -// -// The "string" option signals that a field is stored as JSON inside a -// JSON-encoded string. It applies only to fields of string, floating point, -// integer, or boolean types. This extra level of encoding is sometimes used -// when communicating with JavaScript programs: -// -// Int64String int64 `json:",string"` -// -// The key name will be used if it's a non-empty string consisting of -// only Unicode letters, digits, dollar signs, percent signs, hyphens, -// underscores and slashes. -// -// Anonymous struct fields are usually marshaled as if their inner exported fields -// were fields in the outer struct, subject to the usual Go visibility rules amended -// as described in the next paragraph. -// An anonymous struct field with a name given in its JSON tag is treated as -// having that name, rather than being anonymous. -// An anonymous struct field of interface type is treated the same as having -// that type as its name, rather than being anonymous. -// -// The Go visibility rules for struct fields are amended for JSON when -// deciding which field to marshal or unmarshal. If there are -// multiple fields at the same level, and that level is the least -// nested (and would therefore be the nesting level selected by the -// usual Go rules), the following extra rules apply: -// -// 1) Of those fields, if any are JSON-tagged, only tagged fields are considered, -// even if there are multiple untagged fields that would otherwise conflict. -// 2) If there is exactly one field (tagged or not according to the first rule), that is selected. -// 3) Otherwise there are multiple fields, and all are ignored; no error occurs. -// -// Handling of anonymous struct fields is new in Go 1.1. -// Prior to Go 1.1, anonymous struct fields were ignored. To force ignoring of -// an anonymous struct field in both current and earlier versions, give the field -// a JSON tag of "-". -// -// Map values encode as JSON objects. The map's key type must either be a string -// or implement encoding.TextMarshaler. The map keys are used as JSON object -// keys, subject to the UTF-8 coercion described for string values above. -// -// Pointer values encode as the value pointed to. -// A nil pointer encodes as the null JSON value. -// -// Interface values encode as the value contained in the interface. -// A nil interface value encodes as the null JSON value. -// -// Channel, complex, and function values cannot be encoded in JSON. -// Attempting to encode such a value causes Marshal to return -// an UnsupportedTypeError. -// -// JSON cannot represent cyclic data structures and Marshal does not -// handle them. Passing cyclic structures to Marshal will result in -// an infinite recursion. -// -func Marshal(v interface{}) ([]byte, error) { - e := &encodeState{} - err := e.marshal(v, encOpts{escapeHTML: true}) - if err != nil { - return nil, err - } - return e.Bytes(), nil -} - -// MarshalIndent is like Marshal but applies Indent to format the output. -func MarshalIndent(v interface{}, prefix, indent string) ([]byte, error) { - b, err := Marshal(v) - if err != nil { - return nil, err - } - var buf bytes.Buffer - err = Indent(&buf, b, prefix, indent) - if err != nil { - return nil, err - } - return buf.Bytes(), nil -} - -// HTMLEscape appends to dst the JSON-encoded src with <, >, &, U+2028 and U+2029 -// characters inside string literals changed to \u003c, \u003e, \u0026, \u2028, \u2029 -// so that the JSON will be safe to embed inside HTML