From 1684b9ed25fcd8cb460570c165745758480f68f4 Mon Sep 17 00:00:00 2001 From: Jason Joo Date: Tue, 30 Jun 2020 17:53:10 +0800 Subject: [PATCH 01/46] Add test for services/meta/config Add ignore files Introduce testify --- .gitignore | 2 ++ go.mod | 1 + go.sum | 8 ++++++++ services/controller/shard.go | 4 +--- services/meta/config_test.go | 20 ++++++++++++++++++++ 5 files changed, 32 insertions(+), 3 deletions(-) create mode 100644 .gitignore create mode 100644 services/meta/config_test.go diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..1277a92 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +.history/ +.DS_Store diff --git a/go.mod b/go.mod index d691a0a..93cb196 100644 --- a/go.mod +++ b/go.mod @@ -25,6 +25,7 @@ require ( github.com/pkg/errors v0.8.0 github.com/pkg/profile v1.2.1 // indirect github.com/spf13/cobra v0.0.3 + github.com/stretchr/testify v1.5.1 // test github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca // indirect go.opencensus.io v0.19.0 // indirect go.uber.org/zap v1.9.1 diff --git a/go.sum b/go.sum index 4e9ab42..8d77905 100644 --- a/go.sum +++ b/go.sum @@ -62,6 +62,7 @@ github.com/coreos/bbolt v1.3.1-coreos.6/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE55 github.com/coreos/etcd v3.3.12+incompatible h1:pAWNwdf7QiT1zfaWyqCtNZQWCLByQyA3JrSQyuYAqnQ= github.com/coreos/etcd v3.3.12+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/denisenkom/go-mssqldb v0.0.0-20181014144952-4e0d7dc8888f/go.mod h1:xN/JuLBIz4bjkxNmByTiV1IbhfnYb6oo99phBn4Eqhc= github.com/dgraph-io/badger v1.5.5-0.20181126210712-49a49e321746 h1:cd1s0THXu7KWnykg7V07VSiTde/g38d8PJBTXBaq0mk= @@ -189,8 +190,11 @@ github.com/kevinburke/ssh_config v0.0.0-20180830205328-81db2a75821e/go.mod h1:CT github.com/keybase/go-crypto v0.0.0-20181031135447-f919bfda4fc1/go.mod h1:ghbZscTyKdM07+Fw3KSi0hcJm+AlEUWj8QLlPtijN/M= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.4.1 h1:8VMb5+0wMgdBykOV96DwNwKFQ+WTI4pzYURP99CcB9E= github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/cpuid v1.2.0 h1:NMpwD2G9JSFOE1/TJjGSo5zG7Yb2bTe7eq1jH+irmeE= github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= +github.com/klauspost/pgzip v1.2.1 h1:oIPZROsWuPHpOdMVWLuJZXwgjhrW8r1yEX8UqMyeNHM= github.com/klauspost/pgzip v1.2.1/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -249,6 +253,7 @@ github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pkg/profile v1.2.1 h1:F++O52m40owAmADcojzM+9gyjmMOY/T4oYJkgFDH8RE= github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA= github.com/pkg/term v0.0.0-20180730021639-bffc007b7fd5/go.mod h1:eCbImbZ95eXtAUIbLAuAVnBnwf83mjf6QIVH8SHYwqQ= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v0.0.0-20171201122222-661e31bf844d/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= @@ -290,10 +295,13 @@ github.com/spf13/viper v1.2.1 h1:bIcUwXqLseLF3BDAZduuNfekWG87ibtFxi59Bq+oI9M= github.com/spf13/viper v1.2.1/go.mod h1:P4AexN0a+C9tGAnUFNwDMYYZv3pjFuvmeiMyKRaNVlI= github.com/src-d/gcfg v1.4.0/go.mod h1:p/UMsR43ujA89BJY9duynAwIpvqEujIH/jFlfL7jWoI= github.com/stevvooe/resumable v0.0.0-20180830230917-22b14a53ba50/go.mod h1:1pdIZTAHUz+HDKDVZ++5xg/duPlhKAIzw9qy42CWYp4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.0/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.1/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/tcnksm/go-input v0.0.0-20180404061846-548a7d7a8ee8/go.mod h1:IlWNj9v/13q7xFbaK4mbyzMNwrZLaWSHx/aibKIZuIg= github.com/testcontainers/testcontainer-go v0.0.0-20181115231424-8e868ca12c0f/go.mod h1:SrG3IY071gtmZJjGbKO+POJ57a/MMESerYNWt6ZRtKs= github.com/tinylib/msgp v1.0.2 h1:DfdQrzQa7Yh2es9SuLkixqxuXS2SxsdYn0KbdrOGWD8= diff --git a/services/controller/shard.go b/services/controller/shard.go index 3a9f316..6c9ed87 100644 --- a/services/controller/shard.go +++ b/services/controller/shard.go @@ -31,9 +31,7 @@ type ShardCopyTask struct { shardId uint64 source string destination string - closer interface { - Close() error - } + closer io.Closer } type ShardCopyManager struct { diff --git a/services/meta/config_test.go b/services/meta/config_test.go new file mode 100644 index 0000000..318c3a5 --- /dev/null +++ b/services/meta/config_test.go @@ -0,0 +1,20 @@ +package meta + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestConfig(t *testing.T) { + cfg := NewConfig() + assert.NotNil(t, cfg.Validate()) + cfg.Dir = "some_value" + assert.Nil(t, cfg.Validate()) + + diag, err := cfg.Diagnostics() + assert.Nil(t, err) + assert.Equal(t, 1, len(diag.Rows)) + assert.Equal(t, 1, len(diag.Rows[0])) + assert.Equal(t, "some_value", diag.Rows[0][0]) +} From 578c9a6f7f2be0e029e14a0ace9943ab82f728fa Mon Sep 17 00:00:00 2001 From: Jason Joo Date: Wed, 8 Jul 2020 15:31:52 +0800 Subject: [PATCH 02/46] Refactor migration --- coordinator/cluster_meta_client.go | 5 +- go.mod | 11 +- go.sum | 35 ++++ services/controller/service.go | 43 +++-- services/controller/shard.go | 290 +++-------------------------- services/hh/config_test.go | 9 +- services/hh/limiter.go | 61 ------ services/hh/limiter_test.go | 47 ----- services/hh/node_processor.go | 16 +- services/hh/node_processor_test.go | 6 +- services/meta/client.go | 54 +++--- services/meta/data.go | 22 ++- services/meta/data_test.go | 189 +++++++++++-------- services/migrate/executor.go | 177 ++++++++++++++++++ services/migrate/executor_test.go | 61 ++++++ services/migrate/manager.go | 206 ++++++++++++++++++++ services/migrate/manager_test.go | 82 ++++++++ x/io.go | 29 +++ x/io_test.go | 26 +++ 19 files changed, 843 insertions(+), 526 deletions(-) delete mode 100644 services/hh/limiter.go delete mode 100644 services/hh/limiter_test.go create mode 100644 services/migrate/executor.go create mode 100644 services/migrate/executor_test.go create mode 100644 services/migrate/manager.go create mode 100644 services/migrate/manager_test.go create mode 100644 x/io.go create mode 100644 x/io_test.go diff --git a/coordinator/cluster_meta_client.go b/coordinator/cluster_meta_client.go index ab15f22..6b3f82c 100644 --- a/coordinator/cluster_meta_client.go +++ b/coordinator/cluster_meta_client.go @@ -2,10 +2,11 @@ package coordinator import ( "fmt" + "time" + "github.com/influxdata/influxdb/services/meta" "github.com/influxdata/influxql" "go.uber.org/zap" - "time" imeta "github.com/angopher/chronus/services/meta" ) @@ -164,7 +165,7 @@ func (me *ClusterMetaClient) DataNode(id uint64) (*meta.NodeInfo, error) { } func (me *ClusterMetaClient) DataNodes() ([]meta.NodeInfo, error) { - return me.cache.DataNodes() + return me.cache.DataNodes(), nil } func (me *ClusterMetaClient) ShardOwner(id uint64) (string, string, *meta.ShardGroupInfo) { diff --git a/go.mod b/go.mod index 93cb196..48b6a57 100644 --- a/go.mod +++ b/go.mod @@ -22,18 +22,17 @@ require ( github.com/klauspost/compress v1.4.1 // indirect github.com/klauspost/cpuid v1.2.0 // indirect github.com/klauspost/pgzip v1.2.1 - github.com/pkg/errors v0.8.0 + github.com/pkg/errors v0.8.1 github.com/pkg/profile v1.2.1 // indirect github.com/spf13/cobra v0.0.3 github.com/stretchr/testify v1.5.1 // test github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca // indirect go.opencensus.io v0.19.0 // indirect - go.uber.org/zap v1.9.1 - golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9 - golang.org/x/net v0.0.0-20181217023233-e147a9138326 - golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 // indirect + go.uber.org/zap v1.15.0 + golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529 + golang.org/x/net v0.0.0-20190620200207-3b0461eec859 golang.org/x/text v0.3.0 - golang.org/x/time v0.0.0-20181108054448-85acf8d2951c // indirect + golang.org/x/time v0.0.0-20181108054448-85acf8d2951c gopkg.in/fatih/pool.v2 v2.0.0 labix.org/v2/mgo v0.0.0-20140701140051-000000000287 // indirect launchpad.net/gocheck v0.0.0-20140225173054-000000000087 // indirect diff --git a/go.sum b/go.sum index 8d77905..aa0e0e9 100644 --- a/go.sum +++ b/go.sum @@ -120,6 +120,7 @@ github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/goreleaser/goreleaser v0.94.0/go.mod h1:OjbYR2NhOI6AEUWCowMSBzo9nP1aRif3sYtx+rhp+Zo= @@ -161,6 +162,7 @@ github.com/influxdata/flux v0.13.0 h1:iWNTxfR8m8LFYNAjSXAYMdxMKHqsBxd3xHlZ7rpl1K github.com/influxdata/flux v0.13.0/go.mod h1:81jeDcHVn1rN5uj9aQ81S72Q8ol8If7N0zM0G8TnxTE= github.com/influxdata/influxdb v1.7.4 h1:Ufqfn5xFixUXXj5Fgmhfa9RSke2R2AOvUOXfxgp9SCA= github.com/influxdata/influxdb v1.7.4/go.mod h1:qZna6X/4elxqT3yI9iZYdZrWWdeFOOprn86kgg4+IzY= +github.com/influxdata/influxdb v1.8.0 h1:/X+G+i3udzHVxpBMuXdPZcUbkIE0ouT+6U+CzQTsOys= github.com/influxdata/influxql v0.0.0-20180925231337-1cbfca8e56b6 h1:CFx+pP90q/qg3spoiZjf8donE4WpAdjeJfPOcoNqkWo= github.com/influxdata/influxql v0.0.0-20180925231337-1cbfca8e56b6/go.mod h1:KpVI7okXjK6PRi3Z5B+mtKZli+R1DnZgb3N+tzevNgo= github.com/influxdata/line-protocol v0.0.0-20180522152040-32c6aa80de5e h1:/o3vQtpWJhvnIbXley4/jwzzqNeigJK9z+LZcJZ9zfM= @@ -250,6 +252,8 @@ github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/profile v1.2.1 h1:F++O52m40owAmADcojzM+9gyjmMOY/T4oYJkgFDH8RE= github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA= github.com/pkg/term v0.0.0-20180730021639-bffc007b7fd5/go.mod h1:eCbImbZ95eXtAUIbLAuAVnBnwf83mjf6QIVH8SHYwqQ= @@ -269,6 +273,7 @@ github.com/prometheus/common v0.0.0-20181218105931-67670fe90761/go.mod h1:daVV7q github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a h1:9a8MnZMP0X2nLJdBg+pBmGgkJlSaKC2KaQmTCk1XDtE= github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/ryanuber/go-glob v0.0.0-20170128012129-256dc444b735/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc= github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= @@ -300,6 +305,8 @@ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/testify v1.2.0/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.1/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/tcnksm/go-input v0.0.0-20180404061846-548a7d7a8ee8/go.mod h1:IlWNj9v/13q7xFbaK4mbyzMNwrZLaWSHx/aibKIZuIg= @@ -319,19 +326,31 @@ go.opencensus.io v0.19.0 h1:+jrnNy8MR4GZXvwF9PEuSyHxA4NaTf6601oNRwCSXq0= go.opencensus.io v0.19.0/go.mod h1:AYeH0+ZxYyghG8diqaaIq/9P3VgCCt5GF2ldCY4dkFg= go.uber.org/atomic v1.3.2 h1:2Oa65PReHzfn29GpvgsYwloV9AVFHPDk8tYxt2c2tr4= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.6.0 h1:Ezj3JGmsOnG1MoRWQkPBsKLe9DwWD9QeXzTRzzldNVk= +go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/multierr v1.5.0 h1:KCa4XfM8CWFCpxXRGok+Q0SS/0XBhMDbHHGABQLvD2A= +go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= +go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v1.9.1 h1:XCJQEf3W6eZaVwhRBof6ImoYGJSITeKWsyeh3HFu/5o= go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.15.0 h1:ZZCA22JRF2gQE5FoNmhmrf7jeJJ2uhqDUNRYKm8dvmM= +go.uber.org/zap v1.15.0/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= golang.org/x/crypto v0.0.0-20180505025534-4ec37c66abab/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9 h1:mKdxBk7AujPs8kU4m80U72y/zjbZ3UcXC7dClwKbUI0= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529 h1:iMGN4xG0cnqj3t+zOM8wUB0BiPKHEwSxEZCvzcbZuvk= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20181112044915-a3060d491354/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20181217174547-8f45f776aaf1/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -340,6 +359,10 @@ golang.org/x/net v0.0.0-20181106065722-10aee1819953/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181217023233-e147a9138326 h1:iCzOf0xz39Tstp+Tu/WwyGjUXCk34QhQORRxBeXXTA4= golang.org/x/net v0.0.0-20181217023233-e147a9138326/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -347,6 +370,8 @@ golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 h1:YUO/7uOKsKeq9UokNS62b8FYywz3ker1l1vDZRCRefw= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180903190138-2b024373dcd9/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -356,6 +381,9 @@ golang.org/x/sys v0.0.0-20181030150119-7e31e0c00fa0/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181218192612-074acd46bca6 h1:MXtOG7w2ND9qNCUZSDBGll/SpVIq7ftozR9I8/JGBHY= golang.org/x/sys v0.0.0-20181218192612-074acd46bca6/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d h1:+R4KGOnez64A81RvjARKc4UT5/tI9ujCIVX+P5KiHuI= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -366,6 +394,11 @@ golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181219222714-6e267b5cc78e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181221154417-3ad2d988d5e2/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gonum.org/v1/gonum v0.0.0-20181121035319-3f7ecaa7e8ca/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= gonum.org/v1/netlib v0.0.0-20181029234149-ec6d1f5cefe6/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= google.golang.org/api v0.0.0-20181021000519-a2651947f503/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= @@ -385,6 +418,7 @@ google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3 gopkg.in/asn1-ber.v1 v1.0.0-20170511165959-379148ca0225/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fatih/pool.v2 v2.0.0 h1:xIFeWtxifuQJGk/IEPKsTduEKcKvPmhoiVDGpC40nKg= gopkg.in/fatih/pool.v2 v2.0.0/go.mod h1:8xVGeu1/2jr2wm5V9SPuMht2H5AEmf5aFMGSQixtjTY= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= @@ -405,5 +439,6 @@ gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81 honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20180920025451-e3ad64cb4ed3/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20181108184350-ae8f1f9103cc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= labix.org/v2/mgo v0.0.0-20140701140051-000000000287/go.mod h1:Lg7AYkt1uXJoR9oeSZ3W/8IXLdvOfIITgZnommstyz4= launchpad.net/gocheck v0.0.0-20140225173054-000000000087/go.mod h1:hj7XX3B/0A+80Vse0e+BUHsHMTEhd0O4cpUHr/e/BUM= diff --git a/services/controller/service.go b/services/controller/service.go index 9f1b52f..53aabb5 100644 --- a/services/controller/service.go +++ b/services/controller/service.go @@ -12,9 +12,11 @@ import ( "github.com/influxdata/influxdb" "github.com/influxdata/influxdb/services/meta" + "github.com/influxdata/influxdb/tsdb" "go.uber.org/zap" "github.com/angopher/chronus/coordinator" + "github.com/angopher/chronus/services/migrate" ) const ( @@ -33,26 +35,30 @@ type Service struct { DataNodeByTCPHost(addr string) (*meta.NodeInfo, error) RemoveShardOwner(shardID, nodeID uint64) error DataNodes() ([]meta.NodeInfo, error) + + ShardOwner(shardID uint64) (database, policy string, sgi *meta.ShardGroupInfo) + AddShardOwner(shardID, nodeID uint64) error } TSDBStore interface { + Path() string + ShardRelativePath(id uint64) (string, error) + CreateShard(database, retentionPolicy string, shardID uint64, enabled bool) error DeleteShard(id uint64) error + Shard(id uint64) *tsdb.Shard } Listener net.Listener Logger *zap.Logger - ShardCopier interface { - CopyShard(sourceAddr string, shardId uint64) error - Query() []CopyShardTask - Kill(shardId uint64, source, destination string) - } + migrateManager *migrate.Manager } // NewService returns a new instance of Service. func NewService(c Config) *Service { return &Service{ - Logger: zap.NewNop(), + Logger: zap.NewNop(), + migrateManager: migrate.NewManager(c.MaxShardCopyTasks), } } @@ -199,7 +205,7 @@ func (s *Service) handleCopyShard(conn net.Conn) error { return err } - s.ShardCopier.CopyShard(req.SourceNodeAddr, req.ShardID) + s.copyShard(req.SourceNodeAddr, req.ShardID) return nil } @@ -227,8 +233,23 @@ func (s *Service) copyShardResponse(w io.Writer, e error) { } } +func toCopyTask(t *migrate.Task) CopyShardTask { + task := CopyShardTask{} + task.CurrentSize = t.Copied + task.Database = t.Database + task.Rp = t.Retention + task.ShardID = t.ShardId + task.Source = t.SrcHost + return task +} + func (s *Service) handleCopyShardStatus(conn net.Conn) []CopyShardTask { - return s.ShardCopier.Query() + tasks := s.migrateManager.Tasks() + result := make([]CopyShardTask, len(tasks)) + for i, t := range tasks { + result[i] = toCopyTask(t) + } + return result } func (s *Service) copyShardStatusResponse(w io.Writer, tasks []CopyShardTask) { @@ -263,7 +284,7 @@ func (s *Service) handleKillCopyShard(conn net.Conn) error { return err } - s.ShardCopier.Kill(req.ShardID, req.SourceNodeAddr, req.DestNodeAddr) + s.migrateManager.Kill(req.ShardID, req.SourceNodeAddr) return nil } @@ -451,7 +472,7 @@ type TruncateShardResponse struct { type CopyShardRequest struct { SourceNodeAddr string `json:"source_node_address"` - DestNodeAddr string `json:"dest_node_address"` + DestNodeAddr string `json:"dest_node_address"` // is this necessary? ShardID uint64 `json:"shard_id"` } @@ -463,7 +484,7 @@ type CopyShardTask struct { Database string `json:"database"` Rp string `json:"retention_policy"` ShardID uint64 `json:"shard_id"` - TotalSize uint64 `json:"total_size"` + TotalSize uint64 `json:"total_size"` // is this necessary? currently it's ignored. CurrentSize uint64 `json:"current_size"` Source string `json:"source"` Destination string `json:"destination"` diff --git a/services/controller/shard.go b/services/controller/shard.go index 6c9ed87..972ef7f 100644 --- a/services/controller/shard.go +++ b/services/controller/shard.go @@ -1,305 +1,57 @@ package controller import ( - "encoding/json" - "errors" "fmt" - "io" - "math" "os" "path/filepath" "strconv" - "sync" - "time" - "github.com/influxdata/influxdb" - "github.com/influxdata/influxdb/cmd/influxd/backup_util" - tarstream "github.com/influxdata/influxdb/pkg/tar" - "github.com/influxdata/influxdb/services/meta" - "github.com/influxdata/influxdb/services/snapshotter" - "github.com/influxdata/influxdb/tcp" - "github.com/influxdata/influxdb/tsdb" - "go.uber.org/zap" + "github.com/angopher/chronus/services/migrate" + "github.com/angopher/chronus/x" ) -type ShardCopyTask struct { - start time.Time - db string - rp string - totalSize uint64 - currentSize uint64 - shardId uint64 - source string - destination string - closer io.Closer -} - -type ShardCopyManager struct { - mutex sync.Mutex - tasks map[uint64]*ShardCopyTask - maxRunning int -} - -func NewCopyManager(max int) *ShardCopyManager { - return &ShardCopyManager{ - maxRunning: max, - tasks: make(map[uint64]*ShardCopyTask), - } -} - -func (c *ShardCopyManager) Add(task *ShardCopyTask) error { - c.mutex.Lock() - defer c.mutex.Unlock() - _, ok := c.tasks[task.shardId] - if ok { - return errors.New("shard task already exists") - } - c.tasks[task.shardId] = task - return nil -} - -func (c *ShardCopyManager) Remove(id uint64) { - c.mutex.Lock() - defer c.mutex.Unlock() - delete(c.tasks, id) - return -} - -func (c *ShardCopyManager) Tasks() []*ShardCopyTask { - c.mutex.Lock() - defer c.mutex.Unlock() - var tasks []*ShardCopyTask - for _, task := range c.tasks { - tasks = append(tasks, task) - } - return tasks -} - -func (c *ShardCopyManager) Kill(id uint64, source, dest string) { - c.mutex.Lock() - defer c.mutex.Unlock() - t, ok := c.tasks[id] - if !ok { - return - } - if source == t.source && dest == t.destination { - t.closer.Close() - delete(c.tasks, id) - } -} - -type ShardCopier struct { - Node *influxdb.Node - Logger *zap.Logger - Manager interface { - Add(task *ShardCopyTask) error - Remove(id uint64) - Tasks() []*ShardCopyTask - Kill(id uint64, source, dest string) - } - - MetaClient interface { - ShardOwner(shardID uint64) (database, policy string, sgi *meta.ShardGroupInfo) - AddShardOwner(shardID, nodeID uint64) error - } - TSDBStore interface { - Path() string - ShardRelativePath(id uint64) (string, error) - CreateShard(database, retentionPolicy string, shardID uint64, enabled bool) error - Shard(id uint64) *tsdb.Shard - } -} - -func (s *ShardCopier) WithLogger(log *zap.Logger) { - s.Logger = log.With(zap.String("service", "Controller.ShardCopier")) -} - -func fileExists(fileName string) bool { - _, err := os.Stat(fileName) - return err == nil -} - -func (s *ShardCopier) Query() []CopyShardTask { - var ts []CopyShardTask - tasks := s.Manager.Tasks() - for _, t := range tasks { - task := CopyShardTask{ - Database: t.db, - Rp: t.rp, - ShardID: t.shardId, - TotalSize: t.totalSize, - CurrentSize: t.currentSize, - Source: t.source, - Destination: t.destination, - } - ts = append(ts, task) - } - return ts -} - -func (s *ShardCopier) Kill(shardId uint64, source, destination string) { - s.Manager.Kill(shardId, source, destination) -} - -func (s *ShardCopier) CopyShard(sourceAddr string, shardId uint64) error { - // 1.检查本地是否已经存在shard - // 2.检查是否可以进行此shard的copy任务: 任务管理器 - // 3.检查本地是否有残留的脏数据, 并清理掉 - // 4.下载shard备份 - // 5.解压shard数据包 - // 6.创建shard +func (s *Service) copyShard(sourceAddr string, shardId uint64) error { + task := migrate.Task{} + task.SrcHost = sourceAddr + task.ShardId = shardId db, rp, sgi := s.MetaClient.ShardOwner(shardId) if sgi == nil { return fmt.Errorf("shard %d not exists", shardId) } + task.Database = db + task.Retention = rp path := filepath.Join(s.TSDBStore.Path(), db, rp, strconv.FormatUint(shardId, 10)) - if fileExists(path) { - return fmt.Errorf("path:[%s] exists", path) + if x.Exists(path) != x.NotExisted { + return fmt.Errorf("local shard:[%s] exists", path) } + task.DstStorePath = path - copyDir := s.TSDBStore.Path() + "/.copy_shard" + copyDir := filepath.Join(s.TSDBStore.Path(), ".copy_shard") os.MkdirAll(copyDir, 0755) - tmpPath := fmt.Sprintf("%s/shard_%d", copyDir, shardId) - defer os.RemoveAll(tmpPath) - - if !fileExists(tmpPath) { - req := &snapshotter.Request{ - Type: snapshotter.RequestShardBackup, - BackupDatabase: db, - BackupRetentionPolicy: rp, - ShardID: shardId, - //Since: cmd.since, - //ExportStart: cmd.start, - //ExportEnd: cmd.end, - } - fmt.Println("----------- tmpPath:", tmpPath) - if err := s.downloadAndVerify(req, sourceAddr, tmpPath, nil); err != nil { - return err - } - } - - fmt.Println("----------- shard path:", path) - if _, err := os.Stat(tmpPath); err == nil || os.IsNotExist(err) { - if err := s.unpackTar(tmpPath, path); err != nil { - return err - } - } - - sh := s.TSDBStore.Shard(shardId) - if sh == nil { - if err := s.TSDBStore.CreateShard(db, rp, shardId, true); err != nil { - return err - } - } - - return s.MetaClient.AddShardOwner(shardId, s.Node.ID) -} + task.TmpStorePath = copyDir -func (s *ShardCopier) downloadAndVerify(req *snapshotter.Request, host, path string, validator func(string) error) error { - tmppath := path + backup_util.Suffix - if err := s.download(req, host, tmppath); err != nil { - os.Remove(tmppath) + err := s.migrateManager.Add(&task) + if err != nil { return err } - if validator != nil { - if err := validator(tmppath); err != nil { - if rmErr := os.Remove(tmppath); rmErr != nil { - s.Logger.Sugar().Errorf("Error cleaning up temporary file: %v", rmErr) - } - return err - } - } + err = <-task.C - f, err := os.Stat(tmppath) if err != nil { return err } - // There was nothing downloaded, don't create an empty backup file. - if f.Size() == 0 { - return os.Remove(tmppath) - } - - // Rename temporary file to final path. - if err := os.Rename(tmppath, path); err != nil { - return fmt.Errorf("rename: %s", err) + sh := s.TSDBStore.Shard(shardId) + if sh != nil { + return fmt.Errorf("Shard %d is existed which is unexpected after backuping", shardId) } - return nil -} - -// unpackTar will restore a single tar archive to the data dir -func (s *ShardCopier) unpackTar(tarFile, shardPath string) error { - s.Logger.Sugar().Infof("Restoring from backup %s\n", shardPath) - f, err := os.Open(tarFile) + err = s.TSDBStore.CreateShard(task.Database, task.Retention, task.ShardId, true) if err != nil { return err } - defer f.Close() - - os.MkdirAll(shardPath, 0755) - - return tarstream.Restore(f, shardPath) -} - -func (s *ShardCopier) download(req *snapshotter.Request, host, path string) error { - // Create local file to write to. - f, err := os.Create(path) - if err != nil { - return fmt.Errorf("open temp file: %s", err) - } - defer f.Close() - - min := 2 * time.Second - for i := 0; i < 2; i++ { - if err = func() error { - // Connect to snapshotter service. - conn, err := tcp.Dial("tcp", host, snapshotter.MuxHeader) - if err != nil { - return err - } - defer conn.Close() - - task := &ShardCopyTask{ - shardId: req.ShardID, - source: host, - closer: conn, - } - s.Manager.Add(task) - defer s.Manager.Remove(task.shardId) - - // Write the request type - _, err = conn.Write([]byte{byte(req.Type)}) - if err != nil { - return err - } - - // Write the request - if err := json.NewEncoder(conn).Encode(req); err != nil { - return fmt.Errorf("encode snapshot request: %s", err) - } - - // Read snapshot from the connection - //TODO: 1.Rate limit? - // 2.How to get progress? - if n, err := io.Copy(f, conn); err != nil || n == 0 { - return fmt.Errorf("copy backup to file: err=%v, n=%d", err, n) - } - return nil - }(); err == nil { - break - } else if err != nil { - backoff := time.Duration(math.Pow(3.8, float64(i))) * time.Millisecond - if backoff < min { - backoff = min - } - s.Logger.Sugar().Errorf("Download shard %v failed %s. Waiting %v and retrying (%d)...\n", req.ShardID, err, backoff, i) - time.Sleep(backoff) - } - } - return err + return s.MetaClient.AddShardOwner(task.ShardId, s.Node.ID) } diff --git a/services/hh/config_test.go b/services/hh/config_test.go index 2f57a57..25d2c71 100644 --- a/services/hh/config_test.go +++ b/services/hh/config_test.go @@ -1,16 +1,15 @@ -package hh_test +package hh import ( "testing" "time" "github.com/BurntSushi/toml" - "github.com/influxdata/influxdb/services/hh" ) func TestConfigParse(t *testing.T) { // Parse configuration. - var c hh.Config + var c Config if _, err := toml.Decode(` enabled = false retry-interval = "10m" @@ -56,7 +55,7 @@ purge-interval = "1h" func TestDefaultDisabled(t *testing.T) { // Parse empty configuration. - var c hh.Config + var c Config if _, err := toml.Decode(``, &c); err != nil { t.Fatal(err) } @@ -66,7 +65,7 @@ func TestDefaultDisabled(t *testing.T) { } // Default configuration. - c = hh.NewConfig() + c = NewConfig() if exp := false; c.Enabled == true { t.Fatalf("unexpected default enabled value: got %v, exp %v", c.Enabled, exp) } diff --git a/services/hh/limiter.go b/services/hh/limiter.go deleted file mode 100644 index b2a69f9..0000000 --- a/services/hh/limiter.go +++ /dev/null @@ -1,61 +0,0 @@ -package hh - -import "time" - -type limiter struct { - count int64 - limit int64 - start time.Time - delay float64 -} - -// NewRateLimiter returns a new limiter configured to restrict a process to the limit per second. -// limit is the maximum amount that can be used per second. The limit should be > 0. A limit -// <= 0, will not limit the processes. -func NewRateLimiter(limit int64) *limiter { - return &limiter{ - start: time.Now(), - limit: limit, - delay: 0.5, - } -} - -// Update updates the amount used -func (t *limiter) Update(count int) { - t.count += int64(count) -} - -// Delay returns the amount of time, up to 1 second, that caller should wait -// to maintain the configured rate -func (t *limiter) Delay() time.Duration { - if t.limit > 0 { - - delta := time.Now().Sub(t.start).Seconds() - rate := int64(float64(t.count) / delta) - - // Determine how far off from the max rate we are - delayAdj := float64((t.limit - rate)) / float64(t.limit) - - // Don't adjust by more than 1 second at a time - delayAdj = t.clamp(delayAdj, -1, 1) - - t.delay -= delayAdj - if t.delay < 0 { - t.delay = 0 - } - - return time.Duration(t.delay) * time.Second - } - return time.Duration(0) -} - -func (t *limiter) clamp(value, min, max float64) float64 { - if value < min { - return min - } - - if value > max { - return max - } - return value -} diff --git a/services/hh/limiter_test.go b/services/hh/limiter_test.go deleted file mode 100644 index 5edb6e0..0000000 --- a/services/hh/limiter_test.go +++ /dev/null @@ -1,47 +0,0 @@ -package hh - -import ( - "testing" - "time" -) - -func TestLimiter(t *testing.T) { - l := NewRateLimiter(0) - l.Update(500) - if l.Delay().Nanoseconds() != 0 { - t.Errorf("limiter with no limit mismatch: got %v, exp 0", l.Delay()) - } -} - -func TestLimiterWithinLimit(t *testing.T) { - if testing.Short() { - t.Skip("Shipping TestLimiterWithinLimit") - } - - l := NewRateLimiter(1000) - for i := 0; i < 100; i++ { - // 50 ever 100ms = 500/s which should be within the rate - l.Update(50) - l.Delay() - time.Sleep(100 * time.Millisecond) - } - - // Should not have any delay - delay := l.Delay().Seconds() - if exp := int(0); int(delay) != exp { - t.Errorf("limiter rate mismatch: got %v, exp %v", int(delay), exp) - } - -} - -func TestLimiterExceeded(t *testing.T) { - l := NewRateLimiter(1000) - for i := 0; i < 10; i++ { - l.Update(200) - l.Delay() - } - delay := l.Delay().Seconds() - if int(delay) == 0 { - t.Errorf("limiter rate mismatch. expected non-zero delay") - } -} diff --git a/services/hh/node_processor.go b/services/hh/node_processor.go index e00f320..23666a9 100644 --- a/services/hh/node_processor.go +++ b/services/hh/node_processor.go @@ -1,6 +1,7 @@ package hh import ( + "context" "encoding/binary" "fmt" "io" @@ -9,9 +10,11 @@ import ( "sync" "time" - "github.com/influxdata/influxdb/models" "strings" "sync/atomic" + + "github.com/influxdata/influxdb/models" + "golang.org/x/time/rate" ) const ( @@ -28,7 +31,7 @@ type NodeProcessor struct { RetryMaxInterval time.Duration // Max interval between periodic write-to-node attempts. MaxSize int64 // Maximum size an underlying queue can get. MaxAge time.Duration // Maximum age queue data can get before purging. - RetryRateLimit int64 // Limits the rate data is sent to node. + RetryRateLimit int // Limits the rate data is sent to node. nodeID uint64 dir string @@ -202,7 +205,8 @@ func (n *NodeProcessor) run() { } case <-time.After(currInterval): - limiter := NewRateLimiter(n.RetryRateLimit) + limiter := rate.NewLimiter(rate.Limit(n.RetryRateLimit), 10*n.RetryRateLimit) + ctx := context.Background() for { c, err := n.SendWrite() if err != nil { @@ -221,11 +225,7 @@ func (n *NodeProcessor) run() { // Success! Ensure backoff is cancelled. currInterval = time.Duration(n.RetryInterval) - // Update how many bytes we've sent - limiter.Update(c) - - // Block to maintain the throughput rate - time.Sleep(limiter.Delay()) + limiter.WaitN(ctx, c) } } } diff --git a/services/hh/node_processor_test.go b/services/hh/node_processor_test.go index 10dd560..c8b40c5 100644 --- a/services/hh/node_processor_test.go +++ b/services/hh/node_processor_test.go @@ -8,8 +8,7 @@ import ( "time" "github.com/influxdata/influxdb/models" - - "github.com/angopher/chronus/services/meta" + "github.com/influxdata/influxdb/services/meta" ) type fakeShardWriter struct { @@ -36,7 +35,8 @@ func TestNodeProcessorSendBlock(t *testing.T) { // expected data to be queue and sent to the shardWriter var expShardID, expNodeID, count = uint64(100), uint64(200), 0 - pt := models.MustNewPoint("cpu", models.Tags{"foo": "bar"}, models.Fields{"value": 1.0}, time.Unix(0, 0)) + tag := models.Tag{Key: []byte("foo"), Value: []byte("bar")} + pt := models.MustNewPoint("cpu", models.Tags{tag}, models.Fields{"value": 1.0}, time.Unix(0, 0)) sh := &fakeShardWriter{ ShardWriteFn: func(shardID, nodeID uint64, points []models.Point) error { diff --git a/services/meta/client.go b/services/meta/client.go index d9f5a41..8224f47 100644 --- a/services/meta/client.go +++ b/services/meta/client.go @@ -7,7 +7,6 @@ import ( crand "crypto/rand" "crypto/sha256" "errors" - "fmt" "io" "io/ioutil" "net/http" @@ -27,14 +26,14 @@ import ( ) const ( - // SaltBytes is the number of bytes used for salts. - SaltBytes = 32 + // SALT_LENGTH is the number of bytes used for salts. + SALT_LENGTH = 32 - metaFile = "meta.db" + META_FILE = "meta.db" - // ShardGroupDeletedExpiration is the amount of time before a shard group info will be removed from cached + // SHARDGROUP_INFO_EVICTION is the amount of time before a shard group info will be removed from cached // data after it has been marked deleted (2 weeks). - ShardGroupDeletedExpiration = -2 * 7 * 24 * time.Hour + SHARDGROUP_INFO_EVICTION = -2 * 7 * 24 * time.Hour ) // Client is used to execute commands on and read data from @@ -79,10 +78,6 @@ func NewClient(config *meta.Config) *Client { } } -func (c *Client) Print() { - fmt.Printf("%+v\n", c.cacheData) -} - // Open a connection to a meta service cluster. func (c *Client) Open() error { c.mu.Lock() @@ -134,28 +129,29 @@ func (c *Client) data() *Data { return c.cacheData } -// Node returns a node by id. +// DataNode returns a node by id. func (c *Client) DataNode(id uint64) (*meta.NodeInfo, error) { c.mu.Lock() defer c.mu.Unlock() - for _, n := range c.data().DataNodes { - if n.ID == id { - return &n, nil - } + n := c.data().DataNode(id) + if n == nil { + return nil, ErrNodeNotFound } - return nil, ErrNodeNotFound + return n, nil } -// DataNodes returns the data nodes' info. -func (c *Client) DataNodes() ([]meta.NodeInfo, error) { - return c.data().DataNodes, nil +// DataNodes returns all nodes +func (c *Client) DataNodes() []meta.NodeInfo { + c.mu.Lock() + defer c.mu.Unlock() + return c.data().DataNodes } // CreateDataNode will create a new data node in the metastore func (c *Client) CreateDataNode(httpAddr, tcpAddr string) (*meta.NodeInfo, error) { c.mu.Lock() defer c.mu.Unlock() - err := c.data().CreateDataNode(httpAddr, tcpAddr) + _, err := c.data().CreateDataNode(httpAddr, tcpAddr) if err != nil { return nil, err } @@ -172,10 +168,11 @@ func (c *Client) CreateDataNode(httpAddr, tcpAddr string) (*meta.NodeInfo, error // DataNodeByHTTPHost returns the data node with the give http bind address func (c *Client) DataNodeByHTTPHost(httpAddr string) (*meta.NodeInfo, error) { - nodes, _ := c.DataNodes() + nodes := c.data().DataNodes for _, n := range nodes { if n.Host == httpAddr { - return &n, nil + newN := n + return &newN, nil } } @@ -184,10 +181,11 @@ func (c *Client) DataNodeByHTTPHost(httpAddr string) (*meta.NodeInfo, error) { // DataNodeByTCPHost returns the data node with the give http bind address func (c *Client) DataNodeByTCPHost(tcpAddr string) (*meta.NodeInfo, error) { - nodes, _ := c.DataNodes() + nodes := c.data().DataNodes for _, n := range nodes { if n.TCPHost == tcpAddr { - return &n, nil + newN := n + return &newN, nil } } @@ -482,7 +480,7 @@ func (c *Client) hashWithSalt(salt []byte, password string) []byte { // saltedHash returns a salt and salted hash of password. func (c *Client) saltedHash(password string) (salt, hash []byte, err error) { - salt = make([]byte, SaltBytes) + salt = make([]byte, SALT_LENGTH) if _, err := io.ReadFull(crand.Reader, salt); err != nil { return nil, nil, err } @@ -785,7 +783,7 @@ func (c *Client) TruncateShardGroups(t time.Time) error { // PruneShardGroups remove deleted shard groups from the data store. func (c *Client) PruneShardGroups() error { var changed bool - expiration := time.Now().Add(ShardGroupDeletedExpiration) + expiration := time.Now().Add(SHARDGROUP_INFO_EVICTION) c.mu.Lock() defer c.mu.Unlock() data := c.cacheData.Clone() @@ -1139,7 +1137,7 @@ func (c *Client) WithLogger(log *zap.Logger) { func snapshot(path string, data *Data) error { //TODO: no need write snapshot to disk return nil - filename := filepath.Join(path, metaFile) + filename := filepath.Join(path, META_FILE) tmpFile := filename + "tmp" f, err := os.Create(tmpFile) @@ -1175,7 +1173,7 @@ func snapshot(path string, data *Data) error { func (c *Client) Load() error { //TODO:no need load return nil - file := filepath.Join(c.path, metaFile) + file := filepath.Join(c.path, META_FILE) f, err := os.Open(file) if err != nil { diff --git a/services/meta/data.go b/services/meta/data.go index 4958567..2632191 100644 --- a/services/meta/data.go +++ b/services/meta/data.go @@ -24,18 +24,20 @@ type Data struct { func (data *Data) DataNode(id uint64) *meta.NodeInfo { for i := range data.DataNodes { if data.DataNodes[i].ID == id { - return &data.DataNodes[i] + // prevent unexpected modification + n := data.DataNodes[i] + return &n } } return nil } -// CreateDataNode adds a node to the metadata. -func (data *Data) CreateDataNode(host, tcpHost string) error { +// CreateDataNode adds a node to the metadata, return the nodeId(0 when an error occurred) and error +func (data *Data) CreateDataNode(host, tcpHost string) (uint64, error) { // Ensure a node with the same host doesn't already exist. for _, n := range data.DataNodes { if n.TCPHost == tcpHost || n.Host == host { - return ErrNodeExists + return 0, ErrNodeExists } } @@ -63,7 +65,7 @@ func (data *Data) CreateDataNode(host, tcpHost string) error { }) sort.Sort(meta.NodeInfos(data.DataNodes)) - return nil + return existingID, nil } // DeleteDataNode removes a node from the Meta store. @@ -202,18 +204,20 @@ func newShardOwner(s meta.ShardInfo, ownerFreqs map[int]int) (uint64, error) { func (data *Data) MetaNode(id uint64) *meta.NodeInfo { for i := range data.MetaNodes { if data.MetaNodes[i].ID == id { - return &data.MetaNodes[i] + // prevent unexpected modification + n := data.MetaNodes[i] + return &n } } return nil } // CreateMetaNode will add a new meta node to the metastore -func (data *Data) CreateMetaNode(httpAddr, tcpAddr string) error { +func (data *Data) CreateMetaNode(httpAddr, tcpAddr string) (uint64, error) { // Ensure a node with the same host doesn't already exist. for _, n := range data.MetaNodes { if n.Host == httpAddr { - return ErrNodeExists + return 0, ErrNodeExists } } @@ -242,7 +246,7 @@ func (data *Data) CreateMetaNode(httpAddr, tcpAddr string) error { }) sort.Sort(meta.NodeInfos(data.MetaNodes)) - return nil + return existingID, nil } // DeleteMetaNode will remove the meta node from the store diff --git a/services/meta/data_test.go b/services/meta/data_test.go index fc35c76..988c264 100644 --- a/services/meta/data_test.go +++ b/services/meta/data_test.go @@ -5,6 +5,7 @@ import ( "time" "github.com/influxdata/influxdb/services/meta" + "github.com/stretchr/testify/assert" imeta "github.com/angopher/chronus/services/meta" ) @@ -18,54 +19,93 @@ func newData() *imeta.Data { } } -func TestCreateDataNode(t *testing.T) { - data := newData() +func initialTwoDataNodes(data *imeta.Data) (uint64, uint64) { host := "127.0.0.1:8080" tcpHost := "127.0.0.1:8081" - if err := data.CreateDataNode(host, tcpHost); err != nil { - t.Fatalf("unexpected err: %s", err) - } + id1, _ := data.CreateDataNode(host, tcpHost) - expN := meta.NodeInfo{ID: 1, TCPHost: tcpHost, Host: host} - n := data.DataNode(1) - if n.ID != expN.ID || n.TCPHost != expN.TCPHost || n.Host != expN.Host { - t.Fatalf("expected node: %+v, got: %+v", expN, n) - } + host = "127.0.0.1:9080" + tcpHost = "127.0.0.1:9081" + id2, _ := data.CreateDataNode(host, tcpHost) - if err, exp := data.CreateDataNode(host, tcpHost), imeta.ErrNodeExists; err != exp { - t.Fatalf("expected err: %s, got: %s", exp, err) - } + return id1, id2 } -func TestCreateAndDeleteMetaNode(t *testing.T) { +func TestCreateDataNode(t *testing.T) { data := newData() host := "127.0.0.1:8080" tcpHost := "127.0.0.1:8081" - if err := data.CreateMetaNode(host, tcpHost); err != nil { - t.Fatalf("unexpected err: %s", err) - } - expN := meta.NodeInfo{ID: 1, TCPHost: tcpHost, Host: host} - n := data.MetaNodes[0] - if n.ID != expN.ID || n.TCPHost != expN.TCPHost || n.Host != expN.Host { - t.Fatalf("expected node: %+v, got: %+v", expN, n) - } - - if err, exp := data.CreateMetaNode(host, tcpHost), imeta.ErrNodeExists; err != exp { - t.Fatalf("expected err: %s, got: %s", exp, err) - } - - if err, exp := data.DeleteMetaNode(0), imeta.ErrNodeIDRequired; err != exp { - t.Fatalf("expected err: %s, got: %s", exp, err) - } + // create one node + id, err := data.CreateDataNode(host, tcpHost) + assert.Nil(t, err) + assert.True(t, id > 0) + + // get it by id + n := data.DataNode(id) + assert.NotNil(t, n) + assert.Equal(t, id, n.ID) + assert.Equal(t, host, n.Host) + assert.Equal(t, tcpHost, n.TCPHost) + + // refetch, ensure it should not affect inner store + n.Host = "test" + n = data.DataNode(id) + assert.NotNil(t, n) + assert.Equal(t, id, n.ID) + assert.Equal(t, host, n.Host) + assert.Equal(t, tcpHost, n.TCPHost) + + // duplicated creation + _, err = data.CreateDataNode(host, tcpHost) + assert.Equal(t, imeta.ErrNodeExists, err) + + // try to delete it (simple deletion) + err = data.DeleteDataNode(0) + assert.Equal(t, imeta.ErrNodeIDRequired, err) + err = data.DeleteDataNode(9999999) + assert.Equal(t, imeta.ErrNodeNotFound, err) + err = data.DeleteDataNode(id) + assert.Nil(t, err) + + // fetch after deletion + n = data.DataNode(id) + assert.Nil(t, n) +} - if err, exp := data.DeleteMetaNode(2), imeta.ErrNodeNotFound; err != exp { - t.Fatalf("expected err: %s, got: %s", exp, err) - } +func TestCreateAndDeleteMetaNode(t *testing.T) { + data := newData() + host := "127.0.0.1:8080" + tcpHost := "127.0.0.1:8081" - if err := data.DeleteMetaNode(expN.ID); err != nil { - t.Fatalf("unexpected err: %s", err) - } + // create + id, err := data.CreateMetaNode(host, tcpHost) + assert.Nil(t, err) + + // fetch back + n := data.MetaNode(id) + assert.NotNil(t, n) + assert.Equal(t, id, n.ID) + assert.Equal(t, host, n.Host) + assert.Equal(t, tcpHost, n.TCPHost) + + // recreation + _, err = data.CreateMetaNode(host, tcpHost) + assert.Equal(t, imeta.ErrNodeExists, err) + + // try to delete it (simple deletion) + err = data.DeleteMetaNode(0) + assert.Equal(t, imeta.ErrNodeIDRequired, err) + err = data.DeleteMetaNode(9999999) + assert.Equal(t, imeta.ErrNodeNotFound, err) + err = data.DeleteMetaNode(id) + assert.Nil(t, err) + err = data.DeleteMetaNode(id) + assert.Equal(t, imeta.ErrNodeNotFound, err) + + // fetch after deletion + n = data.MetaNode(id) + assert.Nil(t, n) } func TestCreateShardGroup(t *testing.T) { @@ -75,27 +115,28 @@ func TestCreateShardGroup(t *testing.T) { data.CreateDatabase(name) data.CreateRetentionPolicy(name, meta.DefaultRetentionPolicyInfo(), true) - if err := data.CreateShardGroup(name, policy, time.Now()); err != imeta.ErrNodeNotFound { - t.Fatalf("expected err: %s, got: %s", imeta.ErrNodeNotFound, err) - } + // create group with no node in cluster + err := data.CreateShardGroup(name, policy, time.Now()) + assert.Equal(t, imeta.ErrNodeNotFound, err) - createTwoDataNode(data) + // add nodes to cluster + id1, id2 := initialTwoDataNodes(data) - if err := data.CreateShardGroup(name, policy, time.Now()); err != nil { - t.Fatalf("unexpected err: %s", err) - } + // recreation + err = data.CreateShardGroup(name, policy, time.Now()) + assert.Nil(t, err) rp := data.Database(name).RetentionPolicy(policy) + assert.NotNil(t, rp) + // replica = 1(default), shards = node_cnt / replica + assert.Equal(t, 1, len(rp.ShardGroups)) + assert.Equal(t, 2, len(rp.ShardGroups[0].Shards)) sgi := rp.ShardGroups[0] - if len(sgi.Shards) != 2 { - t.Fatalf("unexpected 2 shards, got: %d", len(sgi.Shards)) - } for _, sh := range sgi.Shards { + assert.Equal(t, 1, len(sh.Owners)) o := sh.Owners[0] - if len(sh.Owners) != 1 || o.NodeID != 1 && o.NodeID != 2 { - t.Fatalf("unexpected shard: %+v", sh) - } + assert.True(t, o.NodeID == id1 || o.NodeID == id2) } replicaN := 2 @@ -111,19 +152,15 @@ func TestCreateShardGroup(t *testing.T) { rp = data.Database(name).RetentionPolicy("rp") sgi = rp.ShardGroups[0] - if len(sgi.Shards) != 1 { - t.Fatalf("unexpected 1 shards, got: %d", len(sgi.Shards)) - } + assert.NotNil(t, rp) + assert.Equal(t, 1, len(rp.ShardGroups)) + assert.Equal(t, 1, len(rp.ShardGroups[0].Shards)) owners := sgi.Shards[0].Owners - if len(owners) != 2 { - t.Fatalf("unexpected shard: %+v", sgi.Shards[0]) - } + assert.Equal(t, 2, len(owners)) for _, o := range owners { - if o.NodeID != 1 && o.NodeID != 2 { - t.Fatalf("unexpected owner: %+v", o) - } + assert.True(t, o.NodeID == id1 || o.NodeID == id2) } } @@ -132,7 +169,7 @@ func TestDeleteDataNode(t *testing.T) { name := "testdb" policy := "rp" data.CreateDatabase(name) - createTwoDataNode(data) + id1, id2 := initialTwoDataNodes(data) replicaN := 2 duration := time.Hour @@ -145,28 +182,26 @@ func TestDeleteDataNode(t *testing.T) { data.CreateRetentionPolicy(name, spec.NewRetentionPolicyInfo(), true) data.CreateShardGroup(name, policy, time.Now()) - data.DeleteDataNode(1) - + // before deletion, 2 nodes rp := data.Database(name).RetentionPolicy("rp") + assert.NotNil(t, rp) + assert.Equal(t, 1, len(rp.ShardGroups)) sgi := rp.ShardGroups[0] - + assert.Equal(t, 1, len(sgi.Shards)) owners := sgi.Shards[0].Owners - if len(owners) != 1 { - t.Fatalf("unexpected shard: %+v", sgi.Shards[0]) - } + assert.Equal(t, 2, len(owners)) - o := owners[0] - if o.NodeID != 2 { - t.Fatalf("unexpected owner: %+v", o) - } -} + data.DeleteDataNode(id1) -func createTwoDataNode(data *imeta.Data) { - host := "127.0.0.1:8080" - tcpHost := "127.0.0.1:8081" - data.CreateDataNode(host, tcpHost) + // delete 1 node + rp = data.Database(name).RetentionPolicy("rp") + assert.NotNil(t, rp) + assert.Equal(t, 1, len(rp.ShardGroups)) + sgi = rp.ShardGroups[0] + assert.Equal(t, 1, len(sgi.Shards)) + owners = sgi.Shards[0].Owners + assert.Equal(t, 1, len(owners)) - host = "127.0.0.1:9080" - tcpHost = "127.0.0.1:9081" - data.CreateDataNode(host, tcpHost) + o := owners[0] + assert.Equal(t, id2, o.NodeID) } diff --git a/services/migrate/executor.go b/services/migrate/executor.go new file mode 100644 index 0000000..58b15d6 --- /dev/null +++ b/services/migrate/executor.go @@ -0,0 +1,177 @@ +package migrate + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "time" + + "github.com/angopher/chronus/x" + "github.com/influxdata/influxdb/pkg/tar" + "github.com/influxdata/influxdb/services/meta" + "github.com/influxdata/influxdb/services/snapshotter" + "github.com/influxdata/influxdb/tcp" + "github.com/influxdata/influxdb/tsdb" + "go.uber.org/zap" +) + +const ( + MAX_RETRY = 3 +) + +type ValidatorFn func(file string) error +type ProgressCB func(copied uint64) + +type ManagerInterface interface { + Add(task *Task) error + Remove(id uint64) + Tasks() []*Task + Kill(id uint64, source, dest string) +} + +type MetaClientInterface interface { + ShardOwner(shardID uint64) (database, policy string, sgi *meta.ShardGroupInfo) + AddShardOwner(shardID, nodeID uint64) error +} + +type TSDBInterface interface { + Path() string + ShardRelativePath(id uint64) (string, error) + CreateShard(database, retentionPolicy string, shardID uint64, enabled bool) error + Shard(id uint64) *tsdb.Shard +} + +func (m *Manager) execute(t *Task) (err error) { + t.Started = true + t.StartTime = time.Now().Unix() + defer func() { + if !t.Finished { + t.error(fmt.Errorf("Unknow error for task on shard %d", t.ShardId)) + } + }() + i := 1 + delay := 2 * time.Second + for i <= MAX_RETRY { + err = m.replicate(t) + if err == nil { + return + } + time.Sleep(delay) + // backoff by 2 + delay *= 2 + i++ + } + return fmt.Errorf("Migration for shard %d failed after retries", t.ShardId) +} + +func backupFromRemote(t *Task, logger *zap.Logger) (*os.File, error) { + // prepare local tmp file + suger := logger.Sugar() + tmpFilePath := filepath.Join(t.TmpStorePath, fmt.Sprint("shard_", t.ShardId)) + tmpFile, err := os.Create(tmpFilePath) + if err != nil { + return nil, fmt.Errorf("open temp file failed: %s", err) + } + defer tmpFile.Close() + // Connect to snapshotter service. + conn, err := tcp.Dial("tcp", t.SrcHost, snapshotter.MuxHeader) + if err != nil { + return tmpFile, err + } + defer conn.Close() + + req := &snapshotter.Request{ + Type: snapshotter.RequestShardBackup, + BackupDatabase: t.Database, + BackupRetentionPolicy: t.Retention, + ShardID: t.ShardId, + } + + // Write the request type + _, err = conn.Write([]byte{byte(req.Type)}) + if err != nil { + return tmpFile, err + } + + // Write the request + if err := json.NewEncoder(conn).Encode(req); err != nil { + return tmpFile, fmt.Errorf("encode snapshot request: %s", err) + } + + buf := make([]byte, 1024*1024) + n := 0 + ctx := context.Background() + for { + t.Limiter.Allow() + n, err = conn.Read(buf) + if err != nil { + break + } + if n == 0 { + break + } + tmpFile.Write(buf[:n]) + t.Copied += uint64(n) + // limiter in post way + t.Limiter.WaitN(ctx, n) + if t.ProgressLimiter.Allow() { + suger.Infof("Mirating shard %d: %d copied", t.ShardId, t.Copied) + } + } + suger.Infof("Transfer Shard %d completely", t.ShardId) + + // XXX: Verify + return tmpFile, nil +} + +func (m *Manager) replicate(t *Task) error { + sugar := m.logger.Sugar() + if x.Exists(t.DstStorePath) != x.NotExisted { + t.error(errors.New("Destination shard directory has already existed")) + return nil + } + if x.Exists(t.TmpStorePath) == x.NotExisted { + t.error(errors.New("Temporary directory is not existed")) + return nil + } + + // transfer to local + tmpFile, err := backupFromRemote(t, m.logger) + defer func() { + // remove tmp file + if tmpFile != nil { + os.Remove(tmpFile.Name()) + } + }() + if err != nil { + return err + } + + // unpack to destination + os.MkdirAll(t.DstStorePath, os.FileMode(0755)) + err = restoreFromTar(tmpFile.Name(), t.DstStorePath) + if err != nil { + sugar.Errorf("Unpack from backup failed for shard %d: %v", t.ShardId, err) + // remove incorrect data + os.RemoveAll(t.DstStorePath) + return err + } + + t.succ() + return nil +} + +// restore restores a tar archive to the target path +func restoreFromTar(tarFile, path string) error { + f, err := os.Open(tarFile) + if err != nil { + return err + } + defer f.Close() + + os.MkdirAll(path, 0755) + return tar.Restore(f, path) +} diff --git a/services/migrate/executor_test.go b/services/migrate/executor_test.go new file mode 100644 index 0000000..a95f296 --- /dev/null +++ b/services/migrate/executor_test.go @@ -0,0 +1,61 @@ +package migrate + +import ( + "os" + "path/filepath" + "testing" + + "github.com/influxdata/influxdb/pkg/tar" + "github.com/stretchr/testify/assert" +) + +func TestExecutorRestore(t *testing.T) { + basePath := os.TempDir() + ".chronus/tmp" + tarFilePath := os.TempDir() + ".chronus/t.tar" + os.RemoveAll(basePath) + assert.Nil(t, os.MkdirAll(basePath, os.FileMode(0755))) + defer os.RemoveAll(basePath) + + // initial files: a, b + f, err := os.Create(basePath + "/a") + assert.Nil(t, err) + f.WriteString("testA") + f.Close() + + f, err = os.Create(basePath + "/b") + assert.Nil(t, err) + f.WriteString("testB") + f.Close() + + os.Remove(tarFilePath) + f, err = os.Create(tarFilePath) + assert.Nil(t, err) + defer os.Remove(tarFilePath) + + // create tar file + err = tar.Stream(f, basePath, "/db/rp/1", nil) + assert.Nil(t, err) + f.Close() + + // verify files before removing + arr, err := filepath.Glob(basePath + "/*") + assert.Nil(t, err) + assert.Equal(t, 2, len(arr)) + + // remove them + os.Remove(basePath + "/a") + os.Remove(basePath + "/b") + + // verify it's empty + arr, err = filepath.Glob(basePath + "/*") + assert.Nil(t, err) + assert.Equal(t, 0, len(arr)) + + // extract tar file + assert.Nil(t, restoreFromTar(tarFilePath, basePath)) + + // verify it's comtent + arr, err = filepath.Glob(basePath + "/1/*") + assert.Nil(t, err) + assert.Equal(t, 2, len(arr)) +} diff --git a/services/migrate/manager.go b/services/migrate/manager.go new file mode 100644 index 0000000..0a046ca --- /dev/null +++ b/services/migrate/manager.go @@ -0,0 +1,206 @@ +package migrate + +import ( + "errors" + "io" + "sync" + + "go.uber.org/zap" + "golang.org/x/time/rate" +) + +const ( + TASK_PARALLEL_MAX = 24 +) + +var ( + ErrTaskDuplicated = errors.New("Shard migration task has already existed") +) + +// Task holds the task info +type Task struct { + // State + Started bool + Finished bool + StartTime int64 // timestamp in second + Error error + C chan error + + // Meta + Database string + Retention string + ShardId uint64 + SrcHost string + + // Store + DstStorePath string + TmpStorePath string + + // Progress + Copied uint64 + Limiter *rate.Limiter + ProgressLimiter *rate.Limiter // for progress logging + + // Callback + Closer io.Closer +} + +func (t *Task) succ() { + t.Finished = true + t.C <- nil + t.Closer.Close() +} + +func (t *Task) error(err error) { + t.Error = err + t.Finished = true + t.C <- err + t.Closer.Close() +} + +// Manager is the container of tasks running or queued +// You should use NewCopyManager to get an instance +type Manager struct { + sync.Mutex + wg sync.WaitGroup + cond *sync.Cond + taskMap map[uint64]*Task + taskQueue []*Task + parallel int + logger *zap.Logger + shouldStop bool +} + +// NewManager creates and returns a new manager +func NewManager(parallel int) *Manager { + if parallel < 1 { + parallel = 1 + } + if parallel > TASK_PARALLEL_MAX { + parallel = TASK_PARALLEL_MAX + } + m := &Manager{ + parallel: parallel, + taskMap: make(map[uint64]*Task), + taskQueue: make([]*Task, 0, TASK_PARALLEL_MAX), + cond: sync.NewCond(&sync.Mutex{}), + } + return m +} + +func (m *Manager) Start() { + for i := 0; i < m.parallel; i++ { + go m.loop() + } +} + +func (m *Manager) loop() { + m.wg.Add(1) + defer m.wg.Done() + for !m.shouldStop { + t := m.pop() + if t == nil { + m.cond.L.Lock() + m.cond.Wait() + m.cond.L.Unlock() + continue + } + + m.execute(t) + m.Remove(t.ShardId) + } +} + +func (m *Manager) Close() { + m.shouldStop = true + m.cond.Broadcast() + m.wg.Wait() + m.logger.Info("Shutdown migrating manager") +} + +func (m *Manager) WithLogger(log *zap.Logger) { + m.logger = log.With(zap.String("service", "migrate.Manager")) +} + +func (m *Manager) pop() *Task { + m.Lock() + defer m.Unlock() + if len(m.taskQueue) == 0 { + return nil + } + t := m.taskQueue[len(m.taskQueue)-1] + m.taskQueue = m.taskQueue[:len(m.taskQueue)-1] + return t +} + +// Add registers a task under specific shard and returns error if there was one already. +func (m *Manager) Add(task *Task) error { + m.Lock() + defer m.Unlock() + if _, ok := m.taskMap[task.ShardId]; ok { + return ErrTaskDuplicated + } + task.C = make(chan error, 1) + // Currently limit to 5MB/s, 10MB/s in burst + task.Limiter = rate.NewLimiter(5*1024*1024, 10*1024*1024) + task.ProgressLimiter = rate.NewLimiter(0.05, 10) // 1 log every 20 seconds + m.taskMap[task.ShardId] = task + m.taskQueue = append(m.taskQueue, task) + m.cond.Signal() + return nil +} + +// Remove removes the task under specific shard id WITHOUT stopping it +func (m *Manager) Remove(id uint64) { + m.Lock() + defer m.Unlock() + if t, ok := m.taskMap[id]; ok { + delete(m.taskMap, id) + m.removeFromQueue(t) + } +} + +// Tasks returns all tasks +func (m *Manager) Tasks() []*Task { + m.Lock() + defer m.Unlock() + tasks := make([]*Task, 0, len(m.taskMap)) + for _, t := range m.taskMap { + tasks = append(tasks, t) + } + return tasks +} + +func (m *Manager) removeFromQueue(t *Task) { + if t == nil { + return + } + pos := -1 + for i, item := range m.taskQueue { + if item == t { + pos = i + break + } + } + if pos >= 0 { + m.taskQueue = append(m.taskQueue[0:pos], m.taskQueue[pos+1:]...) + } +} + +// Kill kills the running task and remove it from list +// Host addresses are required for double check +func (m *Manager) Kill(id uint64, srcHost string) { + m.Lock() + defer m.Unlock() + t, ok := m.taskMap[id] + if !ok { + return + } + if srcHost == t.SrcHost { + if t.Closer != nil { + t.Closer.Close() + } + delete(m.taskMap, id) + m.removeFromQueue(t) + } +} diff --git a/services/migrate/manager_test.go b/services/migrate/manager_test.go new file mode 100644 index 0000000..b0fc637 --- /dev/null +++ b/services/migrate/manager_test.go @@ -0,0 +1,82 @@ +package migrate + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +type testCloser struct { + fn func() +} + +func (t testCloser) Close() error { + t.fn() + return nil +} + +func TestManagerNew(t *testing.T) { + m := NewManager(0) + assert.Equal(t, 1, m.parallel) + + m = NewManager(2) + assert.Equal(t, 2, m.parallel) + + m = NewManager(22222) + assert.Equal(t, TASK_PARALLEL_MAX, m.parallel) +} + +func TestManagerTasks(t *testing.T) { + cnt := 0 + task1 := &Task{ + ShardId: 123, + SrcHost: "a", + Closer: testCloser{ + fn: func() { + cnt++ + }, + }, + } + task2 := &Task{ + ShardId: 1235, + Closer: testCloser{ + fn: func() { + cnt++ + }, + }, + } + m := NewManager(1) + + assert.Equal(t, 0, len(m.Tasks())) + + // add task + assert.Nil(t, m.Add(task1)) + + assert.Equal(t, 1, len(m.Tasks())) + assert.Equal(t, task1, m.Tasks()[0]) + + // add another + assert.Nil(t, m.Add(task2)) + + assert.Equal(t, 2, len(m.Tasks())) + + assert.NotNil(t, m.pop()) + assert.NotNil(t, m.pop()) + assert.Nil(t, m.pop()) + + // kill + m.Kill(task1.ShardId, "") + assert.Equal(t, 2, len(m.Tasks())) + assert.Equal(t, 0, cnt) + m.Kill(task1.ShardId, "a") + assert.Equal(t, 1, len(m.Tasks())) + assert.Equal(t, 1, cnt) + + // remove only + m.Remove(task1.ShardId) + assert.Equal(t, 1, len(m.Tasks())) + assert.Equal(t, 1, cnt) + m.Remove(task2.ShardId) + assert.Equal(t, 0, len(m.Tasks())) + assert.Equal(t, 1, cnt) +} diff --git a/x/io.go b/x/io.go new file mode 100644 index 0000000..7c40bf6 --- /dev/null +++ b/x/io.go @@ -0,0 +1,29 @@ +package x + +import ( + "os" +) + +type ExistStat int + +const ( + ExistUnknow ExistStat = iota + NotExisted + ExistFile + ExistDir +) + +// Exists returns the existed stat for specific path +func Exists(ospath string) ExistStat { + stat, err := os.Stat(ospath) + if err != nil { + if os.IsNotExist(err) { + return NotExisted + } + return ExistUnknow + } + if stat.IsDir() { + return ExistDir + } + return ExistFile +} diff --git a/x/io_test.go b/x/io_test.go new file mode 100644 index 0000000..68140a6 --- /dev/null +++ b/x/io_test.go @@ -0,0 +1,26 @@ +package x + +import ( + "os" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestExists(t *testing.T) { + assert.Equal(t, NotExisted, Exists("")) + assert.Equal(t, NotExisted, Exists("..........")) + assert.Equal(t, NotExisted, Exists("/asdf/asdf/asdf/sad/f/sadf/")) + assert.Equal(t, NotExisted, Exists("/asdf/asdf/asdf/sad/f/sadf/a")) + + tmp := os.TempDir() + "test/" + tmpFile := tmp + "a" + assert.Equal(t, NotExisted, Exists(tmp)) + assert.Equal(t, NotExisted, Exists(tmpFile)) + + os.MkdirAll(tmp, os.FileMode(0755)) + defer os.RemoveAll(tmp) + os.Create(tmpFile) + assert.Equal(t, ExistDir, Exists(tmp)) + assert.Equal(t, ExistFile, Exists(tmpFile)) +} From ec8a7f8aa8851593b868190cd7e5271a3afae180 Mon Sep 17 00:00:00 2001 From: Jason Joo Date: Mon, 20 Jul 2020 14:12:02 +0800 Subject: [PATCH 03/46] Add test for parser --- cmd/parse_test.go | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 cmd/parse_test.go diff --git a/cmd/parse_test.go b/cmd/parse_test.go new file mode 100644 index 0000000..6ada1a6 --- /dev/null +++ b/cmd/parse_test.go @@ -0,0 +1,33 @@ +package cmd + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestParseCommandName(t *testing.T) { + name, args := ParseCommandName([]string{"-wrong", "test"}) + assert.Empty(t, name) + assert.Equal(t, 2, len(args)) + + name, args = ParseCommandName([]string{}) + assert.Empty(t, name) + assert.Equal(t, 0, len(args)) + + name, args = ParseCommandName([]string{"-h", "config"}) + assert.Equal(t, "help", name) + assert.Equal(t, 1, len(args)) + assert.Equal(t, "config", args[0]) + + name, args = ParseCommandName([]string{"--help", "config"}) + assert.Equal(t, "help", name) + assert.Equal(t, 1, len(args)) + assert.Equal(t, "config", args[0]) + + name, args = ParseCommandName([]string{"config", "-c", "a.yml"}) + assert.Equal(t, "config", name) + assert.Equal(t, 2, len(args)) + assert.Equal(t, "-c", args[0]) + assert.Equal(t, "a.yml", args[1]) +} From c453c265b10200c5623bcf299b6ff25f9fa2a309 Mon Sep 17 00:00:00 2001 From: Jason Joo Date: Mon, 20 Jul 2020 15:08:51 +0800 Subject: [PATCH 04/46] add ignore and fix migrate client reference --- .gitignore | 3 +++ cmd/influxd/run/server.go | 8 -------- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/.gitignore b/.gitignore index 1277a92..479919d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,5 @@ .history/ .DS_Store +cmd/influxd-ctl/influxd-ctl +cmd/influxd/influxd +cmd/metad/metad diff --git a/cmd/influxd/run/server.go b/cmd/influxd/run/server.go index 61e6aee..eda31d6 100644 --- a/cmd/influxd/run/server.go +++ b/cmd/influxd/run/server.go @@ -446,15 +446,7 @@ func (s *Server) appendControllerService(c controller.Config) { srv.MetaClient = s.ClusterMetaClient srv.Node = s.Node srv.TSDBStore = s.TSDBStore - shardCopier := &controller.ShardCopier{ - Manager: controller.NewCopyManager(c.MaxShardCopyTasks), - MetaClient: s.ClusterMetaClient, - TSDBStore: s.TSDBStore, - Node: s.Node, - } - shardCopier.WithLogger(s.Logger) - srv.ShardCopier = shardCopier s.ControllerService = srv s.Services = append(s.Services, srv) } From 479ecdda728c4bff1565c324f991e4cedbd1d122 Mon Sep 17 00:00:00 2001 From: Jason Joo Date: Mon, 17 Aug 2020 10:36:00 +0800 Subject: [PATCH 05/46] add logdir parameter to metad --- cmd/metad/main.go | 30 ++++++++++++++++--- go.mod | 3 +- go.sum | 73 +++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 101 insertions(+), 5 deletions(-) diff --git a/cmd/metad/main.go b/cmd/metad/main.go index efb4a20..e45b4af 100644 --- a/cmd/metad/main.go +++ b/cmd/metad/main.go @@ -3,24 +3,34 @@ package main import ( "flag" "fmt" + "os" + "path/filepath" + "strings" + "github.com/BurntSushi/toml" "github.com/angopher/chronus/raftmeta" imeta "github.com/angopher/chronus/services/meta" "github.com/angopher/chronus/x" "github.com/influxdata/influxdb/logger" "github.com/influxdata/influxdb/services/meta" - "os" + "go.uber.org/zap" + "gopkg.in/natefinch/lumberjack.v2" ) func main() { - configFile := flag.String("config", "", "-config config_file") - flag.Parse() + f := flag.NewFlagSet("metad", flag.ExitOnError) + configFile := f.String("config", "", "Specify config file") + logDir := f.String("logdir", "", "Log to specified directory") + f.Parse(os.Args[1:]) config := raftmeta.NewConfig() if *configFile != "" { x.Check((&config).FromTomlFile(*configFile)) } else { + fmt.Print("Sample configuration:\n\n") toml.NewEncoder(os.Stdout).Encode(&config) + fmt.Println() + f.Usage() return } @@ -31,7 +41,19 @@ func main() { LoggingEnabled: true, }) - log := logger.New(os.Stderr) + var log *zap.Logger + if *logDir != "" { + dir := strings.TrimRight(*logDir, string(filepath.Separator)) + log = logger.New(&lumberjack.Logger{ + Filename: filepath.Join(dir, "metad.log"), + MaxSize: 100, + MaxBackups: 5, + Compress: true, + }) + log.WithOptions() + } else { + log = logger.New(os.Stderr) + } metaCli.WithLogger(log) err := metaCli.Open() diff --git a/go.mod b/go.mod index 48b6a57..ee43298 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,7 @@ require ( github.com/influxdata/usage-client v0.0.0-20160829180054-6d3895376368 github.com/klauspost/compress v1.4.1 // indirect github.com/klauspost/cpuid v1.2.0 // indirect - github.com/klauspost/pgzip v1.2.1 + github.com/klauspost/pgzip v1.2.1 // indirect github.com/pkg/errors v0.8.1 github.com/pkg/profile v1.2.1 // indirect github.com/spf13/cobra v0.0.3 @@ -34,6 +34,7 @@ require ( golang.org/x/text v0.3.0 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c gopkg.in/fatih/pool.v2 v2.0.0 + gopkg.in/natefinch/lumberjack.v2 v2.0.0 labix.org/v2/mgo v0.0.0-20140701140051-000000000287 // indirect launchpad.net/gocheck v0.0.0-20140225173054-000000000087 // indirect ) diff --git a/go.sum b/go.sum index aa0e0e9..6693650 100644 --- a/go.sum +++ b/go.sum @@ -10,44 +10,60 @@ github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/DataDog/datadog-go v0.0.0-20180822151419-281ae9f2d895/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= github.com/Jeffail/gabs v1.1.1/go.mod h1:6xMvQMK4k33lb7GUUpaAPh6nKMmemQeg5d4gn7/bOXc= +github.com/Masterminds/semver v1.4.2 h1:WBLTQ37jOCzSLtXNdoo8bNM8876KhNqOKvrlGITgsTc= github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= +github.com/Masterminds/sprig v2.16.0+incompatible h1:QZbMUPxRQ50EKAq3LFMnxddMu88/EUUG3qmxwtDmPsY= github.com/Masterminds/sprig v2.16.0+incompatible/go.mod h1:y6hNFY5UBTIWBxnzTeuNhlNS5hqE0NB0E6fgfo2Br3o= github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA= github.com/NYTimes/gziphandler v1.0.1/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5/go.mod h1:lmUJ/7eu/Q8D7ML55dXQrVaamCz2vxCfdQBasLZfHKk= +github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/RoaringBitmap/roaring v0.4.16 h1:NholfewybRLOwACgfqfzn/N5xa6keKNs4fP00t0cwLo= github.com/RoaringBitmap/roaring v0.4.16/go.mod h1:8khRDP4HmeXns4xIj9oGrKSz7XTQiJx2zgh7AcNke4w= github.com/SAP/go-hdb v0.13.1/go.mod h1:etBT+FAi1t5k3K3tf5vQTnosgYmhDkRi8jEnQqCnxF0= github.com/SermoDigital/jose v0.9.1/go.mod h1:ARgCUhI1MHQH+ONky/PAtmVHQrP5JlGY0F3poXOp/fA= github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7/go.mod h1:6zEj6s6u/ghQa61ZWa/C2Aw3RkjiTBOix7dkqa1VLIs= +github.com/alecthomas/kingpin v2.2.6+incompatible h1:5svnBTFgJjZvGKyYBtMB0+m5wvrbUHiqye8wRJMlnYI= github.com/alecthomas/kingpin v2.2.6+incompatible/go.mod h1:59OFYbFVLKQKq+mqrL6Rw5bR0c3ACQaawgXx0QYndlE= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc h1:cAKDfWh5VpdgMhJosfJnn5/FoN2SRZ4p7fJNX58YPaU= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf h1:qet1QNfXsQxTZqLG4oE62mJzwPIB8+Tee4RNCL9ulrY= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883 h1:bvNMNQO63//z+xNgfBlViaCIJKLlCJ6/fmUseuG0wVQ= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= +github.com/aokoli/goutils v1.0.1 h1:7fpzNGoJ3VA8qcrm++XEE1QUe0mIwNeLa02Nwq7RDkg= github.com/aokoli/goutils v1.0.1/go.mod h1:SijmP0QR8LtwsmDs8Yii5Z/S4trXFGFC2oO5g9DP+DQ= github.com/apache/arrow/go/arrow v0.0.0-20181031164735-a56c009257a7/go.mod h1:GjvccvtI06FGFvRU1In/maF7tKp3h7GBV9Sexo5rNPM= github.com/apache/arrow/go/arrow v0.0.0-20181217213538-e9ed591db9cb h1:p6xQwsjxRtuIrUDjGAFuro04BO0GNJ9V2troYRY8kmQ= github.com/apache/arrow/go/arrow v0.0.0-20181217213538-e9ed591db9cb/go.mod h1:GjvccvtI06FGFvRU1In/maF7tKp3h7GBV9Sexo5rNPM= +github.com/apex/log v1.1.0 h1:J5rld6WVFi6NxA6m8GJ1LJqu3+GiTFIt3mYv27gdQWI= github.com/apex/log v1.1.0/go.mod h1:yA770aXIDQrhVOIGurT/pVdfCpSq1GQV/auzMN5fzvY= +github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da h1:8GUt8eRujhVEGZFFEjBj46YV4rDjvGrNxb0KMWYkL2I= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/aws/aws-sdk-go v1.15.59/go.mod h1:E3/ieXAlvM0XWO57iftYVDLLvQ824smPP3ATZkfNZeM= +github.com/aws/aws-sdk-go v1.15.64 h1:xI5HhxebTF+jVqVOraUDqI3kr24n+yTvslwZCo3OhGA= github.com/aws/aws-sdk-go v1.15.64/go.mod h1:E3/ieXAlvM0XWO57iftYVDLLvQ824smPP3ATZkfNZeM= +github.com/benbjohnson/tmpl v1.0.0 h1:T5QPGJD0W6JJxyEEAlVnX3co/IkUrfHen1/42nlgAHo= github.com/benbjohnson/tmpl v1.0.0/go.mod h1:igT620JFIi44B6awvU9IsDhR77IXWtFigTLil/RPdps= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932/go.mod h1:NOuUCSz6Q9T7+igc/hlvDOUdtWKryOrtFyIVABv/p7k= +github.com/blakesmith/ar v0.0.0-20150311145944-8bd4349a67f2 h1:oMCHnXa6CCCafdPDbMh/lWRhRByN0VFLvv+g+ayx1SI= github.com/blakesmith/ar v0.0.0-20150311145944-8bd4349a67f2/go.mod h1:PkYb9DJNAwrSvRx5DYA+gUcOIgTGVMNkfSCbZM8cWpI= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= github.com/bmizerany/pat v0.0.0-20170815010413-6226ea591a40 h1:y4B3+GPxKlrigF1ha5FFErxK+sr6sWxQovRMzwMhejo= github.com/bmizerany/pat v0.0.0-20170815010413-6226ea591a40/go.mod h1:8rLXio+WjiTceGBHIoTvn60HIbs7Hm7bcHjyrSqYB9c= +github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4= github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= github.com/bouk/httprouter v0.0.0-20160817010721-ee8b3818a7f5/go.mod h1:CDReaxg1cmLrtcasZy43l4EYPAknXLiQSrb7tLw5zXM= github.com/c-bata/go-prompt v0.2.2/go.mod h1:VzqtzE2ksDBcdln8G7mk2RX9QyGjH+OVqOCSiVIqS34= +github.com/caarlos0/ctrlc v1.0.0 h1:2DtF8GSIcajgffDFJzyG15vO+1PuBWOMUdFut7NnXhw= github.com/caarlos0/ctrlc v1.0.0/go.mod h1:CdXpj4rmq0q/1Eb44M9zi2nKB0QraNKuRGYGrrHhcQw= +github.com/campoy/unique v0.0.0-20180121183637-88950e537e7e h1:V9a67dfYqPLAvzk5hMQOXYJlZ4SLIXgyKIE+ZiHzgGQ= github.com/campoy/unique v0.0.0-20180121183637-88950e537e7e/go.mod h1:9IOqJGCPMSc6E5ydlp5NIonxObaeu/Iub/X03EKPVYo= github.com/cenkalti/backoff v2.0.0+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= @@ -58,6 +74,7 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd h1:qMd81Ts1T2OTKmB4acZcyKaMtRnY5Y44NuXGX2GFJ1w= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/containerd/continuity v0.0.0-20181027224239-bea7585dbfac/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= +github.com/coreos/bbolt v1.3.1-coreos.6 h1:uTXKg9gY70s9jMAKdfljFQcuh4e/BXOM+V+d00KFj3A= github.com/coreos/bbolt v1.3.1-coreos.6/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.12+incompatible h1:pAWNwdf7QiT1zfaWyqCtNZQWCLByQyA3JrSQyuYAqnQ= github.com/coreos/etcd v3.3.12+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= @@ -82,9 +99,11 @@ github.com/docker/docker v0.0.0-20180422163414-57142e89befe/go.mod h1:eEKB0N0r5N github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/duosecurity/duo_api_golang v0.0.0-20181024123116-92fea9203dbc/go.mod h1:UqXY1lYT/ERa4OEAywUqdok1T4RCRdArkhic1Opuavo= +github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/elazarl/go-bindata-assetfs v1.0.0/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo7R71Vk1u8bnjau5yw4= github.com/emirpasic/gods v1.9.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o= +github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= @@ -95,6 +114,7 @@ github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeME github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= github.com/glycerine/go-unsnap-stream v0.0.0-20180323001048-9f0cb55181dd h1:r04MMPyLHj/QwZuMJ5+7tJcBr1AQjpiAK/rZWRrQT7o= github.com/glycerine/go-unsnap-stream v0.0.0-20180323001048-9f0cb55181dd/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE= +github.com/glycerine/goconvey v0.0.0-20180728074245-46e3a41ad493 h1:OTanQnFt0bi5iLFSdbEVA/idR6Q2WhCm+deb7ir2CcM= github.com/glycerine/goconvey v0.0.0-20180728074245-46e3a41ad493/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24= github.com/go-ldap/ldap v2.5.1+incompatible/go.mod h1:qfd9rJvER9Q0/D/Sqn1DfHRoBp40uXYvFoEVrNEPqRc= github.com/go-sql-driver/mysql v1.4.0 h1:7LxgVwFb2hIQtMm87NdgAVfXjnt4OePseqT1tKx+opk= @@ -115,15 +135,22 @@ github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y github.com/golang/snappy v0.0.0-20170215233205-553a64147049/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db h1:woRePGFeVFfLKN/pOkfl+p/TAqKOfFu+7KPlMVpok/w= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c h1:964Od4U6p2jUkFxvCydnIczKteheJEzHRToSGK3Bnlw= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY= github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= +github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.0.0 h1:b4Gk+7WdP/d3HZH8EJsZpvV7EtDOgaZLtnaNGIu1adA= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/goreleaser/goreleaser v0.94.0 h1:2CFMxMTLODjYfNOx2sADNzpgCwH9ltMqvQYtj+ntK1Q= github.com/goreleaser/goreleaser v0.94.0/go.mod h1:OjbYR2NhOI6AEUWCowMSBzo9nP1aRif3sYtx+rhp+Zo= +github.com/goreleaser/nfpm v0.9.7 h1:h8RQMDztu6cW7b0/s4PGbdeMYykAbJG0UMXaWG5uBMI= github.com/goreleaser/nfpm v0.9.7/go.mod h1:F2yzin6cBAL9gb+mSiReuXdsfTrOQwDMsuSpULof+y4= github.com/gotestyourself/gotestyourself v2.2.0+incompatible/go.mod h1:zZKM6oeNM8k+FRljX1mnzVYeS8wiGgQyvST1/GafPbY= github.com/grpc-ecosystem/grpc-gateway v1.6.2/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= @@ -133,20 +160,25 @@ github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brv github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-hclog v0.0.0-20180709165350-ff2cf002a8dd/go.mod h1:9bjs9uLqI8l75knNv3lV1kA55veR+WUPSiKIWcQHudI= github.com/hashicorp/go-hclog v0.0.0-20181001195459-61d530d6c27f/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= +github.com/hashicorp/go-immutable-radix v1.0.0 h1:AKDB1HM5PWEA7i4nhcpwOrO2byshxBjXVn/J/3+z5/0= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-memdb v0.0.0-20181108192425-032f93b25bec/go.mod h1:kbfItVoBJwCfKXDXN4YoAXjxcFVZ7MRrJzyTX6H4giE= +github.com/hashicorp/go-msgpack v0.0.0-20150518234257-fa3f63826f7c h1:BTAbnbegUIMB6xmQCwWE8yRzbA4XSpnZY5hvRJC188I= github.com/hashicorp/go-msgpack v0.0.0-20150518234257-fa3f63826f7c/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= github.com/hashicorp/go-plugin v0.0.0-20181030172320-54b6ff97d818/go.mod h1:Ft7ju2vWzhO0ETMKUVo12XmXmII6eSUS4rsPTkY/siA= github.com/hashicorp/go-retryablehttp v0.5.0/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= github.com/hashicorp/go-rootcerts v0.0.0-20160503143440-6bb64b370b90/go.mod h1:o4zcYY1e0GEZI6eSEr+43QDYmuGglw1qSO6qdHUHCgg= github.com/hashicorp/go-sockaddr v0.0.0-20180320115054-6d291a969b86/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= +github.com/hashicorp/go-uuid v1.0.0 h1:RS8zrF7PhGwyNPOtxSClXXj9HA8feRnJzgnI1RJCSnM= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-version v1.0.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/golang-lru v0.5.0 h1:CL2msUPvZTLb5O648aiLNJw3hnBxN2+1Jq8rCOH9wdo= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/memberlist v0.1.0/go.mod h1:ncdBp14cuox2iFOq3kDiquKU6fqsTBc3W6JvZwjxxsE= +github.com/hashicorp/raft v1.0.0 h1:htBVktAOtGs4Le5Z7K8SF5H2+oWsQFYVmOgH5loro7Y= github.com/hashicorp/raft v1.0.0/go.mod h1:DVSAWItjLjTOkVbSpWQ0j0kUADIvDaCtBxIcbNAQLkI= github.com/hashicorp/serf v0.8.1/go.mod h1:h/Ru6tmZazX7WO/GDmwdpS975F019L4t5ng5IgwbNrE= github.com/hashicorp/vault v0.11.5/go.mod h1:KfSyffbKxoVyspOdlaGVjIuwLobi07qD1bAbosPMpP0= @@ -154,9 +186,12 @@ github.com/hashicorp/vault-plugin-secrets-kv v0.0.0-20181106190520-2236f141171e/ github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM= github.com/hashicorp/yamux v0.0.0-20181012175058-2f1d1f20f75d/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/huandu/xstrings v1.0.0 h1:pO2K/gKgKaat5LdpAhxhluX2GPQMaI3W5FUz/I/UnWk= github.com/huandu/xstrings v1.0.0/go.mod h1:4qWG/gcEcfX4z/mBDHJ++3ReCw9ibxbsNJbcucJdbSo= github.com/imdario/mergo v0.3.4/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28= github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/influxdata/flux v0.13.0 h1:iWNTxfR8m8LFYNAjSXAYMdxMKHqsBxd3xHlZ7rpl1KA= github.com/influxdata/flux v0.13.0/go.mod h1:81jeDcHVn1rN5uj9aQ81S72Q8ol8If7N0zM0G8TnxTE= @@ -179,14 +214,17 @@ github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i github.com/jefferai/jsonx v0.0.0-20160721235117-9cc31c3135ee/go.mod h1:N0t2vlmpe8nyZB5ouIbJQPDSR+mH6oe7xHB9VZHSUzM= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jsternberg/zap-logfmt v1.2.0 h1:1v+PK4/B48cy8cfQbxL4FmmNZrjnIMr2BsnyEmXqv2o= github.com/jsternberg/zap-logfmt v1.2.0/go.mod h1:kz+1CUmCutPWABnNkOu9hOHKdT2q3TDYCcsFy9hpqb0= +github.com/jtolds/gls v4.2.1+incompatible h1:fSuqC+Gmlu6l/ZYAoZzx2pyucC8Xza35fpRVWLVmUEE= github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/jwilder/encoding v0.0.0-20170811194829-b4e1701a28ef h1:2jNeR4YUziVtswNP9sEFAI913cVrzH85T+8Q6LpYbT0= github.com/jwilder/encoding v0.0.0-20170811194829-b4e1701a28ef/go.mod h1:Ct9fl0F6iIOGgxJ5npU/IUOhOhqlVrGjyIZc8/MagT0= github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k= +github.com/kevinburke/go-bindata v3.11.0+incompatible h1:RcC+GJNmrBHbGaOpQ9MBD8z22rdzlIm0esDRDkyxd4s= github.com/kevinburke/go-bindata v3.11.0+incompatible/go.mod h1:/pEEZ72flUW2p0yi30bslSp9YqD9pysLxunQDdb2CPM= github.com/kevinburke/ssh_config v0.0.0-20180830205328-81db2a75821e/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= github.com/keybase/go-crypto v0.0.0-20181031135447-f919bfda4fc1/go.mod h1:ghbZscTyKdM07+Fw3KSi0hcJm+AlEUWj8QLlPtijN/M= @@ -199,24 +237,29 @@ github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgo github.com/klauspost/pgzip v1.2.1 h1:oIPZROsWuPHpOdMVWLuJZXwgjhrW8r1yEX8UqMyeNHM= github.com/klauspost/pgzip v1.2.1/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/lib/pq v1.0.0 h1:X5PMW56eZitiTeO7tKzZxFCSpbFZJtkMMooicw2us9A= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/mattn/go-colorable v0.0.9 h1:UVL0vNpWh04HeJXV0KLcaT7r06gOH2l4OW6ddYRUIY4= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-isatty v0.0.4 h1:bnP0vzxcAdeI1zdubAl5PjU6zsERjGZb7raWodagDYs= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-tty v0.0.0-20180907095812-13ff1204f104/go.mod h1:XPvLUNfbS4fJH25nqRHfWLMa1ONC8Amw+mIA639KxkE= github.com/mattn/go-zglob v0.0.0-20171230104132-4959821b4817/go.mod h1:9fxibJccNxU2cnpIKLRRFA7zX7qhkJIQWBb449FYHOo= +github.com/mattn/go-zglob v0.0.0-20180803001819-2ea3427bfa53 h1:tGfIHhDghvEnneeRhODvGYOt305TPwingKt6p90F4MU= github.com/mattn/go-zglob v0.0.0-20180803001819-2ea3427bfa53/go.mod h1:9fxibJccNxU2cnpIKLRRFA7zX7qhkJIQWBb449FYHOo= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/miekg/dns v1.1.1/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= +github.com/mitchellh/go-homedir v1.0.0 h1:vKb8ShqSby24Yrqr/yDYkuFz8d0WUjys40rvnGC8aR0= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-testing-interface v0.0.0-20171004221916-a61a99592b77/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= @@ -224,12 +267,19 @@ github.com/mitchellh/mapstructure v1.0.0/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= +github.com/mna/pigeon v1.0.1-0.20180808201053-bb0192cfc2ae h1:mQO+oxi0kpii/TX+ltfTCFuYkOjEn53JhaOObiMuvnk= github.com/mna/pigeon v1.0.1-0.20180808201053-bb0192cfc2ae/go.mod h1:Iym28+kJVnC1hfQvv5MUtI6AiFFzvQjHcvI4RFTG/04= +github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae h1:VeRdUYdCw49yizlSbMEn2SZ+gT+3IUKx8BqxyQdz+BY= github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg= +github.com/nats-io/gnatsd v1.3.0 h1:+5d80klu3QaJgNbdavVBjWJP7cHd11U2CLnRTFM9ICI= github.com/nats-io/gnatsd v1.3.0/go.mod h1:nqco77VO78hLCJpIcVfygDP2rPGfsEHkGTUk94uh5DQ= +github.com/nats-io/go-nats v1.6.0 h1:FznPwMfrVwGnSCh7JTXyJDRW0TIkD4Tr+M1LPJt9T70= github.com/nats-io/go-nats v1.6.0/go.mod h1:+t7RHT5ApZebkrQdnn6AhQJmhJJiKAvJUio1PiiCtj0= +github.com/nats-io/go-nats-streaming v0.4.0 h1:00wOBnTKzZGvQOFRSxj18kUm4X2TvXzv8LS0skZegPc= github.com/nats-io/go-nats-streaming v0.4.0/go.mod h1:gfq4R3c9sKAINOpelo0gn/b9QDMBZnmrttcsNF+lqyo= +github.com/nats-io/nats-streaming-server v0.11.2 h1:UCqZbfXUKs9Ejw7KiNaFZEbbiVbK7uA8jbK2TsdGbqg= github.com/nats-io/nats-streaming-server v0.11.2/go.mod h1:RyqtDJZvMZO66YmyjIYdIvS69zu/wDAkyNWa8PIUa5c= +github.com/nats-io/nuid v1.0.0 h1:44QGdhbiANq8ZCbUkdn6W5bqtg+mHuDE4wOUuxxndFs= github.com/nats-io/nuid v1.0.0/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= @@ -242,6 +292,7 @@ github.com/opentracing/opentracing-go v1.0.2 h1:3jA2P6O1F9UOrWVpwrIo17pu01KWvNWg github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/openzipkin/zipkin-go v0.1.3/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8= github.com/ory/dockertest v3.3.2+incompatible/go.mod h1:1vX4m9wsvi00u5bseYwXaSnhNrne+V0E6LAcBILJdPs= +github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c h1:Lgl0gzECD8GnQ5QCWA8o6BtfL6mDH5rQgM4/fX3avOs= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= github.com/pelletier/go-buffruneio v0.2.0/go.mod h1:JkE26KsDizTr40EUHkXVtNPvgGtbSNq5BcowyYOWdKo= @@ -249,6 +300,7 @@ github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/philhofer/fwd v1.0.0 h1:UbZqGr5Y38ApvM/V/jEljVxwocdweyH+vmYvRPBnbqQ= github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= +github.com/pierrec/lz4 v2.0.5+incompatible h1:2xWsjqPFWcplujydGg4WmhC/6fZqK42wMM8aXeqhl0I= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -280,10 +332,14 @@ github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdh github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/segmentio/kafka-go v0.1.0 h1:IXCHG+sXPNiIR5pC/vTEItZduPKu4cnpr85YgxpxlW0= github.com/segmentio/kafka-go v0.1.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfPOCvTvk+EJo= +github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v0.0.0-20180222194500-ef6db91d284a h1:JSvGDIbmil4Ui/dDdFBExb7/cmkNjyX5F97oglmvCDo= github.com/smartystreets/goconvey v0.0.0-20180222194500-ef6db91d284a/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= @@ -315,6 +371,7 @@ github.com/tinylib/msgp v1.0.2 h1:DfdQrzQa7Yh2es9SuLkixqxuXS2SxsdYn0KbdrOGWD8= github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= github.com/tylerb/graceful v1.2.15/go.mod h1:LPYTbOYmUTdabwRt0TGhLllQ0MUNbs0Y5q1WXJOI9II= +github.com/willf/bitset v1.1.9 h1:GBtFynGY9ZWZmEC9sWuu41/7VBXPFCOAbCbqTflOg9c= github.com/willf/bitset v1.1.9/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= github.com/xanzy/ssh-agent v0.2.0/go.mod h1:0NyE30eGUDliuLEHJgYte/zncp2zdTStcOnWhgSqHD8= github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca h1:1CFlNzQhALwjS9mBAUkycX616GzgsuYUOCHA5+HSlXI= @@ -332,6 +389,7 @@ go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.5.0 h1:KCa4XfM8CWFCpxXRGok+Q0SS/0XBhMDbHHGABQLvD2A= go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= +go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee h1:0mgffUl7nfd+FpvXMVz4IDEaUSmT1ysygQC7qYo7sG4= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v1.9.1 h1:XCJQEf3W6eZaVwhRBof6ImoYGJSITeKWsyeh3HFu/5o= go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= @@ -345,10 +403,12 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529 h1:iMGN4xG0cnqj3t+zOM8wUB0BiPKHEwSxEZCvzcbZuvk= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20181112044915-a3060d491354 h1:6UAgZ8309zQ9+1iWkHzfszFguqzOdHGyGkd1HmhJ+UE= golang.org/x/exp v0.0.0-20181112044915-a3060d491354/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20181217174547-8f45f776aaf1/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -365,6 +425,7 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwL golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890 h1:uESlIz09WIHT2I+pasSXcpLYqYK8wHcdCetU3VuMBJE= golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -397,14 +458,18 @@ golang.org/x/tools v0.0.0-20181221154417-3ad2d988d5e2/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5 h1:hKsoRgsbwY1NafxrwTs+k64bikrLBkAgPir1TNCj3Zs= golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gonum.org/v1/gonum v0.0.0-20181121035319-3f7ecaa7e8ca h1:PupagGYwj8+I4ubCxcmcBRk3VlUWtTg5huQpZR9flmE= gonum.org/v1/gonum v0.0.0-20181121035319-3f7ecaa7e8ca/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= +gonum.org/v1/netlib v0.0.0-20181029234149-ec6d1f5cefe6 h1:4WsZyVtkthqrHTbDCJfiTs8IWNYE4uvsSDgaV6xpp+o= gonum.org/v1/netlib v0.0.0-20181029234149-ec6d1f5cefe6/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= google.golang.org/api v0.0.0-20181021000519-a2651947f503/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= google.golang.org/api v0.0.0-20181220000619-583d854617af/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.3.0 h1:FBSsiFRMz3LBeXIomRnVzrQwSDj4ibvcRexLG0LZGQk= google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20181016170114-94acd270e44e/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= @@ -417,6 +482,7 @@ google.golang.org/grpc v1.17.0 h1:TRJYBgMclJvGYn2rIMjj+h9KtMt5r1Ij7ODVRIZkwhk= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= gopkg.in/asn1-ber.v1 v1.0.0-20170511165959-379148ca0225/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fatih/pool.v2 v2.0.0 h1:xIFeWtxifuQJGk/IEPKsTduEKcKvPmhoiVDGpC40nKg= @@ -425,11 +491,14 @@ gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMy gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ldap.v2 v2.5.1/go.mod h1:oI0cpe/D7HRtBQl8aTg+ZmzFUAvu4lsv3eLXMLGFxWk= gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= +gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= +gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/robfig/cron.v2 v2.0.0-20150107220207-be2e0b0deed5/go.mod h1:hiOFpYm0ZJbusNj2ywpbrXowU3G8U6GIQzqn2mw1UIE= gopkg.in/src-d/go-billy.v4 v4.2.1/go.mod h1:tm33zBoOwxjYHZIE+OV8bxTWFMJLrconzFMd38aARFk= gopkg.in/src-d/go-git-fixtures.v3 v3.1.1/go.mod h1:dLBcvytrw/TYZsNTWCnkNF2DSIlzWYqTe3rJR56Ac7g= gopkg.in/src-d/go-git.v4 v4.8.1/go.mod h1:Vtut8izDyrM8BUVQnzJ+YvmNcem2J89EmfZYCkLokZk= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/vmihailenco/msgpack.v2 v2.9.1 h1:kb0VV7NuIojvRfzwslQeP3yArBqJHW9tOl4t38VS1jM= gopkg.in/vmihailenco/msgpack.v2 v2.9.1/go.mod h1:/3Dn1Npt9+MYyLpYYXjInO/5jvMLamn+AEGwNEOatn8= gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -439,6 +508,10 @@ gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81 honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20180920025451-e3ad64cb4ed3/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20181108184350-ae8f1f9103cc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXeM= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.4 h1:UoveltGrhghAA7ePc+e+QYDHXrBps2PqFZiHkGR/xK8= +labix.org/v2/mgo v0.0.0-20140701140051-000000000287 h1:L0cnkNl4TfAXzvdrqsYEmxOHOCv2p5I3taaReO8BWFs= labix.org/v2/mgo v0.0.0-20140701140051-000000000287/go.mod h1:Lg7AYkt1uXJoR9oeSZ3W/8IXLdvOfIITgZnommstyz4= +launchpad.net/gocheck v0.0.0-20140225173054-000000000087 h1:Izowp2XBH6Ya6rv+hqbceQyw/gSGoXfH/UPoTGduL54= launchpad.net/gocheck v0.0.0-20140225173054-000000000087/go.mod h1:hj7XX3B/0A+80Vse0e+BUHsHMTEhd0O4cpUHr/e/BUM= From 080db48ebfd5bc840fe1e41dfbfe5cbf17d3f43f Mon Sep 17 00:00:00 2001 From: Jason Joo Date: Mon, 24 Aug 2020 18:48:25 +0800 Subject: [PATCH 06/46] - Add support logging into directory to metad - Add api interfaces to get status of metad cluster Signed-off-by: Jason Joo --- cmd/influxd/run/command.go | 32 ++++--- cmd/influxd/run/server.go | 4 +- cmd/metad/main.go | 1 - coordinator/cluster_meta_client.go | 24 ++--- raftmeta/meta_service.go | 30 ++++++ raftmeta/node.go | 40 ++++---- raftmeta/transport.go | 141 +++++++++++++++++++++++++++-- 7 files changed, 215 insertions(+), 57 deletions(-) diff --git a/cmd/influxd/run/command.go b/cmd/influxd/run/command.go index d6c3dce..61e9679 100644 --- a/cmd/influxd/run/command.go +++ b/cmd/influxd/run/command.go @@ -6,17 +6,18 @@ import ( "fmt" "io" "io/ioutil" - "log" "net/http" _ "net/http/pprof" "os" "path/filepath" "runtime" "strconv" + "strings" "time" "github.com/influxdata/influxdb/logger" "go.uber.org/zap" + "gopkg.in/natefinch/lumberjack.v2" ) const logo = ` @@ -88,8 +89,18 @@ func (cmd *Command) Run(args ...string) error { return fmt.Errorf("%s. To generate a valid configuration file run `influxd config > influxdb.generated.conf`", err) } - var logErr error - if cmd.Logger, logErr = config.Logging.New(cmd.Stderr); logErr != nil { + if options.LogDir != "" { + dir := strings.TrimRight(options.LogDir, string(filepath.Separator)) + cmd.Logger = logger.New(&lumberjack.Logger{ + Filename: filepath.Join(dir, "influxd.log"), + MaxSize: 100, + MaxBackups: 5, + Compress: true, + }) + } else { + cmd.Logger, _ = config.Logging.New(cmd.Stderr) + } + if cmd.Logger == nil { // assign the default logger cmd.Logger = logger.New(cmd.Stderr) } @@ -115,11 +126,6 @@ func (cmd *Command) Run(args ...string) error { zap.String("version", runtime.Version()), zap.Int("maxprocs", runtime.GOMAXPROCS(0))) - // If there was an error on startup when creating the logger, output it now. - if logErr != nil { - cmd.Logger.Error("Unable to configure logger", zap.Error(logErr)) - } - // Write the PID file. if err := cmd.writePIDFile(options.PIDFile); err != nil { return fmt.Errorf("write pid file: %s", err) @@ -139,11 +145,10 @@ func (cmd *Command) Run(args ...string) error { Branch: cmd.Branch, Time: cmd.BuildTime, } - s, err := NewServer(config, buildInfo) + s, err := NewServer(config, buildInfo, cmd.Logger) if err != nil { return fmt.Errorf("create server: %s", err) } - s.Logger = cmd.Logger s.CPUProfile = options.CPUProfile s.MemProfile = options.MemProfile if err := s.Open(); err != nil { @@ -169,11 +174,10 @@ func (cmd *Command) Close() error { } func (cmd *Command) monitorServerErrors() { - logger := log.New(cmd.Stderr, "", log.LstdFlags) for { select { case err := <-cmd.Server.Err(): - logger.Println(err) + cmd.Logger.Error(err.Error()) case <-cmd.closing: return } @@ -198,6 +202,7 @@ func (cmd *Command) ParseFlags(args ...string) (Options, error) { _ = fs.String("hostname", "", "") fs.StringVar(&options.CPUProfile, "cpuprofile", "", "") fs.StringVar(&options.MemProfile, "memprofile", "", "") + fs.StringVar(&options.LogDir, "logdir", "", "Log to specified directory") fs.Usage = func() { fmt.Fprintln(cmd.Stderr, usage) } if err := fs.Parse(args); err != nil { return Options{}, err @@ -262,6 +267,8 @@ Usage: influxd run [flags] Write CPU profiling information to a file. -memprofile Write memory usage information to a file. + -logdir + Write logs to specified path ` // Options represents the command line options that can be parsed. @@ -270,6 +277,7 @@ type Options struct { PIDFile string CPUProfile string MemProfile string + LogDir string } // GetConfigPath returns the config path from the options. diff --git a/cmd/influxd/run/server.go b/cmd/influxd/run/server.go index eda31d6..eef9142 100644 --- a/cmd/influxd/run/server.go +++ b/cmd/influxd/run/server.go @@ -122,7 +122,7 @@ func updateTLSConfig(into **tls.Config, with *tls.Config) { } // NewServer returns a new instance of Server built from a config. -func NewServer(c *Config, buildInfo *BuildInfo) (*Server, error) { +func NewServer(c *Config, buildInfo *BuildInfo, logger *zap.Logger) (*Server, error) { // First grab the base tls config we will use for all clients and servers tlsConfig, err := c.TLS.Parse() if err != nil { @@ -186,7 +186,7 @@ func NewServer(c *Config, buildInfo *BuildInfo) (*Server, error) { Node: node, - Logger: logger.New(os.Stderr), + Logger: logger, ClusterMetaClient: coordinator.NewMetaClient(c.Meta, c.Coordinator, nodeID), diff --git a/cmd/metad/main.go b/cmd/metad/main.go index e45b4af..26817e8 100644 --- a/cmd/metad/main.go +++ b/cmd/metad/main.go @@ -50,7 +50,6 @@ func main() { MaxBackups: 5, Compress: true, }) - log.WithOptions() } else { log = logger.New(os.Stderr) } diff --git a/coordinator/cluster_meta_client.go b/coordinator/cluster_meta_client.go index 6b3f82c..9cf0eee 100644 --- a/coordinator/cluster_meta_client.go +++ b/coordinator/cluster_meta_client.go @@ -7,6 +7,7 @@ import ( "github.com/influxdata/influxdb/services/meta" "github.com/influxdata/influxql" "go.uber.org/zap" + "golang.org/x/time/rate" imeta "github.com/angopher/chronus/services/meta" ) @@ -70,23 +71,9 @@ func (me *ClusterMetaClient) syncData() error { func (me *ClusterMetaClient) RunSyncLoop() { //sync data first and will signal changes me.syncData() - go func() { - printTicker := time.NewTicker(10 * time.Second) - for { - //for print sync status - select { - case <-printTicker.C: - index, err := me.metaCli.Ping() - if err != nil { - me.Logger.Warn("Ping fail", zap.Error(err)) - continue - } - me.Logger.Info(fmt.Sprintf("index=%d local_index=%d", index, me.cache.Data().Index)) - } - } - }() ticker := time.NewTicker(time.Duration(me.pingIntervalMs) * time.Millisecond) + printLimiter := rate.NewLimiter(0.1, 1) for { select { case <-ticker.C: @@ -104,10 +91,15 @@ func (me *ClusterMetaClient) RunSyncLoop() { } } else if index < me.cache.DataIndex() { me.Logger.Warn(fmt.Sprintf("index:%d < local index:%d", index, me.cache.DataIndex())) + } else { + // normal + if printLimiter.Allow() { + // one log in 10 seconds + me.Logger.Info(fmt.Sprintf("index=%d local_index=%d", index, me.cache.Data().Index)) + } } } } - return } func (me *ClusterMetaClient) CreateDatabase(name string) (*meta.DatabaseInfo, error) { diff --git a/raftmeta/meta_service.go b/raftmeta/meta_service.go index 3d901a6..c877128 100644 --- a/raftmeta/meta_service.go +++ b/raftmeta/meta_service.go @@ -21,6 +21,30 @@ type CommonResp struct { RetMsg string `json:"ret_msg"` } +type NodeStatus struct { + ID uint64 `json:"id"` + Addr string `json:"addr"` + Vote uint64 `json:"vote"` + Match uint64 `json:"match"` + Next uint64 `json:"next"` + Role string `json:"role"` + Progress string `json:"progress"` +} + +type StatusNodeResp struct { + CommonResp + Status NodeStatus `json:"status"` +} + +type StatusClusterResp struct { + CommonResp + Term uint64 `json:"term"` + Commit uint64 `json:"commit"` + Applied uint64 `json:"applied"` + Leader uint64 `json:"leader"` + Nodes []NodeStatus `json:"nodes"` +} + type MetaService struct { Logger *zap.Logger Addr string @@ -55,6 +79,12 @@ func (s *MetaService) InitRouter() { http.HandleFunc("/update_cluster", func(w http.ResponseWriter, r *http.Request) { s.Node.HandleUpdateCluster(w, r) }) + http.HandleFunc("/status_cluster", func(w http.ResponseWriter, r *http.Request) { + s.Node.HandleStatusCluster(w, r) + }) + http.HandleFunc("/status_node", func(w http.ResponseWriter, r *http.Request) { + s.Node.HandleStatusNode(w, r) + }) initHttpHandler(s) } diff --git a/raftmeta/node.go b/raftmeta/node.go index 887b7a5..09fdcb0 100644 --- a/raftmeta/node.go +++ b/raftmeta/node.go @@ -5,6 +5,12 @@ import ( "encoding/json" "errors" "fmt" + "math/rand" + "os" + "sync" + "sync/atomic" + "time" + "github.com/angopher/chronus/raftmeta/internal" imeta "github.com/angopher/chronus/services/meta" "github.com/angopher/chronus/x" @@ -17,11 +23,6 @@ import ( "github.com/influxdata/influxdb/services/meta" "go.uber.org/zap" "golang.org/x/net/trace" - "math/rand" - "os" - "sync" - "sync/atomic" - "time" ) var errInternalRetry = errors.New("Retry Raft proposal internally") @@ -264,14 +265,14 @@ func (s *RaftNode) leaderChangedNotify() <-chan struct{} { return s.leaderChanged } -func (s *RaftNode) restoreFromSnapshot() { +func (s *RaftNode) restoreFromSnapshot() bool { s.Logger.Info("restore from snapshot") sp, err := s.Storage.Snapshot() x.Checkf(err, "Unable to get existing snapshot") if raft.IsEmptySnap(sp) { s.Logger.Info("empty snapshot. ignore") - return + return false } s.SetConfState(&sp.Metadata.ConfState) s.setAppliedIndex(sp.Metadata.Index) @@ -288,10 +289,18 @@ func (s *RaftNode) restoreFromSnapshot() { err = s.MetaCli.ReplaceData(metaData) x.Checkf(err, "meta cli ReplaceData fail") + + return true +} + +func (s *RaftNode) resetPeersFromConfig() { + for _, peer := range s.Config.Peers { + s.Transport.SetPeer(peer.RaftId, peer.Addr) + } } func (s *RaftNode) InitAndStartNode() { - peers := []raft.Peer{} + peers := make([]raft.Peer, 0, len(s.Config.Peers)) for _, p := range s.Config.Peers { rc := internal.RaftContext{Addr: p.Addr, ID: p.RaftId} data, err := json.Marshal(&rc) @@ -305,8 +314,11 @@ func (s *RaftNode) InitAndStartNode() { if restart { s.Logger.Info("Restarting node") - s.restoreFromSnapshot() + restored := s.restoreFromSnapshot() s.Node = raft.RestartNode(s.RaftConfig) + if !restored { + s.resetPeersFromConfig() + } } else { s.Logger.Info("Starting node") if len(peers) == 0 { @@ -315,7 +327,7 @@ func (s *RaftNode) InitAndStartNode() { x.Check(err) s.Node = raft.StartNode(s.RaftConfig, []raft.Peer{{ID: s.ID, Context: data}}) } else { - rpeers := make([]raft.Peer, 0) + rpeers := make([]raft.Peer, 0, len(s.Config.Peers)) for _, peer := range s.Config.Peers { rpeers = append(rpeers, raft.Peer{ID: uint64(peer.RaftId)}) } @@ -323,13 +335,7 @@ func (s *RaftNode) InitAndStartNode() { //x.Checkf(err, "join peers fail") //s.Logger.Info("join peers success") s.Node = raft.StartNode(s.RaftConfig, rpeers) - - for _, peer := range s.Config.Peers { - if peer.RaftId != s.ID { - s.Transport.SetPeer(peer.RaftId, peer.Addr) - } - } - + s.resetPeersFromConfig() } } } diff --git a/raftmeta/transport.go b/raftmeta/transport.go index 0c6f2fa..672d8fe 100644 --- a/raftmeta/transport.go +++ b/raftmeta/transport.go @@ -5,17 +5,19 @@ import ( "context" "encoding/json" "fmt" - "github.com/angopher/chronus/raftmeta/internal" - "github.com/angopher/chronus/x" - "github.com/coreos/etcd/raft" - "github.com/coreos/etcd/raft/raftpb" - "go.uber.org/zap" "io/ioutil" "net" "net/http" "strconv" + "strings" "sync" "time" + + "github.com/angopher/chronus/raftmeta/internal" + "github.com/angopher/chronus/x" + "github.com/coreos/etcd/raft" + "github.com/coreos/etcd/raft/raftpb" + "go.uber.org/zap" ) type Transport struct { @@ -117,6 +119,7 @@ func (s *RaftNode) HandleUpdateCluster(w http.ResponseWriter, r *http.Request) { resp := &CommonResp{} resp.RetCode = -1 resp.RetMsg = "fail" + w.Header().Set("Content-Type", "application/json; charset=utf-8") defer WriteResp(w, &resp) var err error @@ -163,6 +166,89 @@ func (s *RaftNode) HandleUpdateCluster(w http.ResponseWriter, r *http.Request) { resp.RetMsg = "ok" } +func (s *RaftNode) HandleStatusNode(w http.ResponseWriter, r *http.Request) { + resp := &StatusNodeResp{} + resp.RetCode = -1 + resp.RetMsg = "fail" + w.Header().Set("Content-Type", "application/json; charset=utf-8") + defer WriteResp(w, &resp) + + peers := s.Transport.ClonePeers() + status := s.Node.Status() + resp.Status.ID = status.ID + resp.Status.Vote = status.Vote + resp.Status.Match = status.Applied + resp.Status.Next = resp.Status.Match + 1 + resp.Status.Role = strings.Replace(status.RaftState.String(), "State", "", 1) + if addr, ok := peers[status.ID]; ok { + resp.Status.Addr = addr + } + resp.RetCode = 0 + resp.RetMsg = "ok" +} + +func (s *RaftNode) HandleStatusCluster(w http.ResponseWriter, r *http.Request) { + resp := &StatusClusterResp{} + resp.RetCode = -1 + resp.RetMsg = "fail" + w.Header().Set("Content-Type", "application/json; charset=utf-8") + defer WriteResp(w, &resp) + + peers := s.Transport.ClonePeers() + leader := s.Node.Status().Lead + if leader != s.ID { + // forward + if r.URL.Query().Get("forward") == "1" { + resp.RetMsg = "Error forwarding" + return + } + if addr, ok := peers[leader]; ok { + r, err := forwardStatusCluster(addr) + if err != nil { + resp.RetMsg = err.Error() + } else { + resp = r + } + } + return + } + nodes := make([]NodeStatus, 0, len(peers)) + status := s.Node.Status() + prs := status.Progress + for id, addr := range peers { + idx := len(nodes) + nodes = append(nodes, NodeStatus{ + ID: id, + Addr: addr, + Match: 0, + Next: 0, + Role: "Unreachable", + Progress: "Unreachable", + }) + + nodeStatus, err := statusPeer(addr) + if err != nil { + continue + } + nodes[idx].Role = nodeStatus.Role + nodes[idx].Vote = nodeStatus.Vote + + if pr, ok := prs[id]; ok { + nodes[idx].Match = pr.Match + nodes[idx].Next = pr.Next + nodes[idx].Progress = strings.Replace(pr.State.String(), "ProgressState", "", 1) + } + } + + resp.RetCode = 0 + resp.RetMsg = "ok" + resp.Nodes = nodes + resp.Applied = status.Applied + resp.Commit = status.Commit + resp.Leader = status.Lead + resp.Term = status.Term +} + func (s *RaftNode) HandleMessage(w http.ResponseWriter, r *http.Request) { resp := &CommonResp{} resp.RetCode = 0 @@ -181,10 +267,10 @@ func (s *RaftNode) HandleMessage(w http.ResponseWriter, r *http.Request) { s.RecvRaftRPC(context.Background(), msg) } -func Request(url string, data []byte) error { +func postRequest(url string, data []byte) ([]byte, error) { req, err := http.NewRequest("POST", url, bytes.NewReader(data)) if err != nil { - return err + return nil, err } req.Header.Set("Content-Type", "application/json") req.Header.Set("Connection", "close") @@ -205,16 +291,53 @@ func Request(url string, data []byte) error { res, err := client.Do(req) if err != nil { - return err + return nil, err } defer res.Body.Close() resData, err := ioutil.ReadAll(res.Body) + if err != nil { + return nil, err + } + return resData, nil +} + +func forwardStatusCluster(addr string) (*StatusClusterResp, error) { + data, err := postRequest(fmt.Sprint("http://", addr, "/status_cluster?forward=1"), nil) + if err != nil { + return nil, err + } + + resp := &StatusClusterResp{} + resp.RetCode = -1 + resp.RetMsg = "fail" + err = json.Unmarshal(data, resp) + return resp, nil +} + +func statusPeer(addr string) (*NodeStatus, error) { + data, err := postRequest(fmt.Sprint("http://", addr, "/status_node"), nil) + if err != nil { + return nil, err + } + + resp := &StatusNodeResp{} + resp.RetCode = -1 + resp.RetMsg = "fail" + err = json.Unmarshal(data, resp) + if resp.RetCode != 0 { + return nil, fmt.Errorf("fail. err:%s", resp.RetMsg) + } + return &resp.Status, nil +} + +func Request(url string, data []byte) error { + data, err := postRequest(url, data) if err != nil { return err } resp := &CommonResp{RetCode: -1} - err = json.Unmarshal(resData, resp) + err = json.Unmarshal(data, resp) if resp.RetCode != 0 { return fmt.Errorf("fail. err:%s", resp.RetMsg) } From 42b5ec4515072bdd850d23b73cf25742d863cf5f Mon Sep 17 00:00:00 2001 From: Jason Joo Date: Tue, 25 Aug 2020 13:25:43 +0800 Subject: [PATCH 07/46] Complete metad-ctl, support add/update/remove/status on cluster Signed-off-by: Jason Joo --- cmd/metad-ctl/.gitignore | 1 + cmd/metad-ctl/cmds/parse.go | 57 ++++++++++++ cmd/metad-ctl/cmds/status.go | 74 +++++++++++++++ cmd/metad-ctl/cmds/update.go | 147 ++++++++++++++++++++++++++++++ cmd/metad-ctl/cmds/var.go | 3 + cmd/metad-ctl/main.go | 31 +++++++ cmd/metad-ctl/util/request.go | 77 ++++++++++++++++ cmd/metad-ctl/util/string.go | 22 +++++ cmd/metad-ctl/util/string_test.go | 31 +++++++ go.mod | 2 + go.sum | 3 + raftmeta/transport.go | 75 ++++++++++----- 12 files changed, 502 insertions(+), 21 deletions(-) create mode 100644 cmd/metad-ctl/.gitignore create mode 100644 cmd/metad-ctl/cmds/parse.go create mode 100644 cmd/metad-ctl/cmds/status.go create mode 100644 cmd/metad-ctl/cmds/update.go create mode 100644 cmd/metad-ctl/cmds/var.go create mode 100644 cmd/metad-ctl/main.go create mode 100644 cmd/metad-ctl/util/request.go create mode 100644 cmd/metad-ctl/util/string.go create mode 100644 cmd/metad-ctl/util/string_test.go diff --git a/cmd/metad-ctl/.gitignore b/cmd/metad-ctl/.gitignore new file mode 100644 index 0000000..3cf3346 --- /dev/null +++ b/cmd/metad-ctl/.gitignore @@ -0,0 +1 @@ +metad-ctl diff --git a/cmd/metad-ctl/cmds/parse.go b/cmd/metad-ctl/cmds/parse.go new file mode 100644 index 0000000..bb1931a --- /dev/null +++ b/cmd/metad-ctl/cmds/parse.go @@ -0,0 +1,57 @@ +package cmds + +import ( + "encoding/json" + "errors" + "strconv" + "strings" + + "github.com/angopher/chronus/raftmeta" +) + +func parseNodeAddr(arg string) (string, error) { + if len(arg) < 6 || !strings.Contains(arg, ":") { + return "", errors.New("Incorrect address, should be ip:port") + } + return arg, nil +} + +func parseNodeId(arg string) (uint64, error) { + id, err := strconv.ParseUint(arg, 10, 64) + if err != nil { + return 0, err + } + if id < 1 { + err = errors.New("Node id should be positive") + return 0, err + } + return id, nil +} + +func parseNodeIdAndAddr(arg0, arg1 string) (id uint64, addr string, err error) { + id, err = parseNodeId(arg0) + if err != nil { + return + } + + addr, err = parseNodeAddr(arg1) + if err != nil { + return + } + + return +} + +func processResponse(data []byte) error { + resp := raftmeta.CommonResp{} + err := json.Unmarshal(data, &resp) + if err != nil { + return err + } + + if resp.RetCode != 0 { + return errors.New(resp.RetMsg) + } + + return nil +} diff --git a/cmd/metad-ctl/cmds/status.go b/cmd/metad-ctl/cmds/status.go new file mode 100644 index 0000000..a9688c7 --- /dev/null +++ b/cmd/metad-ctl/cmds/status.go @@ -0,0 +1,74 @@ +package cmds + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "sort" + + "github.com/angopher/chronus/cmd/metad-ctl/util" + "github.com/angopher/chronus/raftmeta" + "github.com/fatih/color" + "github.com/urfave/cli/v2" +) + +func dumpStatus(resp *raftmeta.StatusClusterResp) { + color.Set(color.Bold) + color.Green("Cluster:\n") + fmt.Println("Leader:", resp.Leader) + fmt.Println("Term:", resp.Term) + fmt.Println("Committed:", resp.Commit) + fmt.Println("Applied:", resp.Applied) + fmt.Println() + + color.Set(color.Bold) + color.Yellow("Nodes:\n") + sort.Slice(resp.Nodes, func(i, j int) bool { + return resp.Nodes[i].ID < resp.Nodes[j].ID + }) + for _, n := range resp.Nodes { + fmt.Print(util.PadRight(fmt.Sprint(n.ID), 6)) + fmt.Print(" ", util.PadRight(n.Role, 15)) + fmt.Print(util.PadRight(n.Addr, 23)) + fmt.Print(util.PadRight(n.Progress, 15)) + fmt.Print(util.PadRight(fmt.Sprint(n.Match, "=>", n.Next), 20)) + if n.Vote > 0 { + fmt.Print("Vote(", n.Vote, ")") + } + fmt.Println() + } +} + +func StatusCommand() *cli.Command { + return &cli.Command{ + Name: "status", + Usage: "status of cluster", + Description: "Get the cluster status", + Action: clusterStatus, + } +} + +func clusterStatus(ctx *cli.Context) (err error) { + resp := &raftmeta.StatusClusterResp{} + data, err := util.GetRequest(fmt.Sprint("http://", MetadAddress, "/status_cluster")) + if err != nil { + goto ERR + } + + err = json.Unmarshal(data, resp) + if err != nil { + goto ERR + } + if resp.RetCode != 0 { + err = errors.New(resp.RetMsg) + goto ERR + } + + dumpStatus(resp) + return nil + +ERR: + fmt.Fprintln(os.Stderr, err.Error()) + return err +} diff --git a/cmd/metad-ctl/cmds/update.go b/cmd/metad-ctl/cmds/update.go new file mode 100644 index 0000000..4765057 --- /dev/null +++ b/cmd/metad-ctl/cmds/update.go @@ -0,0 +1,147 @@ +package cmds + +import ( + "errors" + "fmt" + "net/url" + "os" + + "github.com/angopher/chronus/cmd/metad-ctl/util" + "github.com/fatih/color" + "github.com/urfave/cli/v2" +) + +func AddCommand() *cli.Command { + return &cli.Command{ + Name: "add", + Usage: "Add node to cluster", + Description: "Introdue new node to cluster should follow two phases operation:\n 1. Add node to configuration using `metad-ctl add`\n 2. Boot up new node with confugration up to date", + ArgsUsage: " ", + Action: clusterAdd, + } +} + +func UpdateCommand() *cli.Command { + return &cli.Command{ + Name: "update", + Usage: "Update address of node in cluster", + Description: "Update address of a node which is already in cluster should follow two phases operation:\n 1. Stop the node\n 2. Update address\n 3. Boot up node with confugration up to date", + ArgsUsage: " ", + Action: clusterUpdate, + } +} + +func RemoveCommand() *cli.Command { + return &cli.Command{ + Name: "remove", + Usage: "Remove node from cluster", + Description: "Remove specified node from config.", + ArgsUsage: "", + Action: clusterRemove, + } +} + +func clusterAdd(ctx *cli.Context) (err error) { + var ( + id uint64 + addr string + data []byte + ) + if ctx.Args().Len() < 2 { + err = errors.New("Please specify node-id and address") + goto ERR + } + id, addr, err = parseNodeIdAndAddr(ctx.Args().Get(0), ctx.Args().Get(1)) + if err != nil { + goto ERR + } + + data, err = util.PostRequestJSON(fmt.Sprint("http://", MetadAddress, "/update_cluster?op=add"), map[string]interface{}{ + "ID": id, + "Addr": addr, + }) + if err != nil { + goto ERR + } + err = processResponse(data) + if err != nil { + goto ERR + } + + color.Green("Success") + return nil + +ERR: + fmt.Fprint(os.Stderr, "ERR: ", err.Error(), "\n\n") + return err +} + +func clusterUpdate(ctx *cli.Context) (err error) { + var ( + id uint64 + addr string + data []byte + ) + if ctx.Args().Len() < 2 { + err = errors.New("Please specify node-id and address") + goto ERR + } + id, addr, err = parseNodeIdAndAddr(ctx.Args().Get(0), ctx.Args().Get(1)) + if err != nil { + goto ERR + } + + data, err = util.PostRequestJSON(fmt.Sprint("http://", MetadAddress, "/update_cluster?op=update"), map[string]interface{}{ + "ID": id, + "Addr": addr, + }) + if err != nil { + goto ERR + } + err = processResponse(data) + if err != nil { + goto ERR + } + + color.Green("Success") + return nil + +ERR: + fmt.Fprint(os.Stderr, "ERR: ", err.Error(), "\n\n") + return err +} + +func clusterRemove(ctx *cli.Context) (err error) { + var ( + id uint64 + data []byte + ) + + if ctx.Args().Len() < 1 { + err = errors.New("Please specify node-id") + goto ERR + } + + id, err = parseNodeId(ctx.Args().Get(0)) + if err != nil { + goto ERR + } + + data, err = util.PostRequest(fmt.Sprint("http://", MetadAddress, "/update_cluster?op=remove"), url.Values{ + "node_id": {fmt.Sprint(id)}, + }) + if err != nil { + goto ERR + } + err = processResponse(data) + if err != nil { + goto ERR + } + + color.Green("Success") + return nil + +ERR: + fmt.Fprint(os.Stderr, "ERR: ", err.Error(), "\n\n") + return err +} diff --git a/cmd/metad-ctl/cmds/var.go b/cmd/metad-ctl/cmds/var.go new file mode 100644 index 0000000..3dd176b --- /dev/null +++ b/cmd/metad-ctl/cmds/var.go @@ -0,0 +1,3 @@ +package cmds + +var MetadAddress string diff --git a/cmd/metad-ctl/main.go b/cmd/metad-ctl/main.go new file mode 100644 index 0000000..6948983 --- /dev/null +++ b/cmd/metad-ctl/main.go @@ -0,0 +1,31 @@ +package main + +import ( + "os" + + "github.com/angopher/chronus/cmd/metad-ctl/cmds" + "github.com/urfave/cli/v2" +) + +func main() { + app := &cli.App{} + app.Name = "metad-ctl" + app.Usage = "Maintain the metad cluster" + + app.Commands = []*cli.Command{ + cmds.StatusCommand(), + cmds.AddCommand(), + cmds.UpdateCommand(), + cmds.RemoveCommand(), + } + app.Flags = []cli.Flag{ + &cli.StringFlag{ + Name: "metad", + Aliases: []string{"s"}, + Required: true, + Usage: "Node address in cluster, ip:port", + Destination: &cmds.MetadAddress, + }, + } + app.Run(os.Args) +} diff --git a/cmd/metad-ctl/util/request.go b/cmd/metad-ctl/util/request.go new file mode 100644 index 0000000..3e873cc --- /dev/null +++ b/cmd/metad-ctl/util/request.go @@ -0,0 +1,77 @@ +package util + +import ( + "bytes" + "encoding/json" + "io/ioutil" + "net" + "net/http" + "net/url" + "time" +) + +var client = http.Client{ + Transport: &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: (&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + DualStack: true, + }).DialContext, + ForceAttemptHTTP2: true, + MaxIdleConns: 10, + IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + }, +} + +func bodyData(resp *http.Response) ([]byte, error) { + defer resp.Body.Close() + resData, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, err + } + return resData, nil +} + +func execute(req *http.Request) ([]byte, error) { + resp, err := client.Do(req) + if err != nil { + return nil, err + } + return bodyData(resp) +} + +func GetRequest(url string) ([]byte, error) { + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return nil, err + } + + return execute(req) +} + +func PostRequest(urlStr string, data url.Values) ([]byte, error) { + resp, err := client.PostForm(urlStr, data) + if err != nil { + return nil, err + } + + return bodyData(resp) +} + +func PostRequestJSON(url string, obj interface{}) ([]byte, error) { + data, err := json.Marshal(obj) + if err != nil { + return nil, err + } + + req, err := http.NewRequest("POST", url, bytes.NewReader(data)) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "application/json") + + return execute(req) +} diff --git a/cmd/metad-ctl/util/string.go b/cmd/metad-ctl/util/string.go new file mode 100644 index 0000000..ade40f2 --- /dev/null +++ b/cmd/metad-ctl/util/string.go @@ -0,0 +1,22 @@ +package util + +import ( + "fmt" + "strings" +) + +func PadLeft(str string, width int) string { + if len(str) >= width { + return str + } + + return fmt.Sprint(strings.Repeat(" ", width-len(str)), str) +} + +func PadRight(str string, width int) string { + if len(str) >= width { + return str + } + + return fmt.Sprint(str, strings.Repeat(" ", width-len(str))) +} diff --git a/cmd/metad-ctl/util/string_test.go b/cmd/metad-ctl/util/string_test.go new file mode 100644 index 0000000..5705a16 --- /dev/null +++ b/cmd/metad-ctl/util/string_test.go @@ -0,0 +1,31 @@ +package util + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestPadLeft(t *testing.T) { + assert.Equal(t, " 123", PadLeft("123", 6)) + assert.Equal(t, "123", PadLeft("123", 3)) + assert.Equal(t, "123", PadLeft("123", 2)) + assert.Equal(t, "123", PadLeft("123", 0)) + assert.Equal(t, "123", PadLeft("123", -1)) + assert.Equal(t, " 123", PadLeft("123", 4)) + + assert.Equal(t, " ", PadLeft("", 1)) + assert.Equal(t, " ", PadLeft("", 3)) +} + +func TestPadRight(t *testing.T) { + assert.Equal(t, "123 ", PadRight("123", 6)) + assert.Equal(t, "123", PadRight("123", 3)) + assert.Equal(t, "123", PadRight("123", 2)) + assert.Equal(t, "123", PadRight("123", 0)) + assert.Equal(t, "123", PadRight("123", -1)) + assert.Equal(t, "123 ", PadRight("123", 4)) + + assert.Equal(t, " ", PadRight("", 1)) + assert.Equal(t, " ", PadRight("", 3)) +} diff --git a/go.mod b/go.mod index ee43298..86cda32 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,7 @@ require ( github.com/dgraph-io/dgo v0.0.0-20190201214300-d5a1729ba705 // indirect github.com/dgraph-io/dgraph v1.0.11 github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f // indirect + github.com/fatih/color v1.7.0 github.com/gogo/protobuf v1.2.1 github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db github.com/google/go-cmp v0.2.0 @@ -26,6 +27,7 @@ require ( github.com/pkg/profile v1.2.1 // indirect github.com/spf13/cobra v0.0.3 github.com/stretchr/testify v1.5.1 // test + github.com/urfave/cli/v2 v2.2.0 github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca // indirect go.opencensus.io v0.19.0 // indirect go.uber.org/zap v1.15.0 diff --git a/go.sum b/go.sum index 6693650..554912c 100644 --- a/go.sum +++ b/go.sum @@ -371,6 +371,9 @@ github.com/tinylib/msgp v1.0.2 h1:DfdQrzQa7Yh2es9SuLkixqxuXS2SxsdYn0KbdrOGWD8= github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= github.com/tylerb/graceful v1.2.15/go.mod h1:LPYTbOYmUTdabwRt0TGhLllQ0MUNbs0Y5q1WXJOI9II= +github.com/urfave/cli v1.22.4 h1:u7tSpNPPswAFymm8IehJhy4uJMlUuU/GmqSkvJ1InXA= +github.com/urfave/cli/v2 v2.2.0 h1:JTTnM6wKzdA0Jqodd966MVj4vWbbquZykeX1sKbe2C4= +github.com/urfave/cli/v2 v2.2.0/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ= github.com/willf/bitset v1.1.9 h1:GBtFynGY9ZWZmEC9sWuu41/7VBXPFCOAbCbqTflOg9c= github.com/willf/bitset v1.1.9/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= github.com/xanzy/ssh-agent v0.2.0/go.mod h1:0NyE30eGUDliuLEHJgYte/zncp2zdTStcOnWhgSqHD8= diff --git a/raftmeta/transport.go b/raftmeta/transport.go index 672d8fe..d36ca53 100644 --- a/raftmeta/transport.go +++ b/raftmeta/transport.go @@ -18,6 +18,26 @@ import ( "github.com/coreos/etcd/raft" "github.com/coreos/etcd/raft/raftpb" "go.uber.org/zap" + "golang.org/x/time/rate" +) + +var ( + loggingLimiter = rate.NewLimiter(1, 1) + httpClient = http.Client{ + Transport: &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: (&net.Dialer{ + Timeout: 10 * time.Second, + KeepAlive: 30 * time.Second, + DualStack: true, + }).DialContext, + ForceAttemptHTTP2: true, + MaxIdleConns: 100, + IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + }, + } ) type Transport struct { @@ -89,7 +109,7 @@ func (t *Transport) SendMessage(messages []raftpb.Message) { } url := fmt.Sprintf("http://%s/message", addr) err = Request(url, data) - if err != nil { + if err != nil && loggingLimiter.Allow() { t.Logger.Error("Request fail:", zap.Error(err), zap.String("url", url)) } } @@ -123,11 +143,13 @@ func (s *RaftNode) HandleUpdateCluster(w http.ResponseWriter, r *http.Request) { defer WriteResp(w, &resp) var err error + peers := s.Transport.ClonePeers() typ := raftpb.ConfChangeAddNode data := []byte{} var nodeId uint64 op := r.FormValue("op") - if op == "add" || op == "update" { + switch op { + case "add", "update": data, err = ioutil.ReadAll(r.Body) if err != nil { resp.RetMsg = err.Error() @@ -137,11 +159,37 @@ func (s *RaftNode) HandleUpdateCluster(w http.ResponseWriter, r *http.Request) { err = json.Unmarshal(data, &rc) x.Check(err) nodeId = rc.ID - } else if op == "remove" { - typ = raftpb.ConfChangeRemoveNode + + // check addr + for _, addr := range peers { + if addr == rc.Addr { + resp.RetMsg = "specified node address already exists" + return + } + } + + // check node id + switch op { + case "add": + if _, ok := peers[nodeId]; ok { + resp.RetMsg = fmt.Sprintf("specified node id already exists: %d", nodeId) + return + } + case "update": + if _, ok := peers[nodeId]; !ok { + resp.RetMsg = fmt.Sprintf("specified node id doesn't exist: %d", nodeId) + return + } + } + case "remove": nodeId, err = strconv.ParseUint(r.FormValue("node_id"), 10, 64) + if _, ok := peers[nodeId]; !ok { + resp.RetMsg = fmt.Sprintf("unkown node id: %d", nodeId) + return + } + typ = raftpb.ConfChangeRemoveNode x.Check(err) - } else { + default: resp.RetMsg = fmt.Sprintf("unkown op:%s", op) return } @@ -273,23 +321,8 @@ func postRequest(url string, data []byte) ([]byte, error) { return nil, err } req.Header.Set("Content-Type", "application/json") - req.Header.Set("Connection", "close") - - client := http.Client{ - Transport: &http.Transport{ - Dial: func(netw, addr string) (net.Conn, error) { - deadline := time.Now().Add(10 * time.Second) //TODO: timeout from config - c, err := net.DialTimeout(netw, addr, time.Second) - if err != nil { - return nil, err - } - c.SetDeadline(deadline) - return c, nil - }, - }, - } - res, err := client.Do(req) + res, err := httpClient.Do(req) if err != nil { return nil, err } From 40437ca6c80267a46e1719361ff6a55c4f4217f6 Mon Sep 17 00:00:00 2001 From: Jason Joo Date: Fri, 4 Sep 2020 15:30:01 +0800 Subject: [PATCH 08/46] Changes: - Introduce pool between coordinators (remote executors) - ShardWriter will share the pool - Introduce a one way packet to do write testing on connection - Other enhancement Remained - Remote iterators should not close the connection which hardly to detect Signed-off-by: Jason Joo --- cmd/influxd-ctl/action/action.go | 115 ++++- cmd/influxd-ctl/action/util.go | 12 + cmd/influxd-ctl/command/command.go | 268 +++++++---- cmd/influxd-ctl/main.go | 24 +- cmd/influxd/run/server.go | 42 +- coordinator/client_pool.go | 162 ++++++- coordinator/cluster_executor.go | 655 +++++++++++---------------- coordinator/cluster_meta_client.go | 24 +- coordinator/cluster_planning.go | 183 ++++++++ coordinator/cluster_planning_test.go | 139 ++++++ coordinator/config.go | 12 +- coordinator/pool.go | 188 -------- coordinator/remote_executor.go | 188 ++++---- coordinator/service.go | 499 ++++++++++---------- coordinator/shard_writer.go | 99 +--- go.mod | 1 - go.sum | 6 + services/controller/service.go | 321 +++++++------ services/hh/node_processor.go | 3 +- services/hh/service.go | 34 +- services/meta/client.go | 1 + x/echo_server.go | 112 +++++ x/math.go | 21 + x/pool.go | 281 ++++++++++++ x/pool_test.go | 76 ++++ 25 files changed, 2140 insertions(+), 1326 deletions(-) create mode 100644 cmd/influxd-ctl/action/util.go create mode 100644 coordinator/cluster_planning.go create mode 100644 coordinator/cluster_planning_test.go delete mode 100644 coordinator/pool.go create mode 100644 x/echo_server.go create mode 100644 x/math.go create mode 100644 x/pool.go create mode 100644 x/pool_test.go diff --git a/cmd/influxd-ctl/action/action.go b/cmd/influxd-ctl/action/action.go index f344122..8b2dfbb 100644 --- a/cmd/influxd-ctl/action/action.go +++ b/cmd/influxd-ctl/action/action.go @@ -2,14 +2,17 @@ package action import ( "encoding/json" + "errors" "fmt" "io" "net" "strconv" "time" + "github.com/angopher/chronus/cmd/metad-ctl/util" "github.com/angopher/chronus/coordinator" "github.com/angopher/chronus/services/controller" + "github.com/fatih/color" ) func CopyShard(srcAddr, dstAddr, shardID string) error { @@ -54,6 +57,95 @@ func TruncateShards(delay string, addr string) error { return nil } +func ListShard(addr, db, rp string) error { + var req controller.GetShardsRequest + var resp controller.ShardsResponse + respTyp := byte(controller.ResponseShards) + reqTyp := byte(controller.RequestShards) + req.Database = db + req.RetentionPolicy = rp + if err := RequestAndWaitResp(addr, reqTyp, respTyp, req, &resp); err != nil { + return err + } + if resp.Code != 0 { + color.Set(color.Bold) + color.Red("Error: ", resp.Msg, "\n") + return errors.New(resp.Msg) + } + + if resp.Rp == "" { + return errors.New("Specified retention policy could not be found") + } + + color.Set(color.Bold) + color.Green("Retenion Policy [%s]:\n", resp.Rp) + fmt.Println("Replica:", resp.Replica) + fmt.Println("Duration:", resp.Duration) + fmt.Println("Group Duration:", resp.GroupDuration) + fmt.Println() + + color.Set(color.Bold) + color.Green("Groups\n") + color.Yellow(fmt.Sprint( + util.PadRight("GroupId", 10), + util.PadRight("Start", 21), + util.PadRight("End", 21), + util.PadRight("DeletedAt", 21), + util.PadRight("TruncatedAt", 19), + "\n", + )) + for _, g := range resp.Groups { + fmt.Print( + util.PadRight(fmt.Sprint(g.ID), 10), + formatTimeStamp(g.StartTime), + " ", formatTimeStamp(g.EndTime), + " ", formatTimeStamp(g.DeletedAt), + " ", formatTimeStamp(g.TruncatedAt), + "\n", + ) + for _, shard := range g.Shards { + color.Cyan(" |--Shard: %d\tNodes: %v\n", shard.ID, shard.Nodes) + } + if len(g.Shards) == 0 { + fmt.Println("No shard") + } + } + fmt.Println() + return nil +} + +func GetShard(addr, shard string) error { + var req controller.GetShardRequest + var resp controller.ShardResponse + respTyp := byte(controller.ResponseShard) + reqTyp := byte(controller.RequestShard) + id, err := strconv.ParseUint(shard, 10, 64) + if err != nil || id < 1 { + return errors.New("Please specify correct shard id") + } + req.ShardID = id + if err := RequestAndWaitResp(addr, reqTyp, respTyp, req, &resp); err != nil { + return err + } + if resp.Code != 0 { + color.Set(color.Bold) + color.Red("Error: ", resp.Msg, "\n") + return errors.New(resp.Msg) + } + + color.Set(color.Bold) + fmt.Println(color.GreenString("Shard:"), resp.ID) + color.Set(color.Bold) + fmt.Println(color.GreenString("Database:"), resp.DB) + color.Set(color.Bold) + fmt.Println(color.GreenString("Retenion Policy:"), resp.Rp) + color.Set(color.Bold) + fmt.Print(color.GreenString("Nodes: ")) + fmt.Printf("%v\n", resp.Nodes) + fmt.Println() + return nil +} + func CopyShardStatus(addr string) error { var resp controller.CopyShardStatusResponse respTyp := byte(controller.ResponseCopyShardStatus) @@ -62,7 +154,12 @@ func CopyShardStatus(addr string) error { return err } - fmt.Printf("%+v\n", resp) + color.Set(color.Bold) + color.Green("Running Copy Tasks:\n") + for _, t := range resp.Tasks { + fmt.Print(t.ShardID, "\t", t.Database, "\t", t.Rp, "\t", t.CurrentSize, "/", t.TotalSize, "\t", t.Source, "=>", t.Destination, "\n") + } + fmt.Println() return nil } @@ -111,9 +208,9 @@ func RemoveShard(addr, shardID string) error { return nil } -func RemoveDataNode(addr string) error { +func RemoveDataNode(addr, removed_addr string) error { req := &controller.RemoveDataNodeRequest{ - DataNodeAddr: addr, + DataNodeAddr: removed_addr, } var resp controller.RemoveDataNodeResponse @@ -123,6 +220,8 @@ func RemoveDataNode(addr string) error { return err } + color.Set(color.Bold) + color.Green("Result: ") fmt.Println(resp.Msg) return nil } @@ -135,7 +234,12 @@ func ShowDataNodes(addr string) error { return err } - fmt.Printf("msg:%s, data nodes:%+v\n", resp.Msg, resp.DataNodes) + color.Set(color.Bold) + color.Green("Nodes:\n") + for _, n := range resp.DataNodes { + fmt.Print(n.ID, "\thttp://", n.HttpAddr, "\ttcp://", n.TcpAddr, "\n") + } + fmt.Println() return nil } @@ -160,7 +264,7 @@ func DecodeTLV(r io.Reader, expTyp byte, v interface{}) error { return err } if expTyp != typ { - return fmt.Errorf("invalid type, exp: %s, got: %s", expTyp, typ) + return fmt.Errorf("invalid type, exp: %d, got: %d", expTyp, typ) } buf, err := coordinator.ReadLV(r) @@ -185,6 +289,7 @@ func Dial(addr string) (net.Conn, error) { conn.Close() return nil, err } + conn.SetDeadline(time.Time{}) return conn, nil } diff --git a/cmd/influxd-ctl/action/util.go b/cmd/influxd-ctl/action/util.go new file mode 100644 index 0000000..ac052d3 --- /dev/null +++ b/cmd/influxd-ctl/action/util.go @@ -0,0 +1,12 @@ +package action + +import "time" + +const ( + TIME_FORMAT = "2006-01-02 15:04:05" +) + +func formatTimeStamp(millis int64) string { + t := time.Unix(millis/1000, (millis%1000)*1e6) + return t.Local().Format(TIME_FORMAT) +} diff --git a/cmd/influxd-ctl/command/command.go b/cmd/influxd-ctl/command/command.go index d83d79b..430c1d7 100644 --- a/cmd/influxd-ctl/command/command.go +++ b/cmd/influxd-ctl/command/command.go @@ -1,122 +1,194 @@ package command import ( + "errors" "fmt" + "github.com/angopher/chronus/cmd/influxd-ctl/action" - "github.com/spf13/cobra" + "github.com/fatih/color" + "github.com/urfave/cli/v2" ) const ( defaultHost = "127.0.0.1:8088" ) -func NewCommand() *cobra.Command { - var copyShardCmd = &cobra.Command{ - Use: "copy-shard ", - Short: "copy shard", - Long: `copies a shard from a source data node to a destination data node.`, - Args: cobra.MinimumNArgs(3), - Run: func(cmd *cobra.Command, args []string) { - action.CopyShard(args[0], args[1], args[2]) - }, - } +var ( + DataNodeAddress string +) - var truncateCmd = &cobra.Command{ - Use: "truncate-shards [ip:port]", - Short: "truncates hot shards", - Long: `Truncates hot shards, that is, shards that cover the time range that includes the current time (now()).The truncate-shards command creates a new shard and the system writes all new points to that shard.`, - Args: cobra.RangeArgs(1, 2), - Run: func(cmd *cobra.Command, args []string) { - host := defaultHost - if len(args) == 2 { - host = args[1] - } - if err := action.TruncateShards(args[0], host); err != nil { - fmt.Println(err) - } - }, - } +func DatabaseCommand() *cli.Command { + return &cli.Command{ + Name: "database", + Usage: "Database info", + Subcommands: []*cli.Command{ + { + Name: "list", + Usage: "list all databases", + Action: func(ctx *cli.Context) error { - var copyShardStatusCmd = &cobra.Command{ - Use: "copy-shard-status [ip:port]", - Short: "Displaying all in-progress copy-shard operations", - Long: `Shows all in-progress copy shard operations, including the shard’s source node, - destination node, database, retention policy, shard ID, total size, - current size, and the operation’s start time.`, - Args: cobra.RangeArgs(0, 1), - Run: func(cmd *cobra.Command, args []string) { - host := defaultHost - if len(args) == 1 { - host = args[0] - } - if err := action.CopyShardStatus(host); err != nil { - fmt.Println(err) - } + return nil + }, + }, }, } +} - var killCopyShardCmd = &cobra.Command{ - Use: "kill-copy-shard ", - Short: "Aborts an in-progress copy-shard command.", - Long: "Aborts an in-progress copy-shard command.", - Args: cobra.MinimumNArgs(3), - Run: func(cmd *cobra.Command, args []string) { - if err := action.KillCopyShard(args[0], args[1], args[2]); err != nil { - fmt.Println(err) - } - }, - } +func NodeCommand() *cli.Command { + return &cli.Command{ + Name: "node", + Usage: "node related operations", + Subcommands: []*cli.Command{ + { + Name: "list", + Usage: "show nodes in cluster", + Action: func(ctx *cli.Context) error { + if err := action.ShowDataNodes(DataNodeAddress); err != nil { + fmt.Println(err) + } + return nil + }, + }, { + Name: "remove", + ArgsUsage: "remove ", + Usage: "remove specified node from cluster", + Action: func(ctx *cli.Context) error { + if ctx.Args().Len() < 1 { + fmt.Println("Please specify node addr to be removed from cluster") + fmt.Println() + return errors.New("Please specify node addr to be removed from cluster") + } + if err := action.RemoveDataNode(DataNodeAddress, ctx.Args().Get(0)); err != nil { + fmt.Println(err) + } - var removeShardCmd = &cobra.Command{ - Use: "remove-shard ", - Short: "Removes a shard from a data node. Removing a shard is an irrecoverable, destructive action; please be cautious with this command.", - Long: "Removes a shard from a data node. Removing a shard is an irrecoverable, destructive action; please be cautious with this command.", - Args: cobra.MinimumNArgs(2), - Run: func(cmd *cobra.Command, args []string) { - if err := action.RemoveShard(args[0], args[1]); err != nil { - fmt.Println(err) - } + return nil + }, + }, }, } +} - var removeDataNodeCmd = &cobra.Command{ - Use: "remove-data-node ", - Short: "Removes a data node from a cluster.", - Long: "Removes a data node from a cluster.", - Args: cobra.MinimumNArgs(1), - Run: func(cmd *cobra.Command, args []string) { - if err := action.RemoveDataNode(args[0]); err != nil { - fmt.Println(err) - } - }, - } +func ShardCommand() *cli.Command { + return &cli.Command{ + Name: "shard", + Usage: "shard related operations", + Subcommands: []*cli.Command{ + { + Name: "list", + ArgsUsage: "copy ", + Usage: "show all shards of specified retention policy", + Description: fmt.Sprint( + "List all shards in specified retention policy.", + ), + Action: func(ctx *cli.Context) error { + if ctx.Args().Len() < 2 { + color.Red("Please specify database and retention policy\n") + fmt.Println() + return errors.New("Please specify database and retention policy") + } + if err := action.ListShard(DataNodeAddress, ctx.Args().Get(0), ctx.Args().Get(1)); err != nil { + fmt.Println(err) + } + return nil - var showDataNodesCmd = &cobra.Command{ - Use: "show-data-nodes [ip:port]", - Short: "Show all data node from a cluster.", - Long: "Show all data node from a cluster.", - Args: cobra.RangeArgs(0, 1), - Run: func(cmd *cobra.Command, args []string) { - host := defaultHost - if len(args) == 1 { - host = args[0] - } - if err := action.ShowDataNodes(host); err != nil { - fmt.Println(err) - } + }, + }, { + Name: "info", + ArgsUsage: "info ", + Usage: "show information of specified shard", + Description: fmt.Sprint( + "Show information of specified shard.", + ), + Action: func(ctx *cli.Context) error { + if ctx.Args().Len() < 1 { + color.Red("Please specify shard id\n") + fmt.Println() + return errors.New("Please specify shard id") + } + if err := action.GetShard(DataNodeAddress, ctx.Args().Get(0)); err != nil { + fmt.Println(err) + } + return nil + }, + }, { + Name: "status", + Usage: "show progress of copy-shard tasks", + Description: fmt.Sprint( + "Shows all in-progress copy shard operations, including the shard’s source node,\n", + "destination node, database, retention policy, shard ID, total size,\n", + "current size, and the operation’s start time.", + ), + Action: func(ctx *cli.Context) error { + if err := action.CopyShardStatus(DataNodeAddress); err != nil { + fmt.Println(err) + } + return nil + }, + }, { + Name: "copy", + Usage: "copy a shard to current node", + ArgsUsage: "copy ", + Description: "Copy a shard from a source data node to a current data node which is specified through -s", + Action: func(ctx *cli.Context) error { + if ctx.Args().Len() < 2 { + return errors.New("Please specify source node and shard") + } + action.CopyShard(ctx.Args().Get(0), DataNodeAddress, ctx.Args().Get(1)) + return nil + }, + }, { + Name: "remove", + Usage: "remove a shard", + ArgsUsage: "remove ", + Description: fmt.Sprint( + "Removes a shard from current data node.\n", + "Removing a shard is an irrecoverable, destructive action;\n", + "Please be cautious with this command.", + ), + Action: func(ctx *cli.Context) error { + if ctx.Args().Len() < 1 { + return errors.New("Please specify shard") + } + if err := action.RemoveShard(DataNodeAddress, ctx.Args().First()); err != nil { + fmt.Println(err) + } + return nil + }, + }, { + Name: "stop", + Usage: "stop a task of copy shard", + ArgsUsage: "stop ", + Description: "Stop a task of copy shard to current node which specified through -s.", + Action: func(ctx *cli.Context) error { + if ctx.Args().Len() < 2 { + return errors.New("Please specify source node and shard") + } + if err := action.KillCopyShard(ctx.Args().Get(0), DataNodeAddress, ctx.Args().Get(1)); err != nil { + fmt.Println(err) + } + return nil + }, + }, { + Name: "truncate", + Usage: "truncates hot shards", + ArgsUsage: "truncate ", + Description: fmt.Sprint( + "Truncates hot shards, that is, shards that cover the time range\n", + "that includes the current time (now()).\n", + "The truncate-shards command creates a new shard and \n", + "the system writes all new points to that shard.", + ), + Action: func(ctx *cli.Context) error { + if ctx.Args().Len() < 1 { + return errors.New("delay seconds should be specified") + } + if err := action.TruncateShards(ctx.Args().Get(0), DataNodeAddress); err != nil { + fmt.Println(err) + } + return nil + }, + }, }, } - - var rootCmd = &cobra.Command{Use: "influxd-ctl"} - rootCmd.AddCommand( - copyShardCmd, - truncateCmd, - copyShardStatusCmd, - killCopyShardCmd, - removeShardCmd, - removeDataNodeCmd, - showDataNodesCmd, - ) - - return rootCmd } diff --git a/cmd/influxd-ctl/main.go b/cmd/influxd-ctl/main.go index 1482dfd..9628354 100644 --- a/cmd/influxd-ctl/main.go +++ b/cmd/influxd-ctl/main.go @@ -1,10 +1,30 @@ package main import ( + "os" + "github.com/angopher/chronus/cmd/influxd-ctl/command" + "github.com/urfave/cli/v2" ) func main() { - command := command.NewCommand() - command.Execute() + app := &cli.App{} + app.Name = "influxd-ctl" + app.Usage = "Maintain the data nodes in cluster" + + app.Commands = []*cli.Command{ + command.DatabaseCommand(), + command.NodeCommand(), + command.ShardCommand(), + } + app.Flags = []cli.Flag{ + &cli.StringFlag{ + Name: "node", + Aliases: []string{"s"}, + Required: true, + Usage: "DataNode address in cluster, ip:port", + Destination: &command.DataNodeAddress, + }, + } + app.Run(os.Args) } diff --git a/cmd/influxd/run/server.go b/cmd/influxd/run/server.go index eef9142..bdfa535 100644 --- a/cmd/influxd/run/server.go +++ b/cmd/influxd/run/server.go @@ -45,6 +45,7 @@ import ( "github.com/angopher/chronus/services/controller" "github.com/angopher/chronus/services/hh" imeta "github.com/angopher/chronus/services/meta" + "github.com/angopher/chronus/x" ) var startTime time.Time @@ -205,23 +206,15 @@ func NewServer(c *Config, buildInfo *BuildInfo, logger *zap.Logger) (*Server, er if err := s.ClusterMetaClient.Open(); err != nil { return nil, err } - - wait := s.ClusterMetaClient.WaitForDataChanged() - go s.ClusterMetaClient.RunSyncLoop() - //wait sync meta data from meta server - select { - case <-time.After(5 * time.Second): - //TODO: - panic("sync meta data failed") - case <-wait: - } + s.ClusterMetaClient.Start() // If we've already created a data node for our id, we're done n, err := s.ClusterMetaClient.DataNode(nodeID) if err != nil { + s.Logger.Warn("Node id from store can't be used, try to create new", zap.Error(err)) n, err = s.ClusterMetaClient.CreateDataNode(s.httpAPIAddr, s.tcpAddr) if err != nil { - log.Printf("Unable to create data node. err: %s", err.Error()) + s.Logger.Warn(fmt.Sprint("Unable to create data node. err: ", err.Error())) return nil, err } } @@ -241,12 +234,31 @@ func NewServer(c *Config, buildInfo *BuildInfo, logger *zap.Logger) (*Server, er // Create the Subscriber service s.Subscriber = subscriber.NewService(c.Subscriber) + clientPool := coordinator.NewClientPool(func(nodeId uint64) (x.ConnPool, error) { + return x.NewBoundedPool( + x.Max(1, x.Min(10, s.config.Coordinator.PoolMaxStreamsPerNode/20)), + s.config.Coordinator.PoolMaxStreamsPerNode, + time.Duration(s.config.Coordinator.PoolMaxIdleTimeout), + time.Duration(s.config.Coordinator.DailTimeout), + coordinator.NewClientConnFactory( + nodeId, + time.Duration(s.config.Coordinator.DailTimeout), + s.ClusterMetaClient, + ).Dial, + ) + }) + // Initialize shard writer - s.ShardWriter = coordinator.NewShardWriter(time.Duration(s.config.Coordinator.WriteTimeout), s.config.Coordinator.PoolMaxConnections) + s.ShardWriter = coordinator.NewShardWriter( + time.Duration(s.config.Coordinator.WriteTimeout), + clientPool, + ) + s.ShardWriter.WithLogger(s.Logger) // Create the hinted handoff service s.HintedHandoff = hh.NewService(c.HintedHandoff, s.ShardWriter, s.ClusterMetaClient) s.HintedHandoff.Monitor = s.Monitor + s.HintedHandoff.WithLogger(s.Logger) // Initialize points writer. s.PointsWriter = coordinator.NewPointsWriter() @@ -257,7 +269,11 @@ func NewServer(c *Config, buildInfo *BuildInfo, logger *zap.Logger) (*Server, er s.PointsWriter.HintedHandoff = s.HintedHandoff // Initialize cluster extecutor - clusterExecutor := coordinator.NewClusterExecutor(s.Node, s.TSDBStore, s.ClusterMetaClient, s.config.Coordinator) + clusterExecutor := coordinator.NewClusterExecutor( + s.Node, s.TSDBStore, + s.ClusterMetaClient, clientPool, + s.config.Coordinator, + ) clusterExecutor.WithLogger(s.Logger) // Initialize query executor. diff --git a/coordinator/client_pool.go b/coordinator/client_pool.go index 3bb5041..7ca77a6 100644 --- a/coordinator/client_pool.go +++ b/coordinator/client_pool.go @@ -1,57 +1,169 @@ package coordinator import ( + "fmt" "net" "sync" + "time" - "gopkg.in/fatih/pool.v2" + "github.com/angopher/chronus/x" + "github.com/influxdata/influxdb/services/meta" ) -type clientPool struct { - mu sync.RWMutex - pool map[uint64]pool.Pool +type PoolFactory func(nodeId uint64) (x.ConnPool, error) + +type ClientPool struct { + closer chan int + wg sync.WaitGroup + + mu sync.RWMutex + pools map[uint64]*poolEntry + factory PoolFactory } -func newClientPool() *clientPool { - return &clientPool{ - pool: make(map[uint64]pool.Pool), - } +type poolEntry struct { + pool x.ConnPool + lastUse time.Time } -func (c *clientPool) setPool(nodeID uint64, p pool.Pool) { - c.mu.Lock() - c.pool[nodeID] = p - c.mu.Unlock() +func NewClientPool(factory PoolFactory) *ClientPool { + pool := &ClientPool{ + pools: make(map[uint64]*poolEntry), + closer: make(chan int), + factory: factory, + } + + go pool.idleCheckLoop() + return pool } -func (c *clientPool) getPool(nodeID uint64) (pool.Pool, bool) { +func (c *ClientPool) Len() int { c.mu.RLock() - p, ok := c.pool[nodeID] + var size int + for _, p := range c.pools { + size += p.pool.Len() + } c.mu.RUnlock() - return p, ok + return size } -func (c *clientPool) size() int { +func (c *ClientPool) Total() int { c.mu.RLock() var size int - for _, p := range c.pool { - size += p.Len() + for _, p := range c.pools { + size += p.pool.Total() } c.mu.RUnlock() return size } -func (c *clientPool) conn(nodeID uint64) (net.Conn, error) { +func (c *ClientPool) idleCheckOnce() { + c.mu.Lock() + defer c.mu.Unlock() + + now := time.Now() + var removed []uint64 + for nodeId, entry := range c.pools { + if now.Sub(entry.lastUse) <= 600*time.Second { + continue + } + // close it + removed = append(removed, nodeId) + } + for _, id := range removed { + if entry, ok := c.pools[id]; ok { + entry.pool.Close() + delete(c.pools, id) + } + } +} + +func (c *ClientPool) idleCheckLoop() { + c.wg.Add(1) + defer c.wg.Done() + + ticker := time.NewTicker(60 * time.Second) + defer ticker.Stop() + for { + select { + case <-ticker.C: + c.idleCheckOnce() + case <-c.closer: + return + } + } +} + +func (c *ClientPool) GetConn(nodeID uint64) (x.PooledConn, error) { c.mu.RLock() - conn, err := c.pool[nodeID].Get() + if entry, ok := c.pools[nodeID]; ok { + entry.lastUse = time.Now() + c.mu.RUnlock() + return entry.pool.Get() + } c.mu.RUnlock() - return conn, err + // switch to write lock + c.mu.Lock() + defer c.mu.Unlock() + // create new pool + pool, err := c.factory(nodeID) + if err != nil { + return nil, err + } + c.pools[nodeID] = &poolEntry{ + lastUse: time.Now(), + pool: pool, + } + return pool.Get() } -func (c *clientPool) close() { +func (c *ClientPool) close() { c.mu.Lock() - for _, p := range c.pool { - p.Close() + defer c.mu.Unlock() + for _, entry := range c.pools { + entry.pool.Close() + } + c.pools = nil +} + +type ClientConnFactory struct { + nodeId uint64 + timeout time.Duration + + metaClient interface { + DataNode(id uint64) (ni *meta.NodeInfo, err error) + } +} + +func NewClientConnFactory(nodeId uint64, timeout time.Duration, metaClient MetaClient) *ClientConnFactory { + return &ClientConnFactory{ + nodeId: nodeId, + timeout: timeout, + metaClient: metaClient, } - c.mu.Unlock() +} + +func (c *ClientConnFactory) Dial() (net.Conn, error) { + ni, err := c.metaClient.DataNode(c.nodeId) + if err != nil { + return nil, err + } + + if ni == nil { + return nil, fmt.Errorf("node %d does not exist", c.nodeId) + } + + conn, err := net.DialTimeout("tcp", ni.TCPHost, c.timeout) + if err != nil { + return nil, err + } + + // Write a marker byte for cluster messages. + _, err = conn.Write([]byte{MuxHeader}) + if err != nil { + conn.Close() + return nil, err + } + + return conn, nil } diff --git a/coordinator/cluster_executor.go b/coordinator/cluster_executor.go index 88963e8..f6e8542 100644 --- a/coordinator/cluster_executor.go +++ b/coordinator/cluster_executor.go @@ -3,7 +3,6 @@ package coordinator import ( "context" "fmt" - "math/rand" "sort" "sync" "time" @@ -35,23 +34,26 @@ type ClusterExecutor struct { Logger *zap.Logger } -func NewClusterExecutor(n *influxdb.Node, s TSDBStore, m MetaClient, Config Config) *ClusterExecutor { - return &ClusterExecutor{ +func NewClusterExecutor(n *influxdb.Node, s TSDBStore, m MetaClient, pool *ClientPool, Config Config) *ClusterExecutor { + executor := &ClusterExecutor{ Node: n, TSDBStore: s, MetaClient: m, RemoteNodeExecutor: &remoteNodeExecutor{ - MetaClient: m, + ClientPool: pool, DailTimeout: time.Duration(Config.DailTimeout), ShardReaderTimeout: time.Duration(Config.ShardReaderTimeout), ClusterTracing: Config.ClusterTracing, }, Logger: zap.NewNop(), } + executor.RemoteNodeExecutor.WithLogger(executor.Logger) + return executor } func (me *ClusterExecutor) WithLogger(log *zap.Logger) { me.Logger = log.With(zap.String("service", "ClusterExecutor")) + me.RemoteNodeExecutor.WithLogger(log) } func (me *ClusterExecutor) ExecuteStatement(stmt influxql.Statement, ctx *query.ExecutionContext) error { @@ -65,12 +67,13 @@ func (me *ClusterExecutor) ExecuteStatement(stmt influxql.Statement, ctx *query. return err } - nodes := NewNodeIdsByNodes(nodeInfos) + nodes := toNodeIds(nodeInfos) results := make(map[uint64]*Result) var mutex sync.Mutex var wg sync.WaitGroup - fn := func(nodeId uint64) { +LOOP: + for _, nodeId := range nodes { host := "unkown" node, _ := me.MetaClient.DataNode(nodeId) if node != nil { @@ -80,17 +83,18 @@ func (me *ClusterExecutor) ExecuteStatement(stmt influxql.Statement, ctx *query. switch t := stmt.(type) { case *influxql.KillQueryStatement: if t.Host != "" && t.Host != host { - return + // not this node + continue LOOP } } wg.Add(1) - go func() { - defer wg.Add(-1) + go func(curNodeId uint64) { + defer wg.Done() var err error var qr *query.Result - if nodeId == me.Node.ID { + if curNodeId == me.Node.ID { recvCtx := &query.ExecutionContext{ Context: context.Background(), Results: make(chan *query.Result, 1), @@ -100,7 +104,7 @@ func (me *ClusterExecutor) ExecuteStatement(stmt influxql.Statement, ctx *query. qr = <-recvCtx.Results } } else { - qr, err = me.RemoteNodeExecutor.TaskManagerStatement(nodeId, stmt) + qr, err = me.RemoteNodeExecutor.TaskManagerStatement(curNodeId, stmt) } if qr != nil && len(qr.Series) > 0 { @@ -111,12 +115,10 @@ func (me *ClusterExecutor) ExecuteStatement(stmt influxql.Statement, ctx *query. } mutex.Lock() - results[nodeId] = &Result{qr: qr, err: err} - mutex.Unlock() - }() + defer mutex.Unlock() + results[curNodeId] = &Result{qr: qr, err: err} + }(nodeId) } - - nodes.Apply(fn) wg.Wait() //merge result @@ -168,32 +170,29 @@ func (me *ClusterExecutor) SeriesCardinality(database string) (int64, error) { if err != nil { return -1, err } - nodes := NewNodeIdsByShards(shards) + nodes := getAllRelatedNodes(shards) results := make(map[uint64]*Result) var mutex sync.Mutex var wg sync.WaitGroup - fn := func(nodeId uint64) { + for _, nodeId := range nodes { wg.Add(1) - go func() { - defer wg.Add(-1) + go func(curNodeId uint64) { + defer wg.Done() var n int64 var err error - if nodeId == me.Node.ID { + if curNodeId == me.Node.ID { n, err = me.TSDBStore.SeriesCardinality(database) } else { - n, err = me.RemoteNodeExecutor.SeriesCardinality(nodeId, database) + n, err = me.RemoteNodeExecutor.SeriesCardinality(curNodeId, database) } mutex.Lock() - results[nodeId] = &Result{n: n, err: err} - mutex.Unlock() - return - }() + defer mutex.Unlock() + results[curNodeId] = &Result{n: n, err: err} + }(nodeId) } - - nodes.Apply(fn) wg.Wait() var sum int64 @@ -233,32 +232,29 @@ func (me *ClusterExecutor) MeasurementNames(auth query.Authorizer, database stri if err != nil { return nil, err } - nodes := NewNodeIdsByShards(shards) + nodes := getAllRelatedNodes(shards) results := make(map[uint64]*Result) var mutex sync.Mutex var wg sync.WaitGroup - fn := func(nodeId uint64) { + for _, nodeId := range nodes { wg.Add(1) - go func() { - defer wg.Add(-1) + go func(curNodeId uint64) { + defer wg.Done() var names [][]byte var err error - if nodeId == me.Node.ID { + if curNodeId == me.Node.ID { names, err = me.TSDBStore.MeasurementNames(auth, database, cond) } else { - names, err = me.RemoteNodeExecutor.MeasurementNames(nodeId, database, cond) + names, err = me.RemoteNodeExecutor.MeasurementNames(curNodeId, database, cond) } mutex.Lock() - results[nodeId] = &Result{names: names, err: err} - mutex.Unlock() - return - }() + defer mutex.Unlock() + results[curNodeId] = &Result{names: names, err: err} + }(nodeId) } - - nodes.Apply(fn) wg.Wait() uniq := make(map[string]struct{}) @@ -272,7 +268,7 @@ func (me *ClusterExecutor) MeasurementNames(auth query.Authorizer, database stri } strNames := make([]string, 0, len(uniq)) - for name, _ := range uniq { + for name := range uniq { strNames = append(strNames, name) } sort.Sort(StringSlice(strNames)) @@ -296,33 +292,20 @@ func (me *ClusterExecutor) TagValues(auth query.Authorizer, ids []uint64, cond i return nil, err } - n2s := NewNode2ShardIDs(me.MetaClient, me.Node, shards) - result := make(map[uint64]*TagValuesResult) - - var mutex sync.Mutex - var wg sync.WaitGroup - fn := func(nodeId uint64, shardIDs []uint64) { - wg.Add(1) - go func() { - defer wg.Add(-1) + n2s := PlanNodes(me.Node.ID, shards, nil) - var tagValues []tsdb.TagValues - var err error - if nodeId == me.Node.ID { - tagValues, err = me.TSDBStore.TagValues(auth, shardIDs, cond) - } else { - tagValues, err = me.RemoteNodeExecutor.TagValues(nodeId, shardIDs, cond) - } - - mutex.Lock() - result[nodeId] = &TagValuesResult{values: tagValues, err: err} - mutex.Unlock() - return - }() + fn := func(nodeId uint64, shards []meta.ShardInfo) (result interface{}, err error) { + var tagValues []tsdb.TagValues + shardIDs := toShardIDs(shards) + if nodeId == me.Node.ID { + tagValues, err = me.TSDBStore.TagValues(auth, shardIDs, cond) + } else { + tagValues, err = me.RemoteNodeExecutor.TagValues(nodeId, shardIDs, cond) + } + result = &TagValuesResult{values: tagValues, err: err} + return } - - n2s.Apply(fn) - wg.Wait() + result, err := n2s.ExecuteWithRetry(fn) extractKeyFn := func(kv tsdb.KeyValue) string { //TODO: @@ -330,7 +313,11 @@ func (me *ClusterExecutor) TagValues(auth query.Authorizer, ids []uint64, cond i } uniq := make(map[string]map[string]tsdb.KeyValue) - for nodeId, r := range result { + for nodeId, t := range result { + r, ok := t.(*TagValuesResult) + if !ok { + continue + } if r.err != nil { return nil, fmt.Errorf("TagValues fail, nodeId %d, err:%s", nodeId, r.err) } @@ -369,70 +356,62 @@ func (me *ClusterExecutor) CreateIterator(ctx context.Context, m *influxql.Measu err error } - n2s := NewNode2ShardIDs(me.MetaClient, me.Node, shards) - results := make(map[uint64]*Result) - - var mutex sync.Mutex - var wg sync.WaitGroup - fn := func(nodeId uint64, shardIDs []uint64) { - wg.Add(1) - go func() { - defer wg.Add(-1) + n2s := PlanNodes(me.Node.ID, shards, nil) - var iter query.Iterator - var err error - if nodeId == me.Node.ID { - //localCtx only use for local node - localCtx := ctx - iter, err = func() (query.Iterator, error) { - span := tracing.SpanFromContext(localCtx) - if span != nil { - span = span.StartSpan(fmt.Sprintf("local_node_id: %d", me.Node.ID)) - defer span.Finish() - - localCtx = tracing.NewContextWithSpan(localCtx, span) - } - sg := me.TSDBStore.ShardGroup(shardIDs) - if m.Regex != nil { - measurements := sg.MeasurementsByRegex(m.Regex.Val) - inputs := make([]query.Iterator, 0, len(measurements)) - if err := func() error { - for _, measurement := range measurements { - mm := m.Clone() - mm.Name = measurement - input, err := sg.CreateIterator(localCtx, mm, opt) - if err != nil { - return err - } - inputs = append(inputs, input) + fn := func(nodeId uint64, shards []meta.ShardInfo) (result interface{}, err error) { + var iter query.Iterator + shardIDs := toShardIDs(shards) + if nodeId == me.Node.ID { + //localCtx only use for local node + localCtx := ctx + iter, err = func() (query.Iterator, error) { + span := tracing.SpanFromContext(localCtx) + if span != nil { + span = span.StartSpan(fmt.Sprintf("local_node_id: %d", me.Node.ID)) + defer span.Finish() + + localCtx = tracing.NewContextWithSpan(localCtx, span) + } + sg := me.TSDBStore.ShardGroup(shardIDs) + if m.Regex != nil { + measurements := sg.MeasurementsByRegex(m.Regex.Val) + inputs := make([]query.Iterator, 0, len(measurements)) + if err := func() error { + for _, measurement := range measurements { + mm := m.Clone() + mm.Name = measurement + input, err := sg.CreateIterator(localCtx, mm, opt) + if err != nil { + return err } - return nil - }(); err != nil { - query.Iterators(inputs).Close() - return nil, err + inputs = append(inputs, input) } - - return query.Iterators(inputs).Merge(opt) + return nil + }(); err != nil { + query.Iterators(inputs).Close() + return nil, err } - return sg.CreateIterator(localCtx, m, opt) - }() - } else { - iter, err = me.RemoteNodeExecutor.CreateIterator(nodeId, ctx, m, opt, shardIDs) - } - mutex.Lock() - results[nodeId] = &Result{iter: iter, err: err} - mutex.Unlock() - return - }() - } + return query.Iterators(inputs).Merge(opt) + } + return sg.CreateIterator(localCtx, m, opt) + }() + } else { + iter, err = me.RemoteNodeExecutor.CreateIterator(nodeId, ctx, m, opt, shardIDs) + } - n2s.Apply(fn) - wg.Wait() + result = &Result{iter: iter, err: err} + return + } + result, _ := n2s.ExecuteWithRetry(fn) seriesN := 0 - inputs := make([]query.Iterator, 0, len(results)) - for _, r := range results { + inputs := make([]query.Iterator, 0, len(result)) + for _, t := range result { + r, ok := t.(*Result) + if !ok { + continue + } if r.err != nil { return nil, r.err } @@ -452,13 +431,14 @@ func (me *ClusterExecutor) CreateIterator(ctx context.Context, m *influxql.Measu func (me *ClusterExecutor) MapType(m *influxql.Measurement, field string, shards []meta.ShardInfo) influxql.DataType { type Result struct { + nodeId uint64 dataType influxql.DataType err error } - n2s := NewNode2ShardIDs(me.MetaClient, me.Node, shards) - var result Result - fn := func(nodeId uint64, shardIDs []uint64) { + n2s := PlanNodes(me.Node.ID, shards, nil) + fn := func(nodeId uint64, shards []meta.ShardInfo) (result interface{}, err error) { + shardIDs := toShardIDs(shards) if nodeId == me.Node.ID { sg := me.TSDBStore.ShardGroup(shardIDs) var names []string @@ -478,42 +458,43 @@ func (me *ClusterExecutor) MapType(m *influxql.Measurement, field string, shards typ = t } } - result.dataType = typ + result = &Result{dataType: typ, err: nil, nodeId: nodeId} } return } - - n2s.Apply(fn) - if result.dataType != influxql.Unknown { - return result.dataType + result, _ := n2s.ExecuteWithRetry(fn) + for _, t := range result { + if t == nil { + continue + } + r, ok := t.(*Result) + if !ok { + continue + } + if r.dataType != influxql.Unknown { + return r.dataType + } } - //本地失败, 尝试从remote node获取 - results := make(map[uint64]*Result) - var mutex sync.Mutex - var wg sync.WaitGroup - fn = func(nodeId uint64, shardIDs []uint64) { - wg.Add(1) - go func() { - defer wg.Add(-1) - - if nodeId != me.Node.ID { - mutex.Lock() - typ, err := me.RemoteNodeExecutor.MapType(nodeId, m, field, shardIDs) - results[nodeId] = &Result{dataType: typ, err: err} - mutex.Unlock() - } - return - }() + //本地失败, 尝试仅从remote node获取 + fn = func(nodeId uint64, shards []meta.ShardInfo) (result interface{}, err error) { + shardIDs := toShardIDs(shards) + if nodeId != me.Node.ID { + typ, err := me.RemoteNodeExecutor.MapType(nodeId, m, field, shardIDs) + result = &Result{dataType: typ, err: err} + } + return } - - n2s.Apply(fn) - wg.Wait() + result, _ = n2s.ExecuteWithRetry(fn) typ := influxql.Unknown - for nodeId, r := range results { + for _, t := range result { + r, ok := t.(*Result) + if !ok { + continue + } if r.err != nil { - me.Logger.Warn("results have error", zap.Error(r.err), zap.Uint64("node", nodeId)) + me.Logger.Warn("results have error", zap.Error(r.err), zap.Uint64("node", r.nodeId)) continue } if typ.LessThan(r.dataType) { @@ -530,52 +511,44 @@ func (me *ClusterExecutor) IteratorCost(m *influxql.Measurement, opt query.Itera err error } - n2s := NewNode2ShardIDs(me.MetaClient, me.Node, shards) - results := make(map[uint64]*Result) + n2s := PlanNodes(me.Node.ID, shards, nil) - var mutex sync.Mutex - var wg sync.WaitGroup - fn := func(nodeId uint64, shardIDs []uint64) { - wg.Add(1) - go func() { - defer wg.Add(-1) - - var cost query.IteratorCost - var err error - if nodeId == me.Node.ID { - sg := me.TSDBStore.ShardGroup(shardIDs) - if m.Regex != nil { - cost, err = func() (query.IteratorCost, error) { - var costs query.IteratorCost - measurements := sg.MeasurementsByRegex(m.Regex.Val) - for _, measurement := range measurements { - c, err := sg.IteratorCost(measurement, opt) - if err != nil { - return c, err - } - costs = costs.Combine(c) + fn := func(nodeId uint64, shards []meta.ShardInfo) (result interface{}, err error) { + shardIDs := toShardIDs(shards) + var cost query.IteratorCost + if nodeId == me.Node.ID { + sg := me.TSDBStore.ShardGroup(shardIDs) + if m.Regex != nil { + cost, err = func() (query.IteratorCost, error) { + var costs query.IteratorCost + measurements := sg.MeasurementsByRegex(m.Regex.Val) + for _, measurement := range measurements { + c, err := sg.IteratorCost(measurement, opt) + if err != nil { + return c, err } - return costs, nil - }() - } else { - cost, err = sg.IteratorCost(m.Name, opt) - } + costs = costs.Combine(c) + } + return costs, nil + }() } else { - cost, err = me.RemoteNodeExecutor.IteratorCost(nodeId, m, opt, shardIDs) + cost, err = sg.IteratorCost(m.Name, opt) } + } else { + cost, err = me.RemoteNodeExecutor.IteratorCost(nodeId, m, opt, shardIDs) + } - mutex.Lock() - results[nodeId] = &Result{cost: cost, err: err} - mutex.Unlock() - return - }() + result = &Result{cost: cost, err: err} + return } - - n2s.Apply(fn) - wg.Wait() + result, _ := n2s.ExecuteWithRetry(fn) var costs query.IteratorCost - for _, r := range results { + for _, t := range result { + r, ok := t.(*Result) + if !ok { + continue + } if r.err != nil { return costs, r.err } @@ -591,45 +564,39 @@ func (me *ClusterExecutor) FieldDimensions(m *influxql.Measurement, shards []met err error } - n2s := NewNode2ShardIDs(me.MetaClient, me.Node, shards) - results := make(map[uint64]*Result) - - var mutex sync.Mutex - var wg sync.WaitGroup - fn := func(nodeId uint64, shardIDs []uint64) { - wg.Add(1) - go func() { - defer wg.Add(-1) + n2s := PlanNodes(me.Node.ID, shards, nil) - var fields map[string]influxql.DataType - var dimensions map[string]struct{} - var err error - if nodeId == me.Node.ID { - sg := me.TSDBStore.ShardGroup(shardIDs) - var measurements []string - if m.Regex != nil { - measurements = sg.MeasurementsByRegex(m.Regex.Val) - } else { - measurements = []string{m.Name} - } - fields, dimensions, err = sg.FieldDimensions(measurements) + fn := func(nodeId uint64, shards []meta.ShardInfo) (result interface{}, err error) { + var fields map[string]influxql.DataType + var dimensions map[string]struct{} + shardIDs := toShardIDs(shards) + if nodeId == me.Node.ID { + sg := me.TSDBStore.ShardGroup(shardIDs) + var measurements []string + if m.Regex != nil { + measurements = sg.MeasurementsByRegex(m.Regex.Val) } else { - fields, dimensions, err = me.RemoteNodeExecutor.FieldDimensions(nodeId, m, shardIDs) + measurements = []string{m.Name} } - - mutex.Lock() - results[nodeId] = &Result{fields: fields, dimensions: dimensions, err: err} - mutex.Unlock() - return - }() + fields, dimensions, err = sg.FieldDimensions(measurements) + } else { + fields, dimensions, err = me.RemoteNodeExecutor.FieldDimensions(nodeId, m, shardIDs) + } + if fields != nil { + result = &Result{fields: fields, dimensions: dimensions, err: err} + } + return } - n2s.Apply(fn) - wg.Wait() + result, err := n2s.ExecuteWithRetry(fn) fields = make(map[string]influxql.DataType) dimensions = make(map[string]struct{}) - for _, r := range results { + for _, t := range result { + r, ok := t.(*Result) + if !ok { + continue + } //TODO: merge err if r.err != nil { return nil, nil, r.err @@ -638,7 +605,7 @@ func (me *ClusterExecutor) FieldDimensions(m *influxql.Measurement, shards []met for f, t := range r.fields { fields[f] = t } - for d, _ := range r.dimensions { + for d := range r.dimensions { dimensions[d] = struct{}{} } } @@ -676,39 +643,34 @@ func (me *ClusterExecutor) TagKeys(auth query.Authorizer, ids []uint64, cond inf return nil, err } - n2s := NewNode2ShardIDs(me.MetaClient, me.Node, shards) - result := make(map[uint64]*TagKeysResult) - - var mutex sync.Mutex - var wg sync.WaitGroup - fn := func(nodeId uint64, shardIDs []uint64) { - wg.Add(1) - go func() { - defer wg.Add(-1) + n2s := PlanNodes(me.Node.ID, shards, nil) - var tagKeys []tsdb.TagKeys - var err error - if nodeId == me.Node.ID { - tagKeys, err = me.TSDBStore.TagKeys(auth, shardIDs, cond) - } else { - tagKeys, err = me.RemoteNodeExecutor.TagKeys(nodeId, shardIDs, cond) - } + fn := func(nodeId uint64, shards []meta.ShardInfo) (result interface{}, err error) { + var tagKeys []tsdb.TagKeys + shardIDs := toShardIDs(shards) + if nodeId == me.Node.ID { + tagKeys, err = me.TSDBStore.TagKeys(auth, shardIDs, cond) + } else { + tagKeys, err = me.RemoteNodeExecutor.TagKeys(nodeId, shardIDs, cond) + } - if err != nil { - me.Logger.Error("TagKeys fail", zap.Error(err), zap.Uint64("node", nodeId)) - } - mutex.Lock() - result[nodeId] = &TagKeysResult{keys: tagKeys, err: err} - mutex.Unlock() - return - }() + if err != nil { + me.Logger.Error("TagKeys fail", zap.Error(err), zap.Uint64("node", nodeId)) + } + if len(tagKeys) > 0 { + result = &TagKeysResult{keys: tagKeys, err: err} + } + return } - n2s.Apply(fn) - wg.Wait() + result, err := n2s.ExecuteWithRetry(fn) uniqKeys := make(map[string]map[string]struct{}) - for _, r := range result { + for _, t := range result { + r, ok := t.(*TagKeysResult) + if !ok { + continue + } if r.err != nil { return nil, r.err } @@ -730,7 +692,7 @@ func (me *ClusterExecutor) TagKeys(auth query.Authorizer, ids []uint64, cond inf for m, keys := range uniqKeys { tagKey := &tagKeys[idx] tagKey.Measurement = m - for k, _ := range keys { + for k := range keys { tagKey.Keys = append(tagKey.Keys, k) } sort.Sort(StringSlice(tagKey.Keys)) @@ -750,31 +712,28 @@ func (me *ClusterExecutor) DeleteMeasurement(database, name string) error { if err != nil { return err } - nodes := NewNodeIdsByShards(shards) + nodes := getAllRelatedNodes(shards) results := make(map[uint64]*Result) var mutex sync.Mutex var wg sync.WaitGroup - fn := func(nodeId uint64) { + for _, nodeId := range nodes { wg.Add(1) - go func() { - defer wg.Add(-1) + go func(curNodeId uint64) { + defer wg.Done() var err error - if nodeId == me.Node.ID { + if curNodeId == me.Node.ID { err = me.TSDBStore.DeleteMeasurement(database, name) } else { - err = me.RemoteNodeExecutor.DeleteMeasurement(nodeId, database, name) + err = me.RemoteNodeExecutor.DeleteMeasurement(curNodeId, database, name) } mutex.Lock() - results[nodeId] = &Result{err: err} - mutex.Unlock() - return - }() + defer mutex.Unlock() + results[curNodeId] = &Result{err: err} + }(nodeId) } - - nodes.Apply(fn) wg.Wait() for _, r := range results { @@ -794,31 +753,29 @@ func (me *ClusterExecutor) DeleteDatabase(database string) error { if err != nil { return err } - nodes := NewNodeIdsByShards(shards) - results := make(map[uint64]*Result) + nodes := getAllRelatedNodes(shards) + results := make(map[uint64]*Result, len(nodes)) var mutex sync.Mutex var wg sync.WaitGroup - fn := func(nodeId uint64) { + for _, nodeId := range nodes { wg.Add(1) - go func() { - defer wg.Add(-1) + go func(curNodeId uint64) { + defer wg.Done() var err error - if nodeId == me.Node.ID { + if curNodeId == me.Node.ID { err = me.TSDBStore.DeleteDatabase(database) } else { - err = me.RemoteNodeExecutor.DeleteDatabase(nodeId, database) + err = me.RemoteNodeExecutor.DeleteDatabase(curNodeId, database) } mutex.Lock() - results[nodeId] = &Result{err: err} - mutex.Unlock() + defer mutex.Unlock() + results[curNodeId] = &Result{err: err} return - }() + }(nodeId) } - - nodes.Apply(fn) wg.Wait() for _, r := range results { @@ -838,33 +795,30 @@ func (me *ClusterExecutor) DeleteSeries(database string, sources []influxql.Sour if err != nil { return err } - nodes := NewNodeIdsByShards(shards) + nodes := getAllRelatedNodes(shards) results := make(map[uint64]*Result) var mutex sync.Mutex var wg sync.WaitGroup - fn := func(nodeId uint64) { + for _, nodeId := range nodes { wg.Add(1) - go func() { - defer wg.Add(-1) + go func(curNodeId uint64) { + defer wg.Done() var err error - if nodeId == me.Node.ID { + if curNodeId == me.Node.ID { // Convert "now()" to current time. cond = influxql.Reduce(cond, &influxql.NowValuer{Now: time.Now().UTC()}) err = me.TSDBStore.DeleteSeries(database, sources, cond) } else { - err = me.RemoteNodeExecutor.DeleteSeries(nodeId, database, sources, cond) + err = me.RemoteNodeExecutor.DeleteSeries(curNodeId, database, sources, cond) } mutex.Lock() - results[nodeId] = &Result{err: err} - mutex.Unlock() - return - }() + defer mutex.Unlock() + results[curNodeId] = &Result{err: err} + }(nodeId) } - - nodes.Apply(fn) wg.Wait() for _, r := range results { @@ -881,121 +835,34 @@ func (a StringSlice) Len() int { return len(a) } func (a StringSlice) Swap(i, j int) { a[i], a[j] = a[j], a[i] } func (a StringSlice) Less(i, j int) bool { return a[i] < a[j] } -type NodeIds []uint64 - -func NewNodeIdsByNodes(nodeInfos []meta.NodeInfo) NodeIds { - var ids []uint64 - for _, ni := range nodeInfos { - ids = append(ids, ni.ID) +func toNodeIds(nodeInfos []meta.NodeInfo) []uint64 { + ids := make([]uint64, len(nodeInfos)) + for i, ni := range nodeInfos { + ids[i] = ni.ID } - return NodeIds(ids) + return ids } -//TODO:取个达意的名字 -func NewNodeIdsByShards(Shards []meta.ShardInfo) NodeIds { - m := make(map[uint64]struct{}) - for _, si := range Shards { - for _, owner := range si.Owners { - m[owner.NodeID] = struct{}{} +// getAllRelatedNodes returns all related nodes owning shards +func getAllRelatedNodes(shards []meta.ShardInfo) []uint64 { + nodeMap := make(map[uint64]bool) + for _, shard := range shards { + for _, n := range shard.Owners { + nodeMap[n.NodeID] = true } } - nodes := make([]uint64, 0, len(m)) - for n, _ := range m { - nodes = append(nodes, n) + ids := make([]uint64, 0, len(nodeMap)) + for id := range nodeMap { + ids = append(ids, id) } - return nodes + return ids } -func (me NodeIds) Apply(fn func(nodeId uint64)) { - for _, nodeID := range me { - fn(nodeID) - } -} - -type Node2ShardIDs map[uint64][]uint64 - -func NewNode2ShardIDs(mc interface { - DataNode(nodeId uint64) (*meta.NodeInfo, error) -}, - localNode *influxdb.Node, - shards []meta.ShardInfo) Node2ShardIDs { - allNodes := make([]uint64, 0) - for _, si := range shards { - if si.OwnedBy(localNode.ID) { - continue - } - - for _, owner := range si.Owners { - allNodes = append(allNodes, owner.NodeID) - } - } - - //选出 active node - activeNodes := make(map[uint64]struct{}) - var wg sync.WaitGroup - var mutex sync.Mutex - for _, id := range allNodes { - wg.Add(1) - go func(id uint64) { - defer wg.Add(-1) - dialer := &NodeDialer{ - MetaClient: mc, - Timeout: 100 * time.Millisecond, //TODO: from config - } - - conn, err := dialer.DialNode(id) - if err != nil { - return - } - defer conn.Close() - mutex.Lock() - activeNodes[id] = struct{}{} - mutex.Unlock() - }(id) - } - - wg.Wait() - - shardIDsByNodeID := make(map[uint64][]uint64) - for _, si := range shards { - var nodeID uint64 - if si.OwnedBy(localNode.ID) { - nodeID = localNode.ID - } else if len(si.Owners) > 0 { - nodeID = si.Owners[rand.Intn(len(si.Owners))].NodeID - if _, ok := activeNodes[nodeID]; !ok { - //利用map的顺序不确定特性，随机选一个active的owner - randomOwners := make(map[uint64]struct{}) - for _, owner := range si.Owners { - randomOwners[owner.NodeID] = struct{}{} - } - for id, _ := range randomOwners { - if _, ok := activeNodes[id]; ok { - nodeID = id - break - } - } - } - } else { - continue - } - shardIDsByNodeID[nodeID] = append(shardIDsByNodeID[nodeID], si.ID) - } - - return shardIDsByNodeID -} - -type uint64Slice []uint64 - -func (a uint64Slice) Len() int { return len(a) } -func (a uint64Slice) Swap(i, j int) { a[i], a[j] = a[j], a[i] } -func (a uint64Slice) Less(i, j int) bool { return a[i] < a[j] } - -func (me Node2ShardIDs) Apply(fn func(nodeId uint64, shardIDs []uint64)) { - for nodeID, shardIDs := range me { - // Sort shard IDs so we get more predicable execution. - sort.Sort(uint64Slice(shardIDs)) - fn(nodeID, shardIDs) +func toShardIDs(shards []meta.ShardInfo) []uint64 { + ids := make([]uint64, len(shards)) + for i, shard := range shards { + ids[i] = shard.ID } + return ids } diff --git a/coordinator/cluster_meta_client.go b/coordinator/cluster_meta_client.go index 9cf0eee..9d7ea20 100644 --- a/coordinator/cluster_meta_client.go +++ b/coordinator/cluster_meta_client.go @@ -57,21 +57,18 @@ func (me *ClusterMetaClient) ClusterID() uint64 { } func (me *ClusterMetaClient) syncData() error { - fmt.Println("start sync data") + me.Logger.Info("start sync data") data, err := me.metaCli.Data() if err != nil { fmt.Println("start sync fail ", err) return err } - fmt.Println("start sync done") + me.Logger.Info("start sync done") return me.cache.ReplaceData(data) } -func (me *ClusterMetaClient) RunSyncLoop() { - //sync data first and will signal changes - me.syncData() - +func (me *ClusterMetaClient) syncLoop() { ticker := time.NewTicker(time.Duration(me.pingIntervalMs) * time.Millisecond) printLimiter := rate.NewLimiter(0.1, 1) for { @@ -102,6 +99,20 @@ func (me *ClusterMetaClient) RunSyncLoop() { } } +func (me *ClusterMetaClient) Start() { + wait := me.WaitForDataChanged() + //sync data first and will signal changes + me.syncData() + go me.syncLoop() + //wait sync meta data from meta server + select { + case <-time.After(5 * time.Second): + //TODO: + panic("sync meta data failed") + case <-wait: + } +} + func (me *ClusterMetaClient) CreateDatabase(name string) (*meta.DatabaseInfo, error) { if db, err := me.metaCli.CreateDatabase(name); err != nil { return db, err @@ -152,6 +163,7 @@ func (me *ClusterMetaClient) CreateDataNode(httpAddr, tcpAddr string) (*meta.Nod return me.cache.CreateDataNode(httpAddr, tcpAddr) } +// DataNode returns the node information according to the id, if it's not existed a meta.ErrNodeNotFound is returned. func (me *ClusterMetaClient) DataNode(id uint64) (*meta.NodeInfo, error) { return me.cache.DataNode(id) } diff --git a/coordinator/cluster_planning.go b/coordinator/cluster_planning.go new file mode 100644 index 0000000..8686a91 --- /dev/null +++ b/coordinator/cluster_planning.go @@ -0,0 +1,183 @@ +package coordinator + +import ( + "errors" + "fmt" + "math/rand" + "os" + "sync" + + "github.com/angopher/chronus/x" + "github.com/influxdata/influxdb/services/meta" +) + +var ( + ErrRetry = errors.New("operation needs another chance") +) + +// QueryFn returns ErrRetry indicating operation needs one more try on another node +type QueryFn func(nodeId uint64, shardIds []meta.ShardInfo) (interface{}, error) + +// getOwners returns the availible owners filtered by blacklist +func getOwners(blacklist map[uint64]bool, owners []meta.ShardOwner) []meta.ShardOwner { + if len(blacklist) < 1 { + return owners + } + arr := make([]meta.ShardOwner, 0, len(owners)) + for _, owner := range owners { + if _, ok := blacklist[owner.NodeID]; ok { + continue + } + arr = append(arr, owner) + } + return arr +} + +// PlanNodes distributes shards to correct nodes including those local node doesn't have +// Remote planning is randomized. +// blacklist is used during retries after queries fail on nodes +func PlanNodes(currentNodeId uint64, shard_queried []meta.ShardInfo, blacklist map[uint64]bool) Node2ShardIDs { + // shuffle the shards for randomized node selecting + shards := make([]meta.ShardInfo, len(shard_queried)) + copy(shards, shard_queried) + rand.Shuffle(len(shards), func(i, j int) { + shards[i], shards[j] = shards[j], shards[i] + }) + shardsMap := make(map[uint64][]meta.ShardInfo, x.Min(len(shards), 8)) +LOOP: + for _, shard := range shards { + // local shard first + if shard.OwnedBy(currentNodeId) { + shardsMap[currentNodeId] = append(shardsMap[currentNodeId], shard) + continue LOOP + } + // remote + owners := getOwners(blacklist, shard.Owners) + if len(owners) < 1 { + // no node availible, skip + continue LOOP + } + // node has been selected before has higher priority + for _, owner := range owners { + if _, ok := shardsMap[owner.NodeID]; ok { + shardsMap[owner.NodeID] = append(shardsMap[owner.NodeID], shard) + continue LOOP + } + } + // random pick + selectedNodeId := owners[rand.Intn(len(owners))].NodeID + shardsMap[selectedNodeId] = append(shardsMap[selectedNodeId], shard) + } + + return shardsMap +} + +type Node2ShardIDs map[uint64][]meta.ShardInfo + +type uint64Slice []uint64 + +func (a uint64Slice) Len() int { return len(a) } +func (a uint64Slice) Swap(i, j int) { a[i], a[j] = a[j], a[i] } +func (a uint64Slice) Less(i, j int) bool { return a[i] < a[j] } + +// ExecuteWithRetry executes the query with retry to guarantee all jobs could be executed successfully. +// errLast holds the last error captured but not means result should be empty if any error occurred +func (me Node2ShardIDs) ExecuteWithRetry(fn QueryFn) (result []interface{}, errLast error) { + var ( + lock sync.Mutex + wg sync.WaitGroup + ) + wg.Add(len(me)) + for nodeId := range me { + go func(n uint64, ss []meta.ShardInfo) { + defer wg.Done() + arr, err := executePlanWithRetry(n, ss, fn) + lock.Lock() + defer lock.Unlock() + if err != nil { + errLast = err + } + if len(arr) > 0 { + result = append(result, arr...) + } + }(nodeId, me[nodeId]) + } + wg.Wait() + return +} + +type planSchedule struct { + NodeID uint64 + Shards []meta.ShardInfo +} + +func executePlanWithRetry(nodeId uint64, shards []meta.ShardInfo, fn QueryFn) (result []interface{}, errLast error) { + var ( + skipNodes = make(map[uint64]bool) + arr []interface{} + shardsRemained []meta.ShardInfo + err error + ) + plans := []*planSchedule{ + {NodeID: nodeId, Shards: shards}, + } + for len(plans) > 0 { + arr, plans, err = executePlans(plans, fn) + if err != nil { + errLast = err + } + if len(arr) > 0 { + result = append(result, arr...) + } + if len(plans) > 0 { + // replan + shardsRemained = shardsRemained[:0] + for _, p := range plans { + skipNodes[p.NodeID] = true + shardsRemained = append(shardsRemained, p.Shards...) + } + replanned := PlanNodes(0, shardsRemained, skipNodes) + plans = plans[:0] + cnt := 0 + for n, ss := range replanned { + plans = append(plans, &planSchedule{n, ss}) + cnt += len(ss) + } + if cnt < len(shardsRemained) { + // not all shards can be planned + fmt.Fprintln(os.Stderr, "Not all shards can be replanned") + } + } + } + return +} + +func executePlans(plans []*planSchedule, fn QueryFn) (result []interface{}, need_retry []*planSchedule, err_last error) { + for _, plan := range plans { + obj, err := executePlanSingle(plan.NodeID, plan.Shards, fn) + if err == nil { + if obj != nil { + result = append(result, obj) + } + continue + } + fmt.Fprintln(os.Stderr, "execute remotely error:", err) + if err != ErrRetry { + err_last = err + continue + } + // need retry + need_retry = append(need_retry, plan) + } + return +} + +func executePlanSingle(nodeId uint64, shards []meta.ShardInfo, fn QueryFn) (interface{}, error) { + defer func() { + r := recover() + if r != nil { + fmt.Fprintln(os.Stderr, "panic recover:", r) + } + }() + return fn(nodeId, shards) +} diff --git a/coordinator/cluster_planning_test.go b/coordinator/cluster_planning_test.go new file mode 100644 index 0000000..792d831 --- /dev/null +++ b/coordinator/cluster_planning_test.go @@ -0,0 +1,139 @@ +package coordinator + +import ( + "testing" + + "github.com/influxdata/influxdb/services/meta" + "github.com/stretchr/testify/assert" +) + +func TestGetOwners(t *testing.T) { + owners := []meta.ShardOwner{ + {NodeID: 1}, + {NodeID: 2}, + {NodeID: 3}, + {NodeID: 4}, + {NodeID: 5}, + {NodeID: 6}, + } + ori_len := len(owners) + + assert.Equal(t, ori_len, len(getOwners(nil, owners))) + assert.Equal(t, ori_len, len(getOwners(map[uint64]bool{}, owners))) + assert.Equal(t, ori_len, len(getOwners(map[uint64]bool{ + 0: true, + }, owners))) + assert.Equal(t, ori_len-1, len(getOwners(map[uint64]bool{ + 0: true, + 1: true, + }, owners))) + assert.Equal(t, ori_len-1, len(getOwners(map[uint64]bool{ + 0: true, + 4: true, + }, owners))) + assert.Equal(t, ori_len-2, len(getOwners(map[uint64]bool{ + 0: true, + 2: true, + 4: true, + }, owners))) + assert.Equal(t, 0, len(getOwners(map[uint64]bool{ + 0: true, + 1: true, + 2: true, + 3: true, + 4: true, + 5: true, + 6: true, + }, owners))) +} + +func verifyPlan(t *testing.T, plan Node2ShardIDs, shards []meta.ShardInfo) bool { + shardMap := make(map[uint64]meta.ShardInfo, len(shards)) + for _, shard := range shards { + shardMap[shard.ID] = shard + } + for nodeID, ss := range plan { + for _, s := range ss { + shard, ok := shardMap[s.ID] + assert.True(t, ok) + if !ok { + return false + } + assert.True(t, shard.OwnedBy(nodeID)) + if !shard.OwnedBy(nodeID) { + return false + } + delete(shardMap, s.ID) + } + } + assert.Empty(t, shardMap) + return true +} + +func TestPlanNodes(t *testing.T) { + shards := []meta.ShardInfo{ + { + ID: 1, + Owners: []meta.ShardOwner{ + {NodeID: 1}, + {NodeID: 3}, + }, + }, { + ID: 2, + Owners: []meta.ShardOwner{ + {NodeID: 2}, + {NodeID: 4}, + }, + }, { + ID: 3, + Owners: []meta.ShardOwner{ + {NodeID: 1}, + {NodeID: 5}, + }, + }, { + ID: 4, + Owners: []meta.ShardOwner{ + {NodeID: 2}, + {NodeID: 5}, + }, + }, + } + + result := PlanNodes(0, shards, nil) + assert.True(t, verifyPlan(t, result, shards)) + result = PlanNodes(0, shards, nil) + assert.True(t, verifyPlan(t, result, shards)) + result = PlanNodes(0, shards, nil) + assert.True(t, verifyPlan(t, result, shards)) + result = PlanNodes(0, shards, nil) + assert.True(t, verifyPlan(t, result, shards)) + result = PlanNodes(0, shards, nil) + assert.True(t, verifyPlan(t, result, shards)) + result = PlanNodes(0, shards, nil) + assert.True(t, verifyPlan(t, result, shards)) + + // local node + resultLocal := PlanNodes(1, shards, nil) + assert.True(t, verifyPlan(t, resultLocal, shards)) + result = PlanNodes(1, shards, nil) + assert.True(t, verifyPlan(t, result, shards)) + assert.Equal(t, len(resultLocal[1]), len(result[1])) + result = PlanNodes(1, shards, nil) + assert.True(t, verifyPlan(t, result, shards)) + assert.Equal(t, len(resultLocal[1]), len(result[1])) + result = PlanNodes(1, shards, nil) + assert.True(t, verifyPlan(t, result, shards)) + assert.Equal(t, len(resultLocal[1]), len(result[1])) + + // black list + result = PlanNodes(0, shards, map[uint64]bool{ + 1: true, + }) + assert.True(t, verifyPlan(t, result, shards)) + assert.Equal(t, 0, len(result[1])) + result = PlanNodes(0, shards, map[uint64]bool{ + 2: true, + }) + assert.True(t, verifyPlan(t, result, shards)) + assert.Equal(t, 0, len(result[2])) +} diff --git a/coordinator/config.go b/coordinator/config.go index c9bfbd0..e5817d5 100644 --- a/coordinator/config.go +++ b/coordinator/config.go @@ -23,8 +23,7 @@ const ( // A value of zero will make the maximum query limit unlimited. DefaultMaxConcurrentQueries = 0 - DefaultPoolMaxIdleStreams = 100 - DefaultPoolMaxConnections = 200 + DefaultPoolMaxStreamsPerNode = 200 // DefaultMaxSelectPointN is the maximum number of points a SELECT can process. // A value of zero will make the maximum point count unlimited. @@ -41,8 +40,7 @@ const ( type Config struct { DailTimeout toml.Duration `toml:"dial-timeout"` PoolMaxIdleTimeout toml.Duration `toml:"pool-max-idle-time"` - PoolMaxIdleStreams int `toml:"pool-max-idle-streams"` - PoolMaxConnections int `toml:"pool-max-connections"` + PoolMaxStreamsPerNode int `toml:"pool-max-streams-per-node"` ShardReaderTimeout toml.Duration `toml:"shard-reader-timeout"` ClusterTracing bool `toml:"cluster-tracing"` WriteTimeout toml.Duration `toml:"write-timeout"` @@ -61,8 +59,7 @@ func NewConfig() Config { return Config{ DailTimeout: toml.Duration(DefaultDialTimeout), PoolMaxIdleTimeout: toml.Duration(DefaultPoolMaxIdleTimeout), - PoolMaxIdleStreams: DefaultPoolMaxIdleStreams, - PoolMaxConnections: DefaultPoolMaxConnections, + PoolMaxStreamsPerNode: DefaultPoolMaxStreamsPerNode, ShardReaderTimeout: toml.Duration(DefaultShardReaderTimeout), ClusterTracing: false, WriteTimeout: toml.Duration(DefaultWriteTimeout), @@ -80,8 +77,7 @@ func (c Config) Diagnostics() (*diagnostics.Diagnostics, error) { return diagnostics.RowFromMap(map[string]interface{}{ "dail-timeout": c.DailTimeout, "pool-max-idle-time": c.PoolMaxIdleTimeout, - "pool-max-idle-streams": c.PoolMaxIdleStreams, - "pool-max-connections": c.PoolMaxConnections, + "pool-max-streams-per-node": c.PoolMaxStreamsPerNode, "shard-reader-timeout": c.ShardReaderTimeout, "cluster-tracing": c.ClusterTracing, "write-timeout": c.WriteTimeout, diff --git a/coordinator/pool.go b/coordinator/pool.go deleted file mode 100644 index 8e15b00..0000000 --- a/coordinator/pool.go +++ /dev/null @@ -1,188 +0,0 @@ -package coordinator - -import ( - "errors" - "fmt" - "net" - "sync" - "sync/atomic" - "time" - - "gopkg.in/fatih/pool.v2" -) - -// boundedPool implements the Pool interface based on buffered channels. -type boundedPool struct { - // storage for our net.Conn connections - mu sync.Mutex - conns chan net.Conn - - timeout time.Duration - total int32 - // net.Conn generator - factory Factory -} - -// Factory is a function to create new connections. -type Factory func() (net.Conn, error) - -// NewBoundedPool returns a new pool based on buffered channels with an initial -// capacity, maximum capacity and timeout to wait for a connection from the pool. -// Factory is used when initial capacity is -// greater than zero to fill the pool. A zero initialCap doesn't fill the Pool -// until a new Get() is called. During a Get(), If there is no new connection -// available in the pool and total connections is less than the max, a new connection -// will be created via the Factory() method. Othewise, the call will block until -// a connection is available or the timeout is reached. -func NewBoundedPool(initialCap, maxCap int, timeout time.Duration, factory Factory) (pool.Pool, error) { - if initialCap < 0 || maxCap <= 0 || initialCap > maxCap { - return nil, errors.New("invalid capacity settings") - } - - c := &boundedPool{ - conns: make(chan net.Conn, maxCap), - factory: factory, - timeout: timeout, - } - - // create initial connections, if something goes wrong, - // just close the pool error out. - for i := 0; i < initialCap; i++ { - conn, err := factory() - if err != nil { - c.Close() - return nil, fmt.Errorf("factory is not able to fill the pool: %s", err) - } - c.conns <- conn - atomic.AddInt32(&c.total, 1) - } - - return c, nil -} - -func (c *boundedPool) getConns() chan net.Conn { - c.mu.Lock() - conns := c.conns - c.mu.Unlock() - return conns -} - -// Get implements the Pool interfaces Get() method. If there is no new -// connection available in the pool, a new connection will be created via the -// Factory() method. -func (c *boundedPool) Get() (net.Conn, error) { - conns := c.getConns() - if conns == nil { - return nil, pool.ErrClosed - } - - // Try and grab a connection from the pool - select { - case conn := <-conns: - if conn == nil { - return nil, pool.ErrClosed - } - return c.wrapConn(conn), nil - default: - // Could not get connection, can we create a new one? - if atomic.LoadInt32(&c.total) < int32(cap(conns)) { - conn, err := c.factory() - if err != nil { - return nil, err - } - atomic.AddInt32(&c.total, 1) - - return c.wrapConn(conn), nil - } - } - - // The pool was empty and we couldn't create a new one to - // retry until one is free or we timeout - select { - case conn := <-conns: - if conn == nil { - return nil, pool.ErrClosed - } - return c.wrapConn(conn), nil - case <-time.After(c.timeout): - return nil, fmt.Errorf("timed out waiting for free connection") - } - -} - -// put puts the connection back to the pool. If the pool is full or closed, -// conn is simply closed. A nil conn will be rejected. -func (c *boundedPool) put(conn net.Conn) error { - if conn == nil { - return errors.New("connection is nil. rejecting") - } - - c.mu.Lock() - defer c.mu.Unlock() - - if c.conns == nil { - // pool is closed, close passed connection - return conn.Close() - } - - // put the resource back into the pool. If the pool is full, this will - // block and the default case will be executed. - select { - case c.conns <- conn: - return nil - default: - // pool is full, close passed connection - return conn.Close() - } -} - -func (c *boundedPool) Close() { - c.mu.Lock() - conns := c.conns - c.conns = nil - c.factory = nil - c.mu.Unlock() - - if conns == nil { - return - } - - close(conns) - for conn := range conns { - conn.Close() - } -} - -func (c *boundedPool) Len() int { return len(c.getConns()) } - -// newConn wraps a standard net.Conn to a poolConn net.Conn. -func (c *boundedPool) wrapConn(conn net.Conn) net.Conn { - p := &pooledConn{c: c} - p.Conn = conn - return p -} - -// pooledConn is a wrapper around net.Conn to modify the the behavior of -// net.Conn's Close() method. -type pooledConn struct { - net.Conn - c *boundedPool - unusable bool -} - -// Close() puts the given connects back to the pool instead of closing it. -func (p pooledConn) Close() error { - if p.unusable { - if p.Conn != nil { - return p.Conn.Close() - } - return nil - } - return p.c.put(p.Conn) -} - -// MarkUnusable() marks the connection not usable any more, to let the pool close it instead of returning it to pool. -func (p *pooledConn) MarkUnusable() { - p.unusable = true - atomic.AddInt32(&p.c.total, -1) -} diff --git a/coordinator/remote_executor.go b/coordinator/remote_executor.go index 9ee5381..ddbeb6d 100644 --- a/coordinator/remote_executor.go +++ b/coordinator/remote_executor.go @@ -2,18 +2,22 @@ package coordinator import ( "context" + "encoding/binary" "errors" "net" "time" + "github.com/angopher/chronus/x" "github.com/influxdata/influxdb/pkg/tracing" "github.com/influxdata/influxdb/query" - "github.com/influxdata/influxdb/services/meta" "github.com/influxdata/influxdb/tsdb" "github.com/influxdata/influxql" + "go.uber.org/zap" ) type RemoteNodeExecutor interface { + WithLogger(log *zap.Logger) + TagKeys(nodeId uint64, ShardIDs []uint64, cond influxql.Expr) ([]tsdb.TagKeys, error) TagValues(nodeId uint64, shardIDs []uint64, cond influxql.Expr) ([]tsdb.TagValues, error) MeasurementNames(nodeId uint64, database string, cond influxql.Expr) ([][]byte, error) @@ -29,21 +33,55 @@ type RemoteNodeExecutor interface { } type remoteNodeExecutor struct { - MetaClient interface { - DataNode(nodeId uint64) (*meta.NodeInfo, error) - } + ClientPool *ClientPool DailTimeout time.Duration ShardReaderTimeout time.Duration ClusterTracing bool + Logger *zap.Logger } -func (me *remoteNodeExecutor) CreateIterator(nodeId uint64, ctx context.Context, m *influxql.Measurement, opt query.IteratorOptions, shardIds []uint64) (query.Iterator, error) { - dialer := &NodeDialer{ - MetaClient: me.MetaClient, - Timeout: me.ShardReaderTimeout, +func (me *remoteNodeExecutor) WithLogger(log *zap.Logger) { + me.Logger = log.With(zap.String("service", "RemoteNodeExecutor")) +} + +func writeTestPacket(conn net.Conn) (err error) { + conn.SetDeadline(time.Now().Add(500 * time.Millisecond)) + defer conn.SetDeadline(time.Time{}) + typ := testRequestMessage + size := uint64(0) + if err = binary.Write(conn, binary.BigEndian, &typ); err != nil { + return + } + if err = binary.Write(conn, binary.BigEndian, &size); err != nil { + return + } + return +} + +func getConnWithRetry(pool *ClientPool, nodeId uint64, logger *zap.Logger) (x.PooledConn, error) { + var ( + conn x.PooledConn + err error + ) + retries := 3 + for retries > 0 { + retries-- + conn, err = pool.GetConn(nodeId) + if err != nil { + logger.Warn("Failed to get connection from pool", zap.Error(err)) + return nil, ErrRetry + } + // do write test + if err = writeTestPacket(conn); err != nil { + logger.Warn("Failed to get connection from pool", zap.Error(err)) + continue + } } + return conn, err +} - conn, err := dialer.DialNode(nodeId) +func (me *remoteNodeExecutor) CreateIterator(nodeId uint64, ctx context.Context, m *influxql.Measurement, opt query.IteratorOptions, shardIds []uint64) (query.Iterator, error) { + conn, err := getConnWithRetry(me.ClientPool, nodeId, me.Logger) if err != nil { return nil, err } @@ -63,11 +101,13 @@ func (me *remoteNodeExecutor) CreateIterator(nodeId uint64, ctx context.Context, Opt: opt, SpanContex: spanCtx, }); err != nil { + conn.MarkUnusable() return err } // Read the response. if _, err := DecodeTLV(conn, &resp); err != nil { + conn.MarkUnusable() return err } else if resp.Err != nil { return err @@ -90,12 +130,7 @@ func (me *remoteNodeExecutor) CreateIterator(nodeId uint64, ctx context.Context, } func (me *remoteNodeExecutor) MapType(nodeId uint64, m *influxql.Measurement, field string, shardIds []uint64) (influxql.DataType, error) { - dialer := &NodeDialer{ - MetaClient: me.MetaClient, - Timeout: me.DailTimeout, - } - - conn, err := dialer.DialNode(nodeId) + conn, err := getConnWithRetry(me.ClientPool, nodeId, me.Logger) if err != nil { return influxql.Unknown, err } @@ -113,10 +148,12 @@ func (me *remoteNodeExecutor) MapType(nodeId uint64, m *influxql.Measurement, fi Field: field, ShardIDs: shardIds, }); err != nil { + conn.MarkUnusable() return err } if _, err := DecodeTLV(conn, &resp); err != nil { + conn.MarkUnusable() return err } else if resp.Err != "" { return errors.New(resp.Err) @@ -131,12 +168,7 @@ func (me *remoteNodeExecutor) MapType(nodeId uint64, m *influxql.Measurement, fi } func (me *remoteNodeExecutor) IteratorCost(nodeId uint64, m *influxql.Measurement, opt query.IteratorOptions, shardIds []uint64) (query.IteratorCost, error) { - dialer := &NodeDialer{ - MetaClient: me.MetaClient, - Timeout: me.DailTimeout, - } - - conn, err := dialer.DialNode(nodeId) + conn, err := getConnWithRetry(me.ClientPool, nodeId, me.Logger) if err != nil { return query.IteratorCost{}, err } @@ -149,10 +181,12 @@ func (me *remoteNodeExecutor) IteratorCost(nodeId uint64, m *influxql.Measuremen Opt: opt, ShardIDs: shardIds, }); err != nil { + conn.MarkUnusable() return err } if _, err := DecodeTLV(conn, &resp); err != nil { + conn.MarkUnusable() return err } else if resp.Err != "" { return errors.New(resp.Err) @@ -167,12 +201,7 @@ func (me *remoteNodeExecutor) IteratorCost(nodeId uint64, m *influxql.Measuremen } func (me *remoteNodeExecutor) FieldDimensions(nodeId uint64, m *influxql.Measurement, shardIds []uint64) (fields map[string]influxql.DataType, dimensions map[string]struct{}, err error) { - dialer := &NodeDialer{ - MetaClient: me.MetaClient, - Timeout: me.DailTimeout, - } - - conn, err := dialer.DialNode(nodeId) + conn, err := getConnWithRetry(me.ClientPool, nodeId, me.Logger) if err != nil { return nil, nil, err } @@ -184,10 +213,12 @@ func (me *remoteNodeExecutor) FieldDimensions(nodeId uint64, m *influxql.Measure req.ShardIDs = shardIds req.Sources = influxql.Sources([]influxql.Source{m}) if err := EncodeTLV(conn, fieldDimensionsRequestMessage, &req); err != nil { + conn.MarkUnusable() return err } if _, err := DecodeTLV(conn, &resp); err != nil { + conn.MarkUnusable() return err } else if resp.Err != nil { return resp.Err @@ -202,12 +233,7 @@ func (me *remoteNodeExecutor) FieldDimensions(nodeId uint64, m *influxql.Measure } func (me *remoteNodeExecutor) TaskManagerStatement(nodeId uint64, stmt influxql.Statement) (*query.Result, error) { - dialer := &NodeDialer{ - MetaClient: me.MetaClient, - Timeout: me.DailTimeout, - } - - conn, err := dialer.DialNode(nodeId) + conn, err := getConnWithRetry(me.ClientPool, nodeId, me.Logger) if err != nil { return nil, err } @@ -219,10 +245,12 @@ func (me *remoteNodeExecutor) TaskManagerStatement(nodeId uint64, stmt influxql. req.SetStatement(stmt.String()) req.SetDatabase("") if err := EncodeTLV(conn, executeTaskManagerRequestMessage, &req); err != nil { + conn.MarkUnusable() return err } if _, err := DecodeTLV(conn, &resp); err != nil { + conn.MarkUnusable() return err } else if resp.Err != "" { return errors.New(resp.Err) @@ -239,12 +267,7 @@ func (me *remoteNodeExecutor) TaskManagerStatement(nodeId uint64, stmt influxql. } func (me *remoteNodeExecutor) SeriesCardinality(nodeId uint64, database string) (int64, error) { - dialer := &NodeDialer{ - MetaClient: me.MetaClient, - Timeout: me.DailTimeout, - } - - conn, err := dialer.DialNode(nodeId) + conn, err := getConnWithRetry(me.ClientPool, nodeId, me.Logger) if err != nil { return -1, err } @@ -255,11 +278,13 @@ func (me *remoteNodeExecutor) SeriesCardinality(nodeId uint64, database string) if err := EncodeTLV(conn, seriesCardinalityRequestMessage, &SeriesCardinalityRequest{ Database: database, }); err != nil { + conn.MarkUnusable() return err } var resp SeriesCardinalityResponse if _, err := DecodeTLV(conn, &resp); err != nil { + conn.MarkUnusable() return err } else if resp.Err != "" { return errors.New(resp.Err) @@ -275,12 +300,7 @@ func (me *remoteNodeExecutor) SeriesCardinality(nodeId uint64, database string) } func (me *remoteNodeExecutor) MeasurementNames(nodeId uint64, database string, cond influxql.Expr) ([][]byte, error) { - dialer := &NodeDialer{ - MetaClient: me.MetaClient, - Timeout: me.DailTimeout, - } - - conn, err := dialer.DialNode(nodeId) + conn, err := getConnWithRetry(me.ClientPool, nodeId, me.Logger) if err != nil { return nil, err } @@ -292,11 +312,13 @@ func (me *remoteNodeExecutor) MeasurementNames(nodeId uint64, database string, c Database: database, Cond: cond, }); err != nil { + conn.MarkUnusable() return err } var resp MeasurementNamesResponse if _, err := DecodeTLV(conn, &resp); err != nil { + conn.MarkUnusable() return err } else if resp.Err != "" { return errors.New(resp.Err) @@ -312,12 +334,7 @@ func (me *remoteNodeExecutor) MeasurementNames(nodeId uint64, database string, c } func (me *remoteNodeExecutor) TagValues(nodeId uint64, shardIDs []uint64, cond influxql.Expr) ([]tsdb.TagValues, error) { - dialer := &NodeDialer{ - MetaClient: me.MetaClient, - Timeout: me.DailTimeout, - } - - conn, err := dialer.DialNode(nodeId) + conn, err := getConnWithRetry(me.ClientPool, nodeId, me.Logger) if err != nil { return nil, err } @@ -331,11 +348,13 @@ func (me *remoteNodeExecutor) TagValues(nodeId uint64, shardIDs []uint64, cond i Cond: cond, }, }); err != nil { + conn.MarkUnusable() return err } var resp TagValuesResponse if _, err := DecodeTLV(conn, &resp); err != nil { + conn.MarkUnusable() return err } else if resp.Err != "" { return errors.New(resp.Err) @@ -351,16 +370,10 @@ func (me *remoteNodeExecutor) TagValues(nodeId uint64, shardIDs []uint64, cond i } func (me *remoteNodeExecutor) TagKeys(nodeId uint64, shardIDs []uint64, cond influxql.Expr) ([]tsdb.TagKeys, error) { - dialer := &NodeDialer{ - MetaClient: me.MetaClient, - Timeout: me.DailTimeout, - } - - conn, err := dialer.DialNode(nodeId) + conn, err := getConnWithRetry(me.ClientPool, nodeId, me.Logger) if err != nil { return nil, err } - defer conn.Close() var tagKeys []tsdb.TagKeys @@ -369,11 +382,13 @@ func (me *remoteNodeExecutor) TagKeys(nodeId uint64, shardIDs []uint64, cond inf ShardIDs: shardIDs, Cond: cond, }); err != nil { + conn.MarkUnusable() return err } var resp TagKeysResponse if _, err := DecodeTLV(conn, &resp); err != nil { + conn.MarkUnusable() return err } else if resp.Err != "" { return errors.New(resp.Err) @@ -389,12 +404,7 @@ func (me *remoteNodeExecutor) TagKeys(nodeId uint64, shardIDs []uint64, cond inf } func (me *remoteNodeExecutor) DeleteMeasurement(nodeId uint64, database, name string) error { - dialer := &NodeDialer{ - MetaClient: me.MetaClient, - Timeout: me.DailTimeout, - } - - conn, err := dialer.DialNode(nodeId) + conn, err := getConnWithRetry(me.ClientPool, nodeId, me.Logger) if err != nil { return err } @@ -405,11 +415,13 @@ func (me *remoteNodeExecutor) DeleteMeasurement(nodeId uint64, database, name st Database: database, Name: name, }); err != nil { + conn.MarkUnusable() return err } var resp DeleteMeasurementResponse if _, err := DecodeTLV(conn, &resp); err != nil { + conn.MarkUnusable() return err } else if resp.Err != "" { return errors.New(resp.Err) @@ -424,12 +436,7 @@ func (me *remoteNodeExecutor) DeleteMeasurement(nodeId uint64, database, name st } func (me *remoteNodeExecutor) DeleteDatabase(nodeId uint64, database string) error { - dialer := &NodeDialer{ - MetaClient: me.MetaClient, - Timeout: me.DailTimeout, - } - - conn, err := dialer.DialNode(nodeId) + conn, err := getConnWithRetry(me.ClientPool, nodeId, me.Logger) if err != nil { return err } @@ -439,11 +446,13 @@ func (me *remoteNodeExecutor) DeleteDatabase(nodeId uint64, database string) err if err := EncodeTLV(conn, deleteDatabaseRequestMessage, &DeleteDatabaseRequest{ Database: database, }); err != nil { + conn.MarkUnusable() return err } var resp DeleteDatabaseResponse if _, err := DecodeTLV(conn, &resp); err != nil { + conn.MarkUnusable() return err } else if resp.Err != "" { return errors.New(resp.Err) @@ -458,12 +467,7 @@ func (me *remoteNodeExecutor) DeleteDatabase(nodeId uint64, database string) err } func (me *remoteNodeExecutor) DeleteSeries(nodeId uint64, database string, sources []influxql.Source, cond influxql.Expr) error { - dialer := &NodeDialer{ - MetaClient: me.MetaClient, - Timeout: me.DailTimeout, - } - - conn, err := dialer.DialNode(nodeId) + conn, err := getConnWithRetry(me.ClientPool, nodeId, me.Logger) if err != nil { return err } @@ -475,11 +479,13 @@ func (me *remoteNodeExecutor) DeleteSeries(nodeId uint64, database string, sourc Sources: influxql.Sources(sources), Cond: cond, }); err != nil { + conn.MarkUnusable() return err } var resp DeleteSeriesResponse if _, err := DecodeTLV(conn, &resp); err != nil { + conn.MarkUnusable() return err } else if resp.Err != "" { return errors.New(resp.Err) @@ -492,33 +498,3 @@ func (me *remoteNodeExecutor) DeleteSeries(nodeId uint64, database string, sourc return nil } - -// NodeDialer dials connections to a given node. -type NodeDialer struct { - MetaClient interface { - DataNode(nodeId uint64) (*meta.NodeInfo, error) - } - Timeout time.Duration -} - -// DialNode returns a connection to a node. -func (d *NodeDialer) DialNode(nodeID uint64) (net.Conn, error) { - ni, err := d.MetaClient.DataNode(nodeID) - if err != nil { - return nil, err - } - - conn, err := net.Dial("tcp", ni.TCPHost) - if err != nil { - return nil, err - } - conn.SetDeadline(time.Now().Add(d.Timeout)) - - // Write the cluster multiplexing header byte - if _, err := conn.Write([]byte{MuxHeader}); err != nil { - conn.Close() - return nil, err - } - - return conn, nil -} diff --git a/coordinator/service.go b/coordinator/service.go index 3a97d16..f062eba 100644 --- a/coordinator/service.go +++ b/coordinator/service.go @@ -25,6 +25,15 @@ import ( "github.com/influxdata/influxdb/services/meta" ) +// Service and remote executor are designed for "forwarded" requests +// between shards/nodes. For better performance we separate the design +// into several phases: +// 1. Introduce connection pooling avoiding create/destroy connections frequently. +// 2. Introduce multiplexing to reduce the cost of connections further more. +// But we should also notice that more than one connection are needed to avoid flow control of TCP. +// Now we implement connection pooling first and keep an versioning api which enables +// data node declaring its running version. + // MaxMessageSize defines how large a message can be before we reject it const MaxMessageSize = 1024 * 1024 * 1024 // 1GB @@ -62,6 +71,12 @@ const ( mapTypeFail = "mapTypeFail" ) +type ServerResponse interface { + SetCode(int) + SetMessage(string) + MarshalBinary() ([]byte, error) +} + // Service processes data received over raw TCP connections. type Service struct { mu sync.RWMutex @@ -226,117 +241,137 @@ func (s *Service) Close() error { return nil } +func (s *Service) handle(conn net.Conn, typ byte, data []byte) (respType byte, resp encoding.BinaryMarshaler, err error) { + // Delegate message processing by type. + switch typ { + case writeShardRequestMessage: + err = s.processWriteShardRequest(data) + respType = writeShardResponseMessage + resp = &WriteShardResponse{} + case executeStatementRequestMessage: + err = s.processExecuteStatementRequest(data) + respType = writeShardResponseMessage + resp = &WriteShardResponse{} + case createIteratorRequestMessage: + s.processCreateIteratorRequest(conn, data) + case fieldDimensionsRequestMessage: + respType = fieldDimensionsResponseMessage + resp, err = s.processFieldDimensionsRequest(data) + case tagKeysRequestMessage: + respType = tagKeysResponseMessage + resp, err = s.processTagKeysRequest(data) + case tagValuesRequestMessage: + respType = tagValuesResponseMessage + resp, err = s.processTagValuesRequest(data) + case measurementNamesRequestMessage: + respType = measurementNamesResponseMessage + resp, err = s.processMeasurementNamesRequest(data) + case seriesCardinalityRequestMessage: + respType = seriesCardinalityResponseMessage + resp, err = s.processSeriesCardinalityRequest(data) + case deleteSeriesRequestMessage: + respType = deleteSeriesResponseMessage + resp, err = s.processDeleteSeriesRequest(data) + case deleteDatabaseRequestMessage: + respType = deleteDatabaseResponseMessage + resp, err = s.processDeleteDatabaseRequest(data) + case deleteMeasurementRequestMessage: + respType = deleteMeasurementResponseMessage + resp, err = s.processDeleteMeasurementRequest(data) + case iteratorCostRequestMessage: + respType = iteratorCostResponseMessage + resp, err = s.processIteratorCostRequest(data) + case mapTypeRequestMessage: + respType = mapTypeResponseMessage + resp, err = s.processMapTypeRequest(data) + case executeTaskManagerRequestMessage: + respType = executeTaskManagerResponseMessage + resp, err = s.processTaskManagerRequest(data) + case testRequestMessage: + // do nothing + default: + s.Logger.Error("cluster service message type not found", zap.Uint8("type", typ)) + } + if err != nil { + s.Logger.Error("process request error", zap.Uint8("type", typ), zap.Error(err)) + } + if resp == nil { + return + } + if serverResp, ok := resp.(ServerResponse); ok { + if err != nil { + serverResp.SetCode(1) + serverResp.SetMessage(err.Error()) + } else { + serverResp.SetCode(0) + } + } + return +} + +func (s *Service) writeClusterResponse(conn net.Conn, respType byte, resp encoding.BinaryMarshaler) (err error) { + // Marshal response to binary. + buf, err := resp.MarshalBinary() + if err != nil { + s.Logger.Error("error marshalling response", zap.Uint8("respType", respType), zap.Error(err)) + return + } + + // Write to connection. + if err = WriteTLV(conn, respType, buf); err != nil { + s.Logger.Error("WriteTLV fail", zap.Error(err)) + } + return +} + // handleConn services an individual TCP connection. func (s *Service) handleConn(conn net.Conn) { // Ensure connection is closed when service is closed. - closing := make(chan struct{}) - defer close(closing) - go func() { - select { - case <-closing: - case <-s.closing: - } - conn.Close() - }() - - s.Logger.Info(fmt.Sprintf("accept remote connection from %+v", conn.RemoteAddr())) defer func() { + defer conn.Close() s.Logger.Info(fmt.Sprintf("close remote connection from %+v", conn.RemoteAddr())) }() + s.Logger.Info(fmt.Sprintf("accept remote connection from %+v", conn.RemoteAddr())) for { - // Read type-length-value. - typ, err := ReadType(conn) - if err != nil { - if strings.HasSuffix(err.Error(), "EOF") { - return - } - s.Logger.Error("unable to read type", zap.Error(err)) + select { + case <-s.closing: return + default: } - - // Delegate message processing by type. - switch typ { - case writeShardRequestMessage: - buf, err := ReadLV(conn) - if err != nil { - s.Logger.Error("unable to read length-value", zap.Error(err)) - return - } - - atomic.AddInt64(&s.stats.WriteShardReq, 1) - err = s.processWriteShardRequest(buf) - if err != nil { - s.Logger.Error("process write shard error", zap.Error(err)) - } - s.writeShardResponse(conn, err) - case executeStatementRequestMessage: - buf, err := ReadLV(conn) - if err != nil { - s.Logger.Error("unable to read length-value", zap.Error(err)) - return - } - - err = s.processExecuteStatementRequest(buf) - if err != nil { - s.Logger.Error("process execute statement error", zap.Error(err)) - } - s.writeShardResponse(conn, err) - case createIteratorRequestMessage: - atomic.AddInt64(&s.stats.CreateIteratorReq, 1) - s.processCreateIteratorRequest(conn) - return - case fieldDimensionsRequestMessage: - atomic.AddInt64(&s.stats.FieldDimensionsReq, 1) - s.processFieldDimensionsRequest(conn) - return - case tagKeysRequestMessage: - atomic.AddInt64(&s.stats.TagKeysReq, 1) - s.processTagKeysRequest(conn) - case tagValuesRequestMessage: - atomic.AddInt64(&s.stats.TagValuesReq, 1) - s.processTagValuesRequest(conn) - return - case measurementNamesRequestMessage: - atomic.AddInt64(&s.stats.MeasurementNamesReq, 1) - s.processMeasurementNamesRequest(conn) - return - case seriesCardinalityRequestMessage: - atomic.AddInt64(&s.stats.SeriesCardinalityReq, 1) - s.processSeriesCardinalityRequest(conn) + now := time.Now() + conn.SetReadDeadline(now.Add(500 * time.Millisecond)) + typ, data, err := ReadTLV(conn) + conn.SetDeadline(time.Time{}) + if err == io.EOF { return - case deleteSeriesRequestMessage: - s.processDeleteSeriesRequest(conn) - return - case deleteDatabaseRequestMessage: - s.processDeleteDatabaseRequest(conn) - return - case deleteMeasurementRequestMessage: - s.processDeleteMeasurementRequest(conn) - return - case iteratorCostRequestMessage: - atomic.AddInt64(&s.stats.IteratorCostReq, 1) - s.processIteratorCostRequest(conn) - return - case mapTypeRequestMessage: - atomic.AddInt64(&s.stats.MapTypeReq, 1) - s.processMapTypeRequest(conn) - return - case executeTaskManagerRequestMessage: - s.processTaskManagerRequest(conn) - return - default: - s.Logger.Error("cluster service message type not found", zap.Uint8("type", typ)) + } + if err, ok := err.(net.Error); ok && err.Timeout() { + continue + } + if err != nil { + s.Logger.Error("read error", zap.Error(err)) + break + } + respType, resp, err := s.handle(conn, typ, data) + if resp == nil { + continue + } + err = s.writeClusterResponse(conn, respType, resp) + if err != nil { + // close conn due to error in writing + break } } } -func (s *Service) processTaskManagerRequest(conn net.Conn) { - defer conn.Close() - - var resp TaskManagerStatementResponse - if err := func() error { +func (s *Service) processTaskManagerRequest(buf []byte) (*TaskManagerStatementResponse, error) { + var ( + resp TaskManagerStatementResponse + err error + ) + if err = func() error { var req TaskManagerStatementRequest - if err := DecodeLV(conn, &req); err != nil { + if err := req.UnmarshalBinary(buf); err != nil { return err } @@ -356,21 +391,20 @@ func (s *Service) processTaskManagerRequest(conn net.Conn) { resp.Result = *(<-recvCtx.Results) return nil }(); err != nil { - s.Logger.Error("s.processTaskManagerRequest fail", zap.Error(err)) resp.Err = err.Error() } - - if err := EncodeTLV(conn, executeTaskManagerResponseMessage, &resp); err != nil { - s.Logger.Error("s.processTaskManagerRequest EncodeTLV fail", zap.Error(err)) - } + return &resp, err } -func (s *Service) processMapTypeRequest(conn net.Conn) { - defer conn.Close() - var resp MapTypeResponse - if err := func() error { +func (s *Service) processMapTypeRequest(buf []byte) (*MapTypeResponse, error) { + var ( + resp MapTypeResponse + err error + ) + atomic.AddInt64(&s.stats.MapTypeReq, 1) + if err = func() error { var req MapTypeRequest - if err := DecodeLV(conn, &req); err != nil { + if err := req.UnmarshalBinary(buf); err != nil { return err } @@ -402,22 +436,20 @@ func (s *Service) processMapTypeRequest(conn net.Conn) { return nil }(); err != nil { atomic.AddInt64(&s.stats.MapTypeFail, 1) - s.Logger.Error("processMapTypeRequest fail", zap.Error(err)) resp.Err = err.Error() } - - if err := EncodeTLV(conn, mapTypeResponseMessage, &resp); err != nil { - atomic.AddInt64(&s.stats.MapTypeFail, 1) - s.Logger.Error("processMapTypeRequest EncodeTLV fail", zap.Error(err)) - } + return &resp, err } -func (s *Service) processIteratorCostRequest(conn net.Conn) { - defer conn.Close() - var resp IteratorCostResponse - if err := func() error { +func (s *Service) processIteratorCostRequest(buf []byte) (*IteratorCostResponse, error) { + var ( + resp IteratorCostResponse + err error + ) + atomic.AddInt64(&s.stats.IteratorCostReq, 1) + if err = func() error { var req IteratorCostRequest - if err := DecodeLV(conn, &req); err != nil { + if err := req.UnmarshalBinary(buf); err != nil { return err } @@ -427,7 +459,6 @@ func (s *Service) processIteratorCostRequest(conn net.Conn) { m := req.Sources[0].(*influxql.Measurement) opt := req.Opt - var err error sg := s.TSDBStore.ShardGroup(req.ShardIDs) if m.Regex != nil { resp.Cost, err = func() (query.IteratorCost, error) { @@ -448,60 +479,53 @@ func (s *Service) processIteratorCostRequest(conn net.Conn) { return err }(); err != nil { atomic.AddInt64(&s.stats.IteratorCostFail, 1) - s.Logger.Error("processIteratorCostRequest fail", zap.Error(err)) resp.Err = err.Error() } - - if err := EncodeTLV(conn, iteratorCostResponseMessage, &resp); err != nil { - atomic.AddInt64(&s.stats.IteratorCostFail, 1) - s.Logger.Error("processIteratorCostRequest EncodeTLV fail", zap.Error(err)) - } + return &resp, err } -func (s *Service) processDeleteMeasurementRequest(conn net.Conn) { - defer conn.Close() - var resp DeleteMeasurementResponse - if err := func() error { +func (s *Service) processDeleteMeasurementRequest(buf []byte) (*DeleteMeasurementResponse, error) { + var ( + resp DeleteMeasurementResponse + err error + ) + if err = func() error { var req DeleteMeasurementRequest - if err := DecodeLV(conn, &req); err != nil { + if err := req.UnmarshalBinary(buf); err != nil { return err } return s.TSDBStore.DeleteMeasurement(req.Database, req.Name) }(); err != nil { - s.Logger.Error("processDeleteMeasurementRequest fail", zap.Error(err)) resp.Err = err.Error() } - - if err := EncodeTLV(conn, deleteMeasurementResponseMessage, &resp); err != nil { - s.Logger.Error("processDeleteMeasurementRequest EncodeTLV fail", zap.Error(err)) - } + return &resp, err } -func (s *Service) processDeleteDatabaseRequest(conn net.Conn) { - defer conn.Close() - var resp DeleteDatabaseResponse - if err := func() error { +func (s *Service) processDeleteDatabaseRequest(buf []byte) (*DeleteDatabaseResponse, error) { + var ( + resp DeleteDatabaseResponse + err error + ) + if err = func() error { var req DeleteDatabaseRequest - if err := DecodeLV(conn, &req); err != nil { + if err := req.UnmarshalBinary(buf); err != nil { return err } return s.TSDBStore.DeleteDatabase(req.Database) }(); err != nil { - s.Logger.Error("processDeleteDatabaseRequest fail", zap.Error(err)) resp.Err = err.Error() } - - if err := EncodeTLV(conn, deleteDatabaseResponseMessage, &resp); err != nil { - s.Logger.Error("processDeleteDatabaseRequest EncodeTLV", zap.Error(err)) - } + return &resp, err } -func (s *Service) processDeleteSeriesRequest(conn net.Conn) { - defer conn.Close() - var resp DeleteSeriesResponse - if err := func() error { +func (s *Service) processDeleteSeriesRequest(buf []byte) (*DeleteSeriesResponse, error) { + var ( + resp DeleteSeriesResponse + err error + ) + if err = func() error { var req DeleteSeriesRequest - if err := DecodeLV(conn, &req); err != nil { + if err := req.UnmarshalBinary(buf); err != nil { return err } @@ -509,21 +533,20 @@ func (s *Service) processDeleteSeriesRequest(conn net.Conn) { err := s.TSDBStore.DeleteSeries(req.Database, req.Sources, cond) return err }(); err != nil { - s.Logger.Error("processDeleteSeriesRequest fail", zap.Error(err)) resp.Err = err.Error() } - - if err := EncodeTLV(conn, deleteSeriesResponseMessage, &resp); err != nil { - s.Logger.Error("processDeleteSeriesRequest EncodeTLV fail", zap.Error(err)) - } + return &resp, err } -func (s *Service) processSeriesCardinalityRequest(conn net.Conn) { - defer conn.Close() - var resp SeriesCardinalityResponse - if err := func() error { +func (s *Service) processSeriesCardinalityRequest(buf []byte) (*SeriesCardinalityResponse, error) { + var ( + resp SeriesCardinalityResponse + err error + ) + atomic.AddInt64(&s.stats.SeriesCardinalityReq, 1) + if err = func() error { var req SeriesCardinalityRequest - if err := DecodeLV(conn, &req); err != nil { + if err := req.UnmarshalBinary(buf); err != nil { return err } @@ -532,23 +555,19 @@ func (s *Service) processSeriesCardinalityRequest(conn net.Conn) { return err }(); err != nil { atomic.AddInt64(&s.stats.SeriesCardinalityFail, 1) - s.Logger.Error("processSeriesCardinalityRequest fail", zap.Error(err)) resp.Err = err.Error() } - - if err := EncodeTLV(conn, seriesCardinalityResponseMessage, &resp); err != nil { - atomic.AddInt64(&s.stats.SeriesCardinalityFail, 1) - s.Logger.Error("processSeriesCardinalityRequest EncodeTLV", zap.Error(err)) - } + return &resp, err } -func (s *Service) processMeasurementNamesRequest(conn net.Conn) { - defer conn.Close() - - var resp MeasurementNamesResponse - if err := func() error { +func (s *Service) processMeasurementNamesRequest(buf []byte) (*MeasurementNamesResponse, error) { + var ( + resp MeasurementNamesResponse + err error + ) + if err = func() error { var req MeasurementNamesRequest - if err := DecodeLV(conn, &req); err != nil { + if err := req.UnmarshalBinary(buf); err != nil { return err } @@ -556,24 +575,21 @@ func (s *Service) processMeasurementNamesRequest(conn net.Conn) { resp.Names, err = s.TSDBStore.MeasurementNames(nil, req.Database, req.Cond) return err }(); err != nil { - s.Logger.Error("processMeasurementNamesRequest fail", zap.Error(err)) atomic.AddInt64(&s.stats.MeasurementNamesFail, 1) resp.Err = err.Error() } - - if err := EncodeTLV(conn, measurementNamesResponseMessage, &resp); err != nil { - atomic.AddInt64(&s.stats.MeasurementNamesFail, 1) - s.Logger.Error("processMeasurementNamesRequest EncodeTLV fail", zap.Error(err)) - } + return &resp, err } -func (s *Service) processTagKeysRequest(conn net.Conn) { - defer conn.Close() - - var resp TagKeysResponse - if err := func() error { +func (s *Service) processTagKeysRequest(buf []byte) (*TagKeysResponse, error) { + var ( + resp TagKeysResponse + err error + ) + atomic.AddInt64(&s.stats.TagKeysReq, 1) + if err = func() error { var req TagKeysRequest - if err := DecodeLV(conn, &req); err != nil { + if err := req.UnmarshalBinary(buf); err != nil { return err } @@ -582,23 +598,20 @@ func (s *Service) processTagKeysRequest(conn net.Conn) { return err }(); err != nil { atomic.AddInt64(&s.stats.TagKeysFail, 1) - s.Logger.Error("processTagKeysRequest fail", zap.Error(err)) resp.Err = err.Error() } - - if err := EncodeTLV(conn, tagKeysResponseMessage, &resp); err != nil { - atomic.AddInt64(&s.stats.TagKeysFail, 1) - s.Logger.Error("processTagKeysRequest EncodeTLV", zap.Error(err)) - } + return &resp, err } -func (s *Service) processTagValuesRequest(conn net.Conn) { - defer conn.Close() - - var resp TagValuesResponse - if err := func() error { +func (s *Service) processTagValuesRequest(buf []byte) (*TagValuesResponse, error) { + var ( + resp TagValuesResponse + err error + ) + atomic.AddInt64(&s.stats.TagValuesReq, 1) + if err = func() error { var req TagValuesRequest - if err := DecodeLV(conn, &req); err != nil { + if err := req.UnmarshalBinary(buf); err != nil { return err } @@ -607,14 +620,9 @@ func (s *Service) processTagValuesRequest(conn net.Conn) { return err }(); err != nil { atomic.AddInt64(&s.stats.TagValuesFail, 1) - s.Logger.Error("processTagValuesRequest fail", zap.Error(err)) resp.Err = err.Error() } - - if err := EncodeTLV(conn, tagValuesResponseMessage, &resp); err != nil { - atomic.AddInt64(&s.stats.TagValuesFail, 1) - s.Logger.Error("processTagValuesRequest EncodeTLV fail", zap.Error(err)) - } + return &resp, err } func (s *Service) processExecuteStatementRequest(buf []byte) error { @@ -656,6 +664,8 @@ func (s *Service) processWriteShardRequest(buf []byte) error { } points := req.Points() + // stats + atomic.AddInt64(&s.stats.WriteShardReq, 1) atomic.AddInt64(&s.stats.WriteShardPointsReq, int64(len(points))) err := s.TSDBStore.WriteToShard(req.ShardID(), points) @@ -693,39 +703,21 @@ func (s *Service) processWriteShardRequest(buf []byte) error { return nil } -func (s *Service) writeShardResponse(w io.Writer, e error) { - // Build response. - var resp WriteShardResponse - if e != nil { - resp.SetCode(1) - resp.SetMessage(e.Error()) - } else { - resp.SetCode(0) - } - - // Marshal response to binary. - buf, err := resp.MarshalBinary() - if err != nil { - s.Logger.Error("error marshalling shard response", zap.Error(err)) - return - } - - // Write to connection. - if err := WriteTLV(w, writeShardResponseMessage, buf); err != nil { - s.Logger.Error("write shard WriteTLV fail", zap.Error(err)) - } -} - -func (s *Service) processCreateIteratorRequest(conn net.Conn) { - defer conn.Close() - +func (s *Service) processCreateIteratorRequest(conn net.Conn, buf []byte) { var itr query.Iterator var trace *tracing.Trace var span *tracing.Span + ioError := false + defer func() { + if ioError { + conn.Close() + } + }() + respType := createIteratorResponseMessage if err := func() error { // Parse request. var req CreateIteratorRequest - if err := DecodeLV(conn, &req); err != nil { + if err := req.UnmarshalBinary(buf); err != nil { return err } @@ -782,8 +774,9 @@ func (s *Service) processCreateIteratorRequest(conn net.Conn) { itr.Close() } s.Logger.Error("error reading CreateIterator request fail", zap.Error(err)) - if err = EncodeTLV(conn, createIteratorResponseMessage, &CreateIteratorResponse{Err: err}); err != nil { + if err = EncodeTLV(conn, respType, &CreateIteratorResponse{Err: err}); err != nil { s.Logger.Error("CreateIteratorRequest EncodeTLV fail", zap.Error(err)) + ioError = true } return } @@ -805,11 +798,14 @@ func (s *Service) processCreateIteratorRequest(conn net.Conn) { seriesN = itr.Stats().SeriesN } // Encode success response. - if err := EncodeTLV(conn, createIteratorResponseMessage, &CreateIteratorResponse{DataType: dataType, SeriesN: seriesN}); err != nil { + if err := EncodeTLV(conn, respType, &CreateIteratorResponse{DataType: dataType, SeriesN: seriesN}); err != nil { s.Logger.Error("error writing CreateIterator response, EncodeTLV fail", zap.Error(err)) atomic.AddInt64(&s.stats.CreateIteratorFail, 1) + ioError = true return } + //XXX + ioError = true // Exit if no iterator was produced. if itr == nil { @@ -820,6 +816,7 @@ func (s *Service) processCreateIteratorRequest(conn net.Conn) { if err := query.NewIteratorEncoder(conn).EncodeIterator(itr); err != nil { s.Logger.Error("encoding CreateIterator iterator fail", zap.Error(err)) atomic.AddInt64(&s.stats.CreateIteratorFail, 1) + ioError = true return } @@ -830,18 +827,23 @@ func (s *Service) processCreateIteratorRequest(conn net.Conn) { if err := query.NewIteratorEncoder(conn).EncodeTrace(trace); err != nil { s.Logger.Error("EncodeTrace fail", zap.Error(err)) atomic.AddInt64(&s.stats.CreateIteratorFail, 1) + ioError = true return } } } -func (s *Service) processFieldDimensionsRequest(conn net.Conn) { +func (s *Service) processFieldDimensionsRequest(buf []byte) (*FieldDimensionsResponse, error) { + var ( + err error + ) var fields map[string]influxql.DataType var dimensions map[string]struct{} - if err := func() error { + atomic.AddInt64(&s.stats.FieldDimensionsReq, 1) + if err = func() error { // Parse request. var req FieldDimensionsRequest - if err := DecodeLV(conn, &req); err != nil { + if err := req.UnmarshalBinary(buf); err != nil { return err } @@ -865,20 +867,12 @@ func (s *Service) processFieldDimensionsRequest(conn net.Conn) { return nil }(); err != nil { atomic.AddInt64(&s.stats.FieldDimensionsFail, 1) - s.Logger.Error("error reading FieldDimensions request", zap.Error(err)) - EncodeTLV(conn, fieldDimensionsResponseMessage, &FieldDimensionsResponse{Err: err}) - return + return &FieldDimensionsResponse{Err: err}, err } - - // Encode success response. - if err := EncodeTLV(conn, fieldDimensionsResponseMessage, &FieldDimensionsResponse{ + return &FieldDimensionsResponse{ Fields: fields, Dimensions: dimensions, - }); err != nil { - atomic.AddInt64(&s.stats.FieldDimensionsFail, 1) - s.Logger.Error("error writing FieldDimensions response", zap.Error(err)) - return - } + }, err } // ReadTLV reads a type-length-value record from r. @@ -899,7 +893,7 @@ func ReadTLV(r io.Reader) (byte, []byte, error) { func ReadType(r io.Reader) (byte, error) { var typ [1]byte if _, err := io.ReadFull(r, typ[:]); err != nil { - return 0, fmt.Errorf("read message type: %s", err) + return 0, err } return typ[0], nil } @@ -909,17 +903,22 @@ func ReadLV(r io.Reader) ([]byte, error) { // Read the size of the message. var sz int64 if err := binary.Read(r, binary.BigEndian, &sz); err != nil { - return nil, fmt.Errorf("read message size: %s", err) + return nil, err } if sz >= MaxMessageSize { return nil, fmt.Errorf("max message size of %d exceeded: %d", MaxMessageSize, sz) } + if sz == 0 { + // empty msg + return []byte{}, nil + } + // Read the value. buf := make([]byte, sz) if _, err := io.ReadFull(r, buf); err != nil { - return nil, fmt.Errorf("read message value: %s", err) + return nil, err } return buf, nil @@ -939,7 +938,7 @@ func WriteTLV(w io.Writer, typ byte, buf []byte) error { // WriteType writes the type in a TLV record to w. func WriteType(w io.Writer, typ byte) error { if _, err := w.Write([]byte{typ}); err != nil { - return fmt.Errorf("write message type: %s", err) + return err } return nil } @@ -948,12 +947,12 @@ func WriteType(w io.Writer, typ byte) error { func WriteLV(w io.Writer, buf []byte) error { // Write the size of the message. if err := binary.Write(w, binary.BigEndian, int64(len(buf))); err != nil { - return fmt.Errorf("write message size: %s", err) + return err } // Write the value. if _, err := w.Write(buf); err != nil { - return fmt.Errorf("write message value: %s", err) + return err } return nil } diff --git a/coordinator/shard_writer.go b/coordinator/shard_writer.go index 8ca2bad..410bbf4 100644 --- a/coordinator/shard_writer.go +++ b/coordinator/shard_writer.go @@ -2,11 +2,11 @@ package coordinator import ( "fmt" - "net" "time" "github.com/influxdata/influxdb/models" "github.com/influxdata/influxdb/services/meta" + "go.uber.org/zap" ) const ( @@ -54,13 +54,15 @@ const ( executeTaskManagerRequestMessage executeTaskManagerResponseMessage + + testRequestMessage // one way message ) // ShardWriter writes a set of points to a shard. type ShardWriter struct { - pool *clientPool - timeout time.Duration - maxConnections int + pool *ClientPool + timeout time.Duration + logger *zap.Logger MetaClient interface { DataNode(id uint64) (ni *meta.NodeInfo, err error) @@ -69,28 +71,25 @@ type ShardWriter struct { } // NewShardWriter returns a new instance of ShardWriter. -func NewShardWriter(timeout time.Duration, maxConnections int) *ShardWriter { +func NewShardWriter(timeout time.Duration, pool *ClientPool) *ShardWriter { return &ShardWriter{ - pool: newClientPool(), - timeout: timeout, - maxConnections: maxConnections, + pool: pool, + timeout: timeout, + logger: zap.NewNop(), } } +func (w *ShardWriter) WithLogger(logger *zap.Logger) { + w.logger = logger.With(zap.String("service", "ShardWriter")) +} + // WriteShard writes time series points to a shard func (w *ShardWriter) WriteShard(shardID, ownerID uint64, points []models.Point) error { - c, err := w.dial(ownerID) + conn, err := getConnWithRetry(w.pool, ownerID, w.logger) if err != nil { return err } - - conn, ok := c.(*pooledConn) - if !ok { - panic("wrong connection type") - } - defer func(conn net.Conn) { - conn.Close() // return to pool - }(conn) + defer conn.Close() // Determine the location of this shard and whether it still exists db, rp, sgi := w.MetaClient.ShardOwner(shardID) @@ -128,6 +127,7 @@ func (w *ShardWriter) WriteShard(shardID, ownerID uint64, points []models.Point) conn.MarkUnusable() return err } + conn.SetDeadline(time.Time{}) // Unmarshal response. var response WriteShardResponse @@ -142,22 +142,6 @@ func (w *ShardWriter) WriteShard(shardID, ownerID uint64, points []models.Point) return nil } -func (w *ShardWriter) dial(nodeID uint64) (net.Conn, error) { - // If we don't have a connection pool for that addr yet, create one - _, ok := w.pool.getPool(nodeID) - if !ok { - factory := &connFactory{nodeID: nodeID, clientPool: w.pool, timeout: w.timeout} - factory.metaClient = w.MetaClient - - p, err := NewBoundedPool(1, w.maxConnections, w.timeout, factory.dial) - if err != nil { - return nil, err - } - w.pool.setPool(nodeID, p) - } - return w.pool.conn(nodeID) -} - // Close closes ShardWriter's pool func (w *ShardWriter) Close() error { if w.pool == nil { @@ -167,52 +151,3 @@ func (w *ShardWriter) Close() error { w.pool = nil return nil } - -const ( - maxConnections = 500 - maxRetries = 3 -) - -var errMaxConnectionsExceeded = fmt.Errorf("can not exceed max connections of %d", maxConnections) - -type connFactory struct { - nodeID uint64 - timeout time.Duration - - clientPool interface { - size() int - } - - metaClient interface { - DataNode(id uint64) (ni *meta.NodeInfo, err error) - } -} - -func (c *connFactory) dial() (net.Conn, error) { - if c.clientPool.size() > maxConnections { - return nil, errMaxConnectionsExceeded - } - - ni, err := c.metaClient.DataNode(c.nodeID) - if err != nil { - return nil, err - } - - if ni == nil { - return nil, fmt.Errorf("node %d does not exist", c.nodeID) - } - - conn, err := net.DialTimeout("tcp", ni.TCPHost, c.timeout) - if err != nil { - return nil, err - } - - // Write a marker byte for cluster messages. - _, err = conn.Write([]byte{MuxHeader}) - if err != nil { - conn.Close() - return nil, err - } - - return conn, nil -} diff --git a/go.mod b/go.mod index 86cda32..55dd242 100644 --- a/go.mod +++ b/go.mod @@ -25,7 +25,6 @@ require ( github.com/klauspost/pgzip v1.2.1 // indirect github.com/pkg/errors v0.8.1 github.com/pkg/profile v1.2.1 // indirect - github.com/spf13/cobra v0.0.3 github.com/stretchr/testify v1.5.1 // test github.com/urfave/cli/v2 v2.2.0 github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca // indirect diff --git a/go.sum b/go.sum index 554912c..88e341e 100644 --- a/go.sum +++ b/go.sum @@ -78,6 +78,8 @@ github.com/coreos/bbolt v1.3.1-coreos.6 h1:uTXKg9gY70s9jMAKdfljFQcuh4e/BXOM+V+d0 github.com/coreos/bbolt v1.3.1-coreos.6/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.12+incompatible h1:pAWNwdf7QiT1zfaWyqCtNZQWCLByQyA3JrSQyuYAqnQ= github.com/coreos/etcd v3.3.12+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -326,6 +328,8 @@ github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a h1:9a8MnZMP0X2nLJdBg+pBmGgkJlSaKC2KaQmTCk1XDtE= github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/go-glob v0.0.0-20170128012129-256dc444b735/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc= github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= @@ -334,6 +338,8 @@ github.com/segmentio/kafka-go v0.1.0 h1:IXCHG+sXPNiIR5pC/vTEItZduPKu4cnpr85Ygxpx github.com/segmentio/kafka-go v0.1.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfPOCvTvk+EJo= github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= +github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= diff --git a/services/controller/service.go b/services/controller/service.go index 53aabb5..0b50aa4 100644 --- a/services/controller/service.go +++ b/services/controller/service.go @@ -3,6 +3,7 @@ package controller import ( "encoding/json" + "errors" "fmt" "io" "net" @@ -21,7 +22,8 @@ import ( const ( // MuxHeader is the header byte used for the TCP muxer. - MuxHeader = 4 + MuxHeader = 4 + MILLISECOND = 1e6 ) type Service struct { @@ -35,6 +37,7 @@ type Service struct { DataNodeByTCPHost(addr string) (*meta.NodeInfo, error) RemoveShardOwner(shardID, nodeID uint64) error DataNodes() ([]meta.NodeInfo, error) + RetentionPolicy(database, name string) (*meta.RetentionPolicyInfo, error) ShardOwner(shardID uint64) (database, policy string, sgi *meta.ShardGroupInfo) AddShardOwner(shardID, nodeID uint64) error @@ -145,20 +148,20 @@ func (s *Service) handleConn(conn net.Conn) error { case RequestShowDataNodes: nodes, err := s.handleShowDataNodes() s.showDataNodesResponse(conn, nodes, err) + case RequestShards: + groupInfo, err := s.handleShards(conn) + s.shardsResponse(conn, groupInfo, err) + case RequestShard: + db, rp, info, err := s.handleShard(conn) + s.shardResponse(conn, db, rp, info, err) } return nil } func (s *Service) handleTruncateShard(conn net.Conn) error { - buf, err := coordinator.ReadLV(conn) - if err != nil { - s.Logger.Error("unable to read length-value", zap.Error(err)) - return err - } - var req TruncateShardRequest - if err := json.Unmarshal(buf, &req); err != nil { + if err := s.readRequest(conn, &req); err != nil { return err } @@ -194,14 +197,8 @@ func (s *Service) truncateShardResponse(w io.Writer, e error) { } func (s *Service) handleCopyShard(conn net.Conn) error { - buf, err := coordinator.ReadLV(conn) - if err != nil { - s.Logger.Error("unable to read length-value", zap.Error(err)) - return err - } - var req CopyShardRequest - if err := json.Unmarshal(buf, &req); err != nil { + if err := s.readRequest(conn, &req); err != nil { return err } @@ -243,6 +240,33 @@ func toCopyTask(t *migrate.Task) CopyShardTask { return task } +func (s *Service) writeResponse(w io.Writer, t ResponseType, obj interface{}) { + // Marshal response to binary. + buf, err := json.Marshal(obj) + if err != nil { + s.Logger.Error(fmt.Sprint("Marshal fail: type=", t, ", error=", err.Error())) + return + } + + // Write to connection. + if err := coordinator.WriteTLV(w, byte(t), buf); err != nil { + s.Logger.Error(fmt.Sprint("WriteTLV fail: type=", t, ", error=", err.Error())) + } +} + +func (s *Service) readRequest(conn net.Conn, obj interface{}) error { + buf, err := coordinator.ReadLV(conn) + if err != nil { + s.Logger.Error("unable to read length-value", zap.Error(err)) + return err + } + + if err := json.Unmarshal(buf, obj); err != nil { + return err + } + return nil +} + func (s *Service) handleCopyShardStatus(conn net.Conn) []CopyShardTask { tasks := s.migrateManager.Tasks() result := make([]CopyShardTask, len(tasks)) @@ -258,29 +282,12 @@ func (s *Service) copyShardStatusResponse(w io.Writer, tasks []CopyShardTask) { resp.Code = 0 resp.Msg = "ok" resp.Tasks = tasks - - // Marshal response to binary. - buf, err := json.Marshal(&resp) - if err != nil { - s.Logger.Error("error marshalling show copy shard response", zap.Error(err)) - return - } - - // Write to connection. - if err := coordinator.WriteTLV(w, byte(ResponseCopyShardStatus), buf); err != nil { - s.Logger.Error("show copy shard WriteTLV fail", zap.Error(err)) - } + s.writeResponse(w, ResponseCopyShardStatus, &resp) } func (s *Service) handleKillCopyShard(conn net.Conn) error { - buf, err := coordinator.ReadLV(conn) - if err != nil { - s.Logger.Error("unable to read length-value", zap.Error(err)) - return err - } - var req KillCopyShardRequest - if err := json.Unmarshal(buf, &req); err != nil { + if err := s.readRequest(conn, &req); err != nil { return err } @@ -291,38 +298,14 @@ func (s *Service) handleKillCopyShard(conn net.Conn) error { func (s *Service) killCopyShardResponse(w io.Writer, e error) { // Build response. var resp KillCopyShardResponse - if e != nil { - resp.Code = 1 - resp.Msg = e.Error() - } else { - resp.Code = 0 - resp.Msg = "ok" - } - - // Marshal response to binary. - buf, err := json.Marshal(&resp) - if err != nil { - s.Logger.Error("error marshalling show copy shard response", zap.Error(err)) - return - } - - // Write to connection. - if err := coordinator.WriteTLV(w, byte(ResponseKillCopyShard), buf); err != nil { - s.Logger.Error("kill copy shard WriteTLV fail", zap.Error(err)) - } + setError(&resp.CommonResp, e) + s.writeResponse(w, ResponseKillCopyShard, &resp) } func (s *Service) handleRemoveShard(conn net.Conn) error { s.Logger.Info("handleRemoveShard") - buf, err := coordinator.ReadLV(conn) - if err != nil { - s.Logger.Error("unable to read length-value", zap.Error(err)) - return err - } - var req RemoveShardRequest - if err := json.Unmarshal(buf, &req); err != nil { - s.Logger.Error("unmarshal fail.", zap.Error(err)) + if err := s.readRequest(conn, &req); err != nil { return err } @@ -353,36 +336,13 @@ func (s *Service) handleRemoveShard(conn net.Conn) error { func (s *Service) removeShardResponse(w io.Writer, e error) { // Build response. var resp RemoveShardResponse - if e != nil { - resp.Code = 1 - resp.Msg = e.Error() - } else { - resp.Code = 0 - resp.Msg = "ok" - } - - // Marshal response to binary. - buf, err := json.Marshal(&resp) - if err != nil { - s.Logger.Error("error marshalling remove shard response", zap.Error(err)) - return - } - - // Write to connection. - if err := coordinator.WriteTLV(w, byte(ResponseRemoveShard), buf); err != nil { - s.Logger.Error("remove shard WriteTLV fail", zap.Error(err)) - } + setError(&resp.CommonResp, e) + s.writeResponse(w, ResponseRemoveShard, &resp) } func (s *Service) handleRemoveDataNode(conn net.Conn) error { - buf, err := coordinator.ReadLV(conn) - if err != nil { - s.Logger.Error("unable to read length-value", zap.Error(err)) - return err - } - var req RemoveDataNodeRequest - if err := json.Unmarshal(buf, &req); err != nil { + if err := s.readRequest(conn, &req); err != nil { return err } @@ -399,25 +359,88 @@ func (s *Service) handleRemoveDataNode(conn net.Conn) error { func (s *Service) removeDataNodeResponse(w io.Writer, e error) { // Build response. var resp RemoveDataNodeResponse - if e != nil { - resp.Code = 1 - resp.Msg = e.Error() - } else { - resp.Code = 0 - resp.Msg = "ok" + setError(&resp.CommonResp, e) + s.writeResponse(w, ResponseRemoveDataNode, &resp) +} + +func (s *Service) handleShards(conn net.Conn) (*meta.RetentionPolicyInfo, error) { + var req GetShardsRequest + if err := s.readRequest(conn, &req); err != nil { + return nil, err } + if req.Database == "" || req.RetentionPolicy == "" { + return nil, errors.New("Both database and retention policy should be specified") + } + return s.MetaClient.RetentionPolicy(req.Database, req.RetentionPolicy) +} - // Marshal response to binary. - buf, err := json.Marshal(&resp) - if err != nil { - s.Logger.Error("error marshalling remove data node response", zap.Error(err)) - return +func fromMetaOwners(owners []meta.ShardOwner) []uint64 { + nodes := make([]uint64, len(owners)) + for i, owner := range owners { + nodes[i] = owner.NodeID } + return nodes +} - // Write to connection. - if err := coordinator.WriteTLV(w, byte(ResponseRemoveDataNode), buf); err != nil { - s.Logger.Error("remove data node WriteTLV fail", zap.Error(err)) +func fromMetaShards(shards []meta.ShardInfo) []Shard { + list := make([]Shard, len(shards)) + for i, shard := range shards { + list[i].ID = shard.ID + list[i].Nodes = fromMetaOwners(shard.Owners) } + return list +} + +func (s *Service) shardsResponse(w io.Writer, rp *meta.RetentionPolicyInfo, e error) { + // Build response. + var resp ShardsResponse + setError(&resp.CommonResp, e) + if rp != nil { + resp.Rp = rp.Name + resp.Duration = int64(rp.Duration / time.Millisecond) + resp.GroupDuration = int64(rp.ShardGroupDuration / time.Millisecond) + resp.Replica = rp.ReplicaN + resp.Groups = make([]ShardGroup, len(rp.ShardGroups)) + for i, g := range rp.ShardGroups { + resp.Groups[i].ID = g.ID + resp.Groups[i].StartTime = g.StartTime.UnixNano() / MILLISECOND + resp.Groups[i].EndTime = g.EndTime.UnixNano() / MILLISECOND + resp.Groups[i].DeletedAt = g.DeletedAt.UnixNano() / MILLISECOND + resp.Groups[i].TruncatedAt = g.TruncatedAt.UnixNano() / MILLISECOND + resp.Groups[i].Shards = fromMetaShards(g.Shards) + } + } + + s.writeResponse(w, ResponseShards, &resp) +} + +func (s *Service) handleShard(conn net.Conn) (string, string, *meta.ShardInfo, error) { + var req GetShardRequest + if err := s.readRequest(conn, &req); err != nil { + return "", "", nil, err + } + if req.ShardID < 1 { + return "", "", nil, errors.New("ShardID should be specified") + } + db, rp, groupInfo := s.MetaClient.ShardOwner(req.ShardID) + for _, shard := range groupInfo.Shards { + if shard.ID == req.ShardID { + return db, rp, &shard, nil + } + } + return "", "", nil, errors.New("Specified shard could not be found") +} + +func (s *Service) shardResponse(w io.Writer, db, rp string, shard *meta.ShardInfo, e error) { + var resp ShardResponse + setError(&resp.CommonResp, e) + if e != nil { + resp.ID = shard.ID + resp.DB = db + resp.Rp = rp + resp.Nodes = fromMetaOwners(shard.Owners) + } + s.writeResponse(w, ResponseShard, &resp) } func (s *Service) handleShowDataNodes() ([]DataNode, error) { @@ -436,26 +459,19 @@ func (s *Service) handleShowDataNodes() ([]DataNode, error) { func (s *Service) showDataNodesResponse(w io.Writer, nodes []DataNode, e error) { // Build response. var resp ShowDataNodesResponse - if e != nil { + setError(&resp.CommonResp, e) + resp.DataNodes = nodes + s.writeResponse(w, ResponseShowDataNodes, &resp) +} + +func setError(resp *CommonResp, err error) { + if err != nil { resp.Code = 1 - resp.Msg = e.Error() + resp.Msg = err.Error() } else { resp.Code = 0 resp.Msg = "ok" } - resp.DataNodes = nodes - - // Marshal response to binary. - buf, err := json.Marshal(&resp) - if err != nil { - s.Logger.Error("error marshalling show data nodes response", zap.Error(err)) - return - } - - // Write to connection. - if err := coordinator.WriteTLV(w, byte(ResponseShowDataNodes), buf); err != nil { - s.Logger.Error("show data nodes WriteTLV fail", zap.Error(err)) - } } type CommonResp struct { @@ -505,6 +521,14 @@ type KillCopyShardResponse struct { CommonResp } +type GetShardsRequest struct { + Database, RetentionPolicy string +} + +type GetShardRequest struct { + ShardID uint64 +} + type RemoveShardRequest struct { DataNodeAddr string `json:"data_node_addr"` ShardID uint64 `json:"shard_id"` @@ -528,6 +552,37 @@ type DataNode struct { HttpAddr string `json:"http_addr"` } +type ShardGroup struct { + ID uint64 `json:"id"` + StartTime int64 `json:"begin"` + EndTime int64 `json:"end"` + DeletedAt int64 `json:"deleted_at"` + TruncatedAt int64 `json:"truncated_at"` + Shards []Shard `json:"shards"` +} + +type Shard struct { + ID uint64 `json:"id"` + Nodes []uint64 `json:"nodes"` +} + +type ShardsResponse struct { + CommonResp + Rp string `json:"rp"` + Replica int `json:"replica"` + Duration int64 `json:"duration"` + GroupDuration int64 `json:"group_duration"` + Groups []ShardGroup `json:"groups"` +} + +type ShardResponse struct { + CommonResp + ID uint64 `json:"id"` + DB string `json:"db"` + Rp string `json:"rp"` + Nodes []uint64 `json:"nodes"` +} + type ShowDataNodesResponse struct { CommonResp DataNodes []DataNode `json:"data_nodes"` @@ -538,23 +593,29 @@ type RequestType byte const ( // RequestTruncateShard represents a request for truncating shard. - RequestTruncateShard RequestType = 1 - RequestCopyShard = 2 - RequestCopyShardStatus = 3 - RequestKillCopyShard = 4 - RequestRemoveShard = 5 - RequestRemoveDataNode = 6 - RequestShowDataNodes = 7 + _ RequestType = iota + RequestTruncateShard + RequestCopyShard + RequestCopyShardStatus + RequestKillCopyShard + RequestRemoveShard + RequestRemoveDataNode + RequestShowDataNodes + RequestShards + RequestShard ) type ResponseType byte const ( - ResponseTruncateShard ResponseType = 1 - ResponseCopyShard = 2 - ResponseCopyShardStatus = 3 - ResponseKillCopyShard = 4 - ResponseRemoveShard = 5 - ResponseRemoveDataNode = 6 - ResponseShowDataNodes = 7 + _ ResponseType = iota + ResponseTruncateShard + ResponseCopyShard + ResponseCopyShardStatus + ResponseKillCopyShard + ResponseRemoveShard + ResponseRemoveDataNode + ResponseShowDataNodes + ResponseShards + ResponseShard ) diff --git a/services/hh/node_processor.go b/services/hh/node_processor.go index 23666a9..d2d6696 100644 --- a/services/hh/node_processor.go +++ b/services/hh/node_processor.go @@ -13,6 +13,7 @@ import ( "strings" "sync/atomic" + "github.com/angopher/chronus/services/meta" "github.com/influxdata/influxdb/models" "golang.org/x/time/rate" ) @@ -298,7 +299,7 @@ func (n *NodeProcessor) Tail() string { // Active returns whether this node processor is for a currently active node. func (n *NodeProcessor) Active() (bool, error) { nio, err := n.meta.DataNode(n.nodeID) - if err != nil { + if err != nil && err != meta.ErrNodeNotFound { n.Logger.Printf("failed to determine if node %d is active: %s", n.nodeID, err.Error()) return false, err } diff --git a/services/hh/service.go b/services/hh/service.go index 211df93..120f779 100644 --- a/services/hh/service.go +++ b/services/hh/service.go @@ -3,17 +3,18 @@ package hh // import "github.com/influxdata/influxdb/services/hh" import ( "fmt" "io/ioutil" - "log" "os" "path/filepath" "strconv" "sync" "time" + "sync/atomic" + "github.com/influxdata/influxdb/models" - "github.com/influxdata/influxdb/services/meta" "github.com/influxdata/influxdb/monitor/diagnostics" - "sync/atomic" + "github.com/influxdata/influxdb/services/meta" + "go.uber.org/zap" ) // ErrHintedHandoffDisabled is returned when attempting to use a @@ -33,7 +34,7 @@ type Service struct { processors map[uint64]*NodeProcessor - Logger *log.Logger + Logger *zap.SugaredLogger cfg Config shardWriter shardWriter @@ -64,13 +65,17 @@ func NewService(c Config, w shardWriter, m metaClient) *Service { cfg: c, closing: make(chan struct{}), processors: make(map[uint64]*NodeProcessor), - Logger: log.New(os.Stderr, "[handoff] ", log.LstdFlags), + Logger: zap.NewNop().Sugar(), shardWriter: w, MetaClient: m, stats: &HHStatistics{}, } } +func (s *Service) WithLogger(log *zap.Logger) { + s.Logger = log.With(zap.String("service", "handoff")).Sugar() +} + // Open opens the hinted handoff service. func (s *Service) Open() error { s.mu.Lock() @@ -79,7 +84,7 @@ func (s *Service) Open() error { // Allow Open to proceed, but don't do anything. return nil } - s.Logger.Printf("Starting hinted handoff service") + s.Logger.Info("Starting hinted handoff service") s.closing = make(chan struct{}) // Register diagnostics if a Monitor service is available. @@ -88,7 +93,7 @@ func (s *Service) Open() error { } // Create the root directory if it doesn't already exist. - s.Logger.Printf("Using data dir: %v", s.cfg.Dir) + s.Logger.Infof("Using data dir: %v", s.cfg.Dir) if err := os.MkdirAll(s.cfg.Dir, 0700); err != nil { return fmt.Errorf("mkdir all: %s", err) } @@ -121,7 +126,7 @@ func (s *Service) Open() error { // Close closes the hinted handoff service. func (s *Service) Close() error { - s.Logger.Println("shutting down hh service") + s.Logger.Info("shutting down hh service") s.mu.Lock() defer s.mu.Unlock() @@ -166,11 +171,6 @@ func (s *Service) Statistics(tags map[string]string) []models.Statistic { return statistics } -// SetLogger sets the internal logger to the logger passed in. -func (s *Service) SetLogger(l *log.Logger) { - s.Logger = l -} - // WriteShard queues the points write for shardID to node ownerID to handoff queue func (s *Service) WriteShard(shardID, ownerID uint64, points []models.Point) error { if !s.cfg.Enabled { @@ -253,13 +253,13 @@ func (s *Service) purgeInactiveProcessors() { for k, v := range s.processors { lm, err := v.LastModified() if err != nil { - s.Logger.Printf("failed to determine LastModified for processor %d: %s", k, err.Error()) + s.Logger.Warnf("failed to determine LastModified for processor %d: %s", k, err.Error()) continue } active, err := v.Active() if err != nil { - s.Logger.Printf("failed to determine if node %d is active: %s", k, err.Error()) + s.Logger.Warnf("failed to determine if node %d is active: %s", k, err.Error()) continue } if active { @@ -273,11 +273,11 @@ func (s *Service) purgeInactiveProcessors() { } if err := v.Close(); err != nil { - s.Logger.Printf("failed to close node processor %d: %s", k, err.Error()) + s.Logger.Warnf("failed to close node processor %d: %s", k, err.Error()) continue } if err := v.Purge(); err != nil { - s.Logger.Printf("failed to purge node processor %d: %s", k, err.Error()) + s.Logger.Warnf("failed to purge node processor %d: %s", k, err.Error()) continue } delete(s.processors, k) diff --git a/services/meta/client.go b/services/meta/client.go index 8224f47..dea46ec 100644 --- a/services/meta/client.go +++ b/services/meta/client.go @@ -130,6 +130,7 @@ func (c *Client) data() *Data { } // DataNode returns a node by id. +// If specified node doesn't exist a meta.ErrNodeNotFound error will be returned. func (c *Client) DataNode(id uint64) (*meta.NodeInfo, error) { c.mu.Lock() defer c.mu.Unlock() diff --git a/x/echo_server.go b/x/echo_server.go new file mode 100644 index 0000000..d984659 --- /dev/null +++ b/x/echo_server.go @@ -0,0 +1,112 @@ +package x + +import ( + "io" + "net" + "sync" + "time" +) + +type EchoServer struct { + network string + address string + listener net.Listener + closing chan int + once sync.Once + wg sync.WaitGroup +} + +func NewEchoServer(network, addr string) *EchoServer { + return &EchoServer{ + network: network, + address: addr, + closing: make(chan int), + } +} + +func (s *EchoServer) Start() error { + listener, err := net.Listen(s.network, s.address) + if err != nil { + return err + } + s.listener = listener + go s.loop() + return nil +} + +func (s *EchoServer) Close() { + s.once.Do(func() { + close(s.closing) + s.listener.Close() + s.wg.Wait() + }) +} + +func (s *EchoServer) loop() { + defer s.Close() + + s.wg.Add(1) + defer s.wg.Done() +LOOP: + for { + select { + case <-s.closing: + break LOOP + default: + conn, err := s.listener.Accept() + if err != nil { + break LOOP + } + go s.connLoop(conn) + } + } +} + +func (s *EchoServer) connLoop(conn net.Conn) { + s.wg.Add(1) + defer func() { + conn.Close() + s.wg.Done() + }() + var ( + buf = make([]byte, 1024) + n int + err error + ) +LOOP: + for { + select { + case <-s.closing: + break LOOP + default: + conn.SetReadDeadline(time.Now().Add(100 * time.Millisecond)) + n, err = conn.Read(buf) + conn.SetDeadline(time.Time{}) + } + if err == io.EOF { + break LOOP + } + if err != nil { + continue + } + if n < 1 { + continue + } + if write(conn, buf[:n]) != nil { + break LOOP + } + } +} + +func write(conn net.Conn, data []byte) error { + size := len(data) + pos := 0 + for pos < size { + n, err := conn.Write(data[pos:size]) + if err != nil { + return err + } + pos += n + } + return nil +} diff --git a/x/math.go b/x/math.go new file mode 100644 index 0000000..5d0fd57 --- /dev/null +++ b/x/math.go @@ -0,0 +1,21 @@ +package x + +func Max(args ...int) int { + result := args[0] + for i := 1; i < len(args); i++ { + if args[i] > result { + result = args[i] + } + } + return result +} + +func Min(args ...int) int { + result := args[0] + for i := 1; i < len(args); i++ { + if args[i] < result { + result = args[i] + } + } + return result +} diff --git a/x/pool.go b/x/pool.go new file mode 100644 index 0000000..4068e6a --- /dev/null +++ b/x/pool.go @@ -0,0 +1,281 @@ +package x + +import ( + "errors" + "fmt" + "net" + "sync" + "sync/atomic" + "time" +) + +var ( + // ErrClosed is the error resulting if the pool is closed via pool.Close(). + ErrClosed = errors.New("pool is closed") +) + +type ConnPool interface { + // Get returns a new connection from the pool. Closing the connections puts + // it back to the Pool. Closing it when the pool is destroyed or full will + // be counted as an error. + Get() (PooledConn, error) + + // Close closes the pool and all its connections. After Close() the pool is + // no longer usable. + Close() + + // Len returns current idled connection count in pool + Len() int + // Total returns total connection count managed by pool + Total() int +} + +type PooledConn interface { + net.Conn + MarkUnusable() +} + +// boundedPool implements the Pool interface based on buffered channels. +type boundedPool struct { + // storage for our net.Conn connections + mu sync.RWMutex + // Using a channel to hold idled connections in pool is not a good idea + // for closing idled ones later. We may change it into an array to make + // more senses. + conns chan *pooledConn + + closer chan int + wg sync.WaitGroup + + waitTimeout time.Duration + idleTimeout time.Duration + initialCnt int + total int32 + // net.Conn generator + factory Factory +} + +// Factory is a function to create new connections. +type Factory func() (net.Conn, error) + +// NewBoundedPool returns a new pool based on buffered channels with an initial +// capacity, maximum capacity and waitTimeout to wait for a connection from the pool. +// Factory is used when initial capacity is +// greater than zero to fill the pool. A zero initialCap doesn't fill the Pool +// until a new Get() is called. During a Get(), If there is no new connection +// available in the pool and total connections is less than the max, a new connection +// will be created via the Factory() method. Othewise, the call will block until +// a connection is available or the waitTimeout is reached. +func NewBoundedPool(initialCnt, maxCap int, idleTimeout time.Duration, waitTimeout time.Duration, factory Factory) (ConnPool, error) { + if initialCnt < 0 || maxCap <= 0 || initialCnt > maxCap { + return nil, errors.New("invalid capacity settings") + } + + c := &boundedPool{ + closer: make(chan int), + conns: make(chan *pooledConn, maxCap), + factory: factory, + initialCnt: initialCnt, + waitTimeout: waitTimeout, + idleTimeout: idleTimeout, + } + + // create initial connections, if something goes wrong, + // just close the pool error out. + for i := 0; i < initialCnt; i++ { + conn, err := factory() + if err != nil { + c.Close() + return nil, fmt.Errorf("factory is not able to fill the pool: %s", err) + } + c.conns <- c.wrapConn(conn) + atomic.AddInt32(&c.total, 1) + } + + go c.idleChecker() + + return c, nil +} + +func (c *boundedPool) checkOnce() { + now := time.Now() + for int(atomic.LoadInt32(&c.total)) > c.initialCnt && len(c.conns) > 0 { + select { + case conn := <-c.conns: + if now.Sub(conn.lastUse) <= c.idleTimeout { + select { + case c.conns <- conn: + default: + // can't be returned, drop + conn.MarkUnusable() + conn.Close() + } + return + } + // drop it + conn.MarkUnusable() + conn.Close() + default: + return + } + } + +} + +// idleChecker checks the connections in pool and closes those haven't been used for idleTimeout and more than initialCnt. +func (c *boundedPool) idleChecker() { + c.wg.Add(1) + defer c.wg.Done() + ticker := time.NewTicker(60 * time.Second) + defer ticker.Stop() +LOOP: + for { + select { + case <-ticker.C: + c.checkOnce() + case <-c.closer: + break LOOP + } + } +} + +// Get implements the Pool interfaces Get() method. If there is no new +// connection available in the pool, a new connection will be created via the +// Factory() method. +func (c *boundedPool) Get() (PooledConn, error) { + conns := c.conns + if conns == nil { + return nil, ErrClosed + } + + // Try and grab a connection from the pool + select { + case conn := <-conns: + if conn == nil { + return nil, ErrClosed + } + return conn, nil + default: + // Could not get connection, can we create a new one? + total := atomic.LoadInt32(&c.total) + capacity := int32(cap(conns)) + if total < capacity && atomic.AddInt32(&c.total, 1) <= capacity { + conn, err := c.factory() + if err != nil { + atomic.AddInt32(&c.total, -1) + return nil, err + } + + return c.wrapConn(conn), nil + } + } + + // The pool was empty and we couldn't create a new one to + // retry until one is free or we timeout + select { + case conn := <-conns: + if conn == nil { + return nil, ErrClosed + } + return conn, nil + case <-time.After(c.waitTimeout): + return nil, fmt.Errorf("timed out waiting for free connection") + } + +} + +// put puts the connection back to the pool. If the pool is full or closed, +// conn is simply closed. A nil conn will be rejected. +func (c *boundedPool) put(conn *pooledConn) error { + if conn == nil { + return errors.New("connection is nil. rejecting") + } + + c.mu.RLock() + defer c.mu.RUnlock() + + if c.conns == nil { + // pool is closed, close passed connection + goto DROP + } + + // put the resource back into the pool. If the pool is full, drop it + select { + case c.conns <- conn: + return nil + default: + // pool is full, close passed connection + goto DROP + } + +DROP: + conn.MarkUnusable() + return conn.Close() +} + +func (c *boundedPool) Close() { + close(c.closer) + c.wg.Wait() + c.mu.Lock() + conns := c.conns + c.conns = nil + c.factory = nil + c.mu.Unlock() + + if conns == nil { + return + } + + close(conns) + for conn := range conns { + conn.Close() + } +} + +// Len returns current idled connection count in pool +func (c *boundedPool) Len() int { + return len(c.conns) +} + +// Total returns total connection count managed by pool +func (c *boundedPool) Total() int { + return int(atomic.LoadInt32(&c.total)) +} + +// newConn wraps a standard net.Conn to a poolConn net.Conn. +func (c *boundedPool) wrapConn(conn net.Conn) *pooledConn { + p := &pooledConn{c: c} + p.Conn = conn + p.lastUse = time.Now() + return p +} + +// pooledConn is a wrapper around net.Conn to modify the the behavior of +// net.Conn's Close() method. +type pooledConn struct { + net.Conn + c *boundedPool + lastUse time.Time + unusable bool +} + +// Close() puts the given connects back to the pool instead of closing it. +func (p *pooledConn) Close() error { + if p.unusable { + if p.Conn != nil { + return p.Conn.Close() + } + return nil + } + p.lastUse = time.Now() + return p.c.put(p) +} + +// MarkUnusable() marks the connection not usable any more, to let the pool close it instead of returning it to pool. +func (p *pooledConn) MarkUnusable() { + if p.unusable { + return + } + p.unusable = true + atomic.AddInt32(&p.c.total, -1) +} diff --git a/x/pool_test.go b/x/pool_test.go new file mode 100644 index 0000000..cb0112e --- /dev/null +++ b/x/pool_test.go @@ -0,0 +1,76 @@ +package x + +import ( + "net" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestPoolCapacity(t *testing.T) { + echoServer := NewEchoServer("tcp", "127.0.0.1:12345") + err := echoServer.Start() + assert.Nil(t, err) + cnt := 0 + p, err := NewBoundedPool(1, 3, time.Second, time.Second, func() (net.Conn, error) { + cnt++ + return net.Dial("tcp", "127.0.0.1:12345") + }) + assert.Equal(t, 1, p.Len()) + assert.Equal(t, 1, p.Total()) + assert.Equal(t, 1, cnt) + + conn1, err := p.Get() + (p.(*boundedPool)).checkOnce() + assert.Nil(t, err) + assert.Equal(t, 0, p.Len()) + assert.Equal(t, 1, p.Total()) + assert.Equal(t, 1, cnt) + + conn2, err := p.Get() + (p.(*boundedPool)).checkOnce() + assert.Nil(t, err) + assert.Equal(t, 0, p.Len()) + assert.Equal(t, 2, p.Total()) + assert.Equal(t, 2, cnt) + + conn3, err := p.Get() + (p.(*boundedPool)).checkOnce() + assert.Nil(t, err) + assert.Equal(t, 0, p.Len()) + assert.Equal(t, 3, p.Total()) + assert.Equal(t, 3, cnt) + + _, err = p.Get() + (p.(*boundedPool)).checkOnce() + assert.NotNil(t, err) + assert.Equal(t, 0, p.Len()) + assert.Equal(t, 3, p.Total()) + assert.Equal(t, 3, cnt) + + conn3.Close() + (p.(*boundedPool)).checkOnce() + assert.Equal(t, 1, p.Len()) + assert.Equal(t, 3, p.Total()) + assert.Equal(t, 3, cnt) + + conn2.Close() + conn1.Close() + (p.(*boundedPool)).checkOnce() + assert.Equal(t, 3, p.Len()) + assert.Equal(t, 3, p.Total()) + assert.Equal(t, 3, cnt) + + time.Sleep(time.Second) + // recycle idled connections + (p.(*boundedPool)).checkOnce() + (p.(*boundedPool)).checkOnce() + (p.(*boundedPool)).checkOnce() + (p.(*boundedPool)).checkOnce() + assert.Equal(t, 1, p.Len()) + assert.Equal(t, 1, p.Total()) + + p.Close() + echoServer.Close() +} From 452d319a32a996662b7ab99057549dfb7d7a1d09 Mon Sep 17 00:00:00 2001 From: Jason Joo Date: Thu, 10 Sep 2020 22:08:41 +0800 Subject: [PATCH 09/46] Mainly optimize remote iterators over persistent connections. Changes: - Re-split connection pool into two individual pools for executors and writers - Polish configuration items, related: * coordinator: pool-max-idle-time, pool-min-streams-per-node, pool-max-streams-per-node * handoff: retry-concurrency, retry-rate-limit - Optimize some loggings and migrate from logger to zap engine to make it more unified - Introduce `Termination` property to remote iterator response which will be sent if the iterating finished - Make purging of handoff more reliable - Introduce concurrency control to handoff - Improve safety of connection pool Signed-off-by: Jason Joo --- cmd/influxd/run/server.go | 43 +++++---- coordinator/config.go | 4 + coordinator/points_writer.go | 2 +- coordinator/remote_executor.go | 69 +++++++++++++- coordinator/remote_executor_test.go | 35 +++++++- coordinator/rpc.go | 16 ++-- coordinator/service.go | 22 +++-- services/hh/config.go | 2 +- services/hh/node_processor.go | 135 +++++++++++++++++++++------- services/hh/service.go | 14 ++- x/cyclic_buffer.go | 93 +++++++++++++++++++ x/cyclic_buffer_test.go | 30 +++++++ x/math.go | 30 +++++++ x/math_test.go | 25 ++++++ x/pool.go | 19 +++- 15 files changed, 467 insertions(+), 72 deletions(-) create mode 100644 x/cyclic_buffer.go create mode 100644 x/cyclic_buffer_test.go create mode 100644 x/math_test.go diff --git a/cmd/influxd/run/server.go b/cmd/influxd/run/server.go index bdfa535..a707b43 100644 --- a/cmd/influxd/run/server.go +++ b/cmd/influxd/run/server.go @@ -234,24 +234,22 @@ func NewServer(c *Config, buildInfo *BuildInfo, logger *zap.Logger) (*Server, er // Create the Subscriber service s.Subscriber = subscriber.NewService(c.Subscriber) - clientPool := coordinator.NewClientPool(func(nodeId uint64) (x.ConnPool, error) { - return x.NewBoundedPool( - x.Max(1, x.Min(10, s.config.Coordinator.PoolMaxStreamsPerNode/20)), - s.config.Coordinator.PoolMaxStreamsPerNode, - time.Duration(s.config.Coordinator.PoolMaxIdleTimeout), - time.Duration(s.config.Coordinator.DailTimeout), - coordinator.NewClientConnFactory( - nodeId, - time.Duration(s.config.Coordinator.DailTimeout), - s.ClusterMetaClient, - ).Dial, - ) - }) - // Initialize shard writer s.ShardWriter = coordinator.NewShardWriter( time.Duration(s.config.Coordinator.WriteTimeout), - clientPool, + coordinator.NewClientPool(func(nodeId uint64) (x.ConnPool, error) { + return x.NewBoundedPool( + 1, + 100, + time.Duration(s.config.Coordinator.PoolMaxIdleTimeout), + time.Duration(s.config.Coordinator.DailTimeout), + coordinator.NewClientConnFactory( + nodeId, + time.Duration(s.config.Coordinator.DailTimeout), + s.ClusterMetaClient, + ).Dial, + ) + }), ) s.ShardWriter.WithLogger(s.Logger) @@ -271,7 +269,20 @@ func NewServer(c *Config, buildInfo *BuildInfo, logger *zap.Logger) (*Server, er // Initialize cluster extecutor clusterExecutor := coordinator.NewClusterExecutor( s.Node, s.TSDBStore, - s.ClusterMetaClient, clientPool, + s.ClusterMetaClient, + coordinator.NewClientPool(func(nodeId uint64) (x.ConnPool, error) { + return x.NewBoundedPool( + x.Max(1, s.config.Coordinator.PoolMinStreamsPerNode), + s.config.Coordinator.PoolMaxStreamsPerNode, + time.Duration(s.config.Coordinator.PoolMaxIdleTimeout), + time.Duration(s.config.Coordinator.DailTimeout), + coordinator.NewClientConnFactory( + nodeId, + time.Duration(s.config.Coordinator.DailTimeout), + s.ClusterMetaClient, + ).Dial, + ) + }), s.config.Coordinator, ) clusterExecutor.WithLogger(s.Logger) diff --git a/coordinator/config.go b/coordinator/config.go index e5817d5..1d13771 100644 --- a/coordinator/config.go +++ b/coordinator/config.go @@ -23,6 +23,7 @@ const ( // A value of zero will make the maximum query limit unlimited. DefaultMaxConcurrentQueries = 0 + DefaultPoolMinStreamsPerNode = 5 DefaultPoolMaxStreamsPerNode = 200 // DefaultMaxSelectPointN is the maximum number of points a SELECT can process. @@ -40,6 +41,7 @@ const ( type Config struct { DailTimeout toml.Duration `toml:"dial-timeout"` PoolMaxIdleTimeout toml.Duration `toml:"pool-max-idle-time"` + PoolMinStreamsPerNode int `toml:"pool-min-streams-per-node"` PoolMaxStreamsPerNode int `toml:"pool-max-streams-per-node"` ShardReaderTimeout toml.Duration `toml:"shard-reader-timeout"` ClusterTracing bool `toml:"cluster-tracing"` @@ -59,6 +61,7 @@ func NewConfig() Config { return Config{ DailTimeout: toml.Duration(DefaultDialTimeout), PoolMaxIdleTimeout: toml.Duration(DefaultPoolMaxIdleTimeout), + PoolMinStreamsPerNode: DefaultPoolMinStreamsPerNode, PoolMaxStreamsPerNode: DefaultPoolMaxStreamsPerNode, ShardReaderTimeout: toml.Duration(DefaultShardReaderTimeout), ClusterTracing: false, @@ -77,6 +80,7 @@ func (c Config) Diagnostics() (*diagnostics.Diagnostics, error) { return diagnostics.RowFromMap(map[string]interface{}{ "dail-timeout": c.DailTimeout, "pool-max-idle-time": c.PoolMaxIdleTimeout, + "pool-min-streams-per-node": c.PoolMinStreamsPerNode, "pool-max-streams-per-node": c.PoolMaxStreamsPerNode, "shard-reader-timeout": c.ShardReaderTimeout, "cluster-tracing": c.ClusterTracing, diff --git a/coordinator/points_writer.go b/coordinator/points_writer.go index a7ac455..81a5cbf 100644 --- a/coordinator/points_writer.go +++ b/coordinator/points_writer.go @@ -406,7 +406,7 @@ func (w *PointsWriter) writeToShard(shard *meta.ShardInfo, database, retentionPo atomic.AddInt64(&w.stats.PointWriteReqRemote, int64(len(points))) err := w.ShardWriter.WriteShard(shardID, owner.NodeID, points) if err != nil && IsRetryable(err) { - w.Logger.Warn("ShardWriter.WriteShard fail", zap.Error(err)) + w.Logger.Warn(fmt.Sprint("ShardWriter.WriteShard fail to ", owner.NodeID), zap.Error(err)) // The remote write failed so queue it via hinted handoff atomic.AddInt64(&w.stats.WritePointReqHH, int64(len(points))) hherr := w.HintedHandoff.WriteShard(shardID, owner.NodeID, points) diff --git a/coordinator/remote_executor.go b/coordinator/remote_executor.go index ddbeb6d..fcf4847 100644 --- a/coordinator/remote_executor.go +++ b/coordinator/remote_executor.go @@ -4,6 +4,7 @@ import ( "context" "encoding/binary" "errors" + "io" "net" "time" @@ -73,11 +74,14 @@ func getConnWithRetry(pool *ClientPool, nodeId uint64, logger *zap.Logger) (x.Po } // do write test if err = writeTestPacket(conn); err != nil { + conn.MarkUnusable() + conn.Close() logger.Warn("Failed to get connection from pool", zap.Error(err)) continue } + return conn, nil } - return conn, err + return nil, err } func (me *remoteNodeExecutor) CreateIterator(nodeId uint64, ctx context.Context, m *influxql.Measurement, opt query.IteratorOptions, shardIds []uint64) (query.Iterator, error) { @@ -125,7 +129,7 @@ func (me *remoteNodeExecutor) CreateIterator(nodeId uint64, ctx context.Context, stats := query.IteratorStats{SeriesN: resp.SeriesN} //conn.Close will be invoked when iterator.Close - itr := query.NewReaderIterator(ctx, conn, resp.DataType, stats) + itr := query.NewReaderIterator(ctx, newIteratorReader(conn, resp.Termination), resp.DataType, stats) return itr, nil } @@ -498,3 +502,64 @@ func (me *remoteNodeExecutor) DeleteSeries(nodeId uint64, database string, sourc return nil } + +type iteratorReader struct { + io.ReadCloser + terminator []byte + buf []byte + reader io.ReadCloser + judger *x.CyclicBuffer + terminated bool + bytes int +} + +func newIteratorReader(rd io.ReadCloser, terminator []byte) *iteratorReader { + return &iteratorReader{ + reader: rd, + terminator: terminator, + buf: make([]byte, len(terminator)), + judger: x.NewCyclicBuffer(len(terminator)), + } +} + +func shiftBuffer(data []byte, header []byte) { + offset := len(header) + if offset > len(data) { + return + } + for i := len(data) - 1; i >= offset; i-- { + data[i] = data[i-offset] + } + for i := 0; i < offset; i++ { + data[i] = header[i] + } +} + +func (r *iteratorReader) Read(p []byte) (n int, err error) { + if r.terminated { + return 0, io.EOF + } + n, err = r.reader.Read(p) + bytes := r.bytes + r.bytes += n + if n == 0 { + return n, err + } + dumped := r.judger.Dump(r.buf) + r.judger.Write(p[:n]) + if r.bytes <= len(r.terminator) { + return 0, nil + } + shiftBuffer(p, r.buf[:x.Min(dumped, n)]) + if r.judger.Compare(r.terminator) { + r.terminated = true + } + if bytes < len(r.buf) { + return n - (len(r.buf) - bytes), err + } + return n, err +} + +func (r *iteratorReader) Close() error { + return r.reader.Close() +} diff --git a/coordinator/remote_executor_test.go b/coordinator/remote_executor_test.go index 01bc78f..326e0b9 100644 --- a/coordinator/remote_executor_test.go +++ b/coordinator/remote_executor_test.go @@ -1,9 +1,42 @@ -package coordinator_test +package coordinator import ( + "fmt" + "io" "testing" + + "github.com/stretchr/testify/assert" ) +// small buffer to test +func testWithBuffer(buf []byte) []byte { + rd, wr := io.Pipe() + defer rd.Close() + go func() { + defer wr.Close() + wr.Write([]byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}) + wr.Write([]byte{1, 2}) + fmt.Println("write end") + }() + var result []byte + reader := newIteratorReader(rd, []byte{9, 10, 11, 12, 13, 14, 15, 16}) + for { + n, err := reader.Read(buf) + if err != nil { + break + } + if n > 0 { + result = append(result, buf[:n]...) + } + } + return result +} + +func TestIteratorReader(t *testing.T) { + assert.Equal(t, []byte{1, 2, 3, 4, 5, 6, 7, 8}, testWithBuffer(make([]byte, 3))) + assert.Equal(t, []byte{1, 2, 3, 4, 5, 6, 7, 8}, testWithBuffer(make([]byte, 9))) +} + func TestRemoteNodeExecuterTagKeys(t *testing.T) { //TODO } diff --git a/coordinator/rpc.go b/coordinator/rpc.go index a4aa5eb..f2b4ce1 100644 --- a/coordinator/rpc.go +++ b/coordinator/rpc.go @@ -4,6 +4,9 @@ import ( "encoding/json" "errors" "fmt" + "regexp" + "time" + "github.com/angopher/chronus/coordinator/internal" "github.com/gogo/protobuf/proto" "github.com/influxdata/influxdb/models" @@ -11,8 +14,6 @@ import ( "github.com/influxdata/influxdb/query" "github.com/influxdata/influxdb/tsdb" "github.com/influxdata/influxql" - "regexp" - "time" ) //go:generate protoc --gogo_out=. internal/data.proto @@ -159,7 +160,9 @@ func (r *ExecuteStatementRequest) SetStatement(statement string) { func (r *ExecuteStatementRequest) Database() string { return r.pb.GetDatabase() } // SetDatabase sets the database name. -func (r *ExecuteStatementRequest) SetDatabase(database string) { r.pb.Database = proto.String(database) } +func (r *ExecuteStatementRequest) SetDatabase(database string) { + r.pb.Database = proto.String(database) +} // MarshalBinary encodes the object to a binary format. func (r *ExecuteStatementRequest) MarshalBinary() ([]byte, error) { @@ -685,9 +688,10 @@ func (r *CreateIteratorRequest) UnmarshalBinary(data []byte) error { // CreateIteratorResponse represents a response from remote iterator creation. type CreateIteratorResponse struct { - DataType influxql.DataType - SeriesN int - Err error + DataType influxql.DataType + SeriesN int + Err error + Termination []byte } // MarshalBinary encodes r to a binary format. diff --git a/coordinator/service.go b/coordinator/service.go index f062eba..02285fb 100644 --- a/coordinator/service.go +++ b/coordinator/service.go @@ -14,6 +14,7 @@ import ( "sync/atomic" "time" + "github.com/angopher/chronus/x" "github.com/influxdata/influxdb" "github.com/influxdata/influxdb/models" "github.com/influxdata/influxdb/pkg/tracing" @@ -338,8 +339,7 @@ func (s *Service) handleConn(conn net.Conn) { return default: } - now := time.Now() - conn.SetReadDeadline(now.Add(500 * time.Millisecond)) + conn.SetReadDeadline(time.Now().Add(500 * time.Millisecond)) typ, data, err := ReadTLV(conn) conn.SetDeadline(time.Time{}) if err == io.EOF { @@ -352,6 +352,9 @@ func (s *Service) handleConn(conn net.Conn) { s.Logger.Error("read error", zap.Error(err)) break } + if typ == 0 { + continue + } respType, resp, err := s.handle(conn, typ, data) if resp == nil { continue @@ -798,14 +801,13 @@ func (s *Service) processCreateIteratorRequest(conn net.Conn, buf []byte) { seriesN = itr.Stats().SeriesN } // Encode success response. - if err := EncodeTLV(conn, respType, &CreateIteratorResponse{DataType: dataType, SeriesN: seriesN}); err != nil { + itrTerminator := x.RandBytes(8) + if err := EncodeTLV(conn, respType, &CreateIteratorResponse{DataType: dataType, SeriesN: seriesN, Termination: itrTerminator}); err != nil { s.Logger.Error("error writing CreateIterator response, EncodeTLV fail", zap.Error(err)) atomic.AddInt64(&s.stats.CreateIteratorFail, 1) ioError = true return } - //XXX - ioError = true // Exit if no iterator was produced. if itr == nil { @@ -813,7 +815,8 @@ func (s *Service) processCreateIteratorRequest(conn net.Conn, buf []byte) { } // Stream iterator to connection. - if err := query.NewIteratorEncoder(conn).EncodeIterator(itr); err != nil { + encoder := query.NewIteratorEncoder(conn) + if err := encoder.EncodeIterator(itr); err != nil { s.Logger.Error("encoding CreateIterator iterator fail", zap.Error(err)) atomic.AddInt64(&s.stats.CreateIteratorFail, 1) ioError = true @@ -824,13 +827,18 @@ func (s *Service) processCreateIteratorRequest(conn net.Conn, buf []byte) { if trace != nil { span.Finish() - if err := query.NewIteratorEncoder(conn).EncodeTrace(trace); err != nil { + if err := encoder.EncodeTrace(trace); err != nil { s.Logger.Error("EncodeTrace fail", zap.Error(err)) atomic.AddInt64(&s.stats.CreateIteratorFail, 1) ioError = true return } } + + // Write termination of iterator + if _, err := conn.Write(itrTerminator); err != nil { + ioError = true + } } func (s *Service) processFieldDimensionsRequest(buf []byte) (*FieldDimensionsResponse, error) { diff --git a/services/hh/config.go b/services/hh/config.go index 0e1fe52..dcc292c 100644 --- a/services/hh/config.go +++ b/services/hh/config.go @@ -41,7 +41,7 @@ type Config struct { Dir string `toml:"dir"` MaxSize int64 `toml:"max-size"` MaxAge toml.Duration `toml:"max-age"` - RetryConcurrency int64 `toml:"retry-concurrency"` + RetryConcurrency int32 `toml:"retry-concurrency"` RetryRateLimit int64 `toml:"retry-rate-limit"` RetryInterval toml.Duration `toml:"retry-interval"` RetryMaxInterval toml.Duration `toml:"retry-max-interval"` diff --git a/services/hh/node_processor.go b/services/hh/node_processor.go index d2d6696..9dfffb2 100644 --- a/services/hh/node_processor.go +++ b/services/hh/node_processor.go @@ -5,7 +5,6 @@ import ( "encoding/binary" "fmt" "io" - "log" "os" "sync" "time" @@ -15,6 +14,7 @@ import ( "github.com/angopher/chronus/services/meta" "github.com/influxdata/influxdb/models" + "go.uber.org/zap" "golang.org/x/time/rate" ) @@ -24,6 +24,12 @@ const ( writeNodeReqPoints = "writeNodeReqPoints" ) +var ( + // for concurrency control + maxActiveProcessorCount = int32(0) + activeProcessorCount = int32(0) +) + // NodeProcessor encapsulates a queue of hinted-handoff data for a node, and the // transmission of the data to the node. type NodeProcessor struct { @@ -45,7 +51,7 @@ type NodeProcessor struct { writer shardWriter stats *NodeProcessorStatistics - Logger *log.Logger + Logger *zap.SugaredLogger } type NodeProcessorStatistics struct { @@ -56,6 +62,10 @@ type NodeProcessorStatistics struct { WriteNodeReqPoints int64 } +func SetMaxActiveProcessorCount(n int32) { + maxActiveProcessorCount = n +} + // NewNodeProcessor returns a new NodeProcessor for the given node, using dir for // the hinted-handoff data. func NewNodeProcessor(nodeID uint64, dir string, w shardWriter, m metaClient) *NodeProcessor { @@ -70,10 +80,14 @@ func NewNodeProcessor(nodeID uint64, dir string, w shardWriter, m metaClient) *N writer: w, meta: m, stats: &NodeProcessorStatistics{}, - Logger: log.New(os.Stderr, "[handoff] ", log.LstdFlags), + Logger: zap.NewNop().Sugar(), } } +func (n *NodeProcessor) WithLogger(logger *zap.Logger) { + n.Logger = logger.With(zap.String("service", "hh_processor")).Sugar() +} + // Open opens the NodeProcessor. It will read and write data present in dir, and // start transmitting data to the node. A NodeProcessor must be opened before it // can accept hinted data. @@ -190,46 +204,99 @@ func (n *NodeProcessor) LastModified() (time.Time, error) { func (n *NodeProcessor) run() { defer n.wg.Done() - currInterval := time.Duration(n.RetryInterval) - if currInterval > time.Duration(n.RetryMaxInterval) { - currInterval = time.Duration(n.RetryMaxInterval) + waitTime := time.Duration(n.RetryInterval) + if waitTime > time.Duration(n.RetryMaxInterval) { + waitTime = time.Duration(n.RetryMaxInterval) } + purgeTimer := time.NewTimer(n.PurgeInterval) + defer purgeTimer.Stop() + sendingTimer := time.NewTimer(waitTime) + defer sendingTimer.Stop() for { select { case <-n.done: return - case <-time.After(n.PurgeInterval): + case <-purgeTimer.C: if err := n.queue.PurgeOlderThan(time.Now().Add(-n.MaxAge)); err != nil { - n.Logger.Printf("failed to purge for node %d: %s", n.nodeID, err.Error()) + n.Logger.Warnf("failed to purge for node %d: %s", n.nodeID, err.Error()) + } + purgeTimer.Reset(n.PurgeInterval) + + case <-sendingTimer.C: + waitTime = n.sendingLoop(waitTime) + sendingTimer.Reset(waitTime) + + } + } +} + +func concurrencyAllow() bool { + if maxActiveProcessorCount < 1 { + return true + } + waiter := time.NewTimer(time.Second) + defer waiter.Stop() + for { + select { + case <-waiter.C: + // timeout + return false + default: + if atomic.AddInt32(&activeProcessorCount, 1) <= maxActiveProcessorCount { + return true } + // restore & next + atomic.AddInt32(&activeProcessorCount, -1) + } + } +} - case <-time.After(currInterval): - limiter := rate.NewLimiter(rate.Limit(n.RetryRateLimit), 10*n.RetryRateLimit) - ctx := context.Background() - for { - c, err := n.SendWrite() - if err != nil { - if err == io.EOF { - // No more data, return to configured interval - currInterval = time.Duration(n.RetryInterval) - } else { - currInterval = currInterval * 2 - if currInterval > time.Duration(n.RetryMaxInterval) { - currInterval = time.Duration(n.RetryMaxInterval) - } - } - break - } - - // Success! Ensure backoff is cancelled. - currInterval = time.Duration(n.RetryInterval) - - limiter.WaitN(ctx, c) +func (n *NodeProcessor) sendingLoop(curDelay time.Duration) (nextDelay time.Duration) { + var ( + sent int + err error + ) + + // concurrency check + if maxActiveProcessorCount > 0 { + if !concurrencyAllow() { + n.Logger.Info("concurrency control, skip scheduling once") + return n.RetryInterval + } + defer atomic.AddInt32(&activeProcessorCount, -1) + } + + // Bytes rate limit + if n.RetryRateLimit > 0 { + bytesLimiter := rate.NewLimiter(rate.Limit(n.RetryRateLimit), 10*n.RetryRateLimit) + defer func() { + if sent > 0 { + n.Logger.Infof("write to %d with %d bytes", n.nodeID, sent) + bytesLimiter.WaitN(context.Background(), sent) } + }() + } + + sent, err = n.SendWrite() + if err == nil { + // Success! Ensure backoff is cancelled. + nextDelay = n.RetryInterval + return + } + + if err == io.EOF { + // No more data, return to configured interval + nextDelay = n.RetryInterval + } else { + // backoff + nextDelay = 2 * curDelay + if nextDelay > n.RetryMaxInterval { + nextDelay = n.RetryMaxInterval } } + return } // SendWrite attempts to sent the current block of hinted data to the target node. If successful, @@ -256,10 +323,10 @@ func (n *NodeProcessor) SendWrite() (int, error) { // unmarshal the byte slice back to shard ID and points shardID, points, err := unmarshalWrite(buf) if err != nil { - n.Logger.Printf("unmarshal write failed: %v", err) + n.Logger.Warnf("unmarshal write failed: %v", err) // Try to skip it. if err := n.queue.Advance(); err != nil { - n.Logger.Printf("failed to advance queue for node %d: %s", n.nodeID, err.Error()) + n.Logger.Warnf("failed to advance queue for node %d: %s", n.nodeID, err.Error()) } return 0, err } @@ -272,7 +339,7 @@ func (n *NodeProcessor) SendWrite() (int, error) { atomic.AddInt64(&n.stats.WriteNodeReqPoints, int64(len(points))) if err := n.queue.Advance(); err != nil { - n.Logger.Printf("failed to advance queue for node %d: %s", n.nodeID, err.Error()) + n.Logger.Warnf("failed to advance queue for node %d: %s", n.nodeID, err.Error()) } return len(buf), nil @@ -300,7 +367,7 @@ func (n *NodeProcessor) Tail() string { func (n *NodeProcessor) Active() (bool, error) { nio, err := n.meta.DataNode(n.nodeID) if err != nil && err != meta.ErrNodeNotFound { - n.Logger.Printf("failed to determine if node %d is active: %s", n.nodeID, err.Error()) + n.Logger.Warnf("failed to determine if node %d is active: %s", n.nodeID, err.Error()) return false, err } return nio != nil, nil diff --git a/services/hh/service.go b/services/hh/service.go index 120f779..de6b124 100644 --- a/services/hh/service.go +++ b/services/hh/service.go @@ -60,6 +60,7 @@ type metaClient interface { func NewService(c Config, w shardWriter, m metaClient) *Service { //key := strings.Join([]string{"hh", c.Dir}, ":") //tags := map[string]string{"path": c.Dir} + SetMaxActiveProcessorCount(c.RetryConcurrency) return &Service{ cfg: c, @@ -76,6 +77,15 @@ func (s *Service) WithLogger(log *zap.Logger) { s.Logger = log.With(zap.String("service", "handoff")).Sugar() } +func (s *Service) createProcessor(nodeID uint64) *NodeProcessor { + n := NewNodeProcessor(nodeID, s.pathforNode(nodeID), s.shardWriter, s.MetaClient) + n.RetryInterval = time.Duration(s.cfg.RetryInterval) + n.RetryMaxInterval = time.Duration(s.cfg.RetryMaxInterval) + n.RetryRateLimit = int(s.cfg.RetryRateLimit) + n.WithLogger(s.Logger.Desugar()) + return n +} + // Open opens the hinted handoff service. func (s *Service) Open() error { s.mu.Lock() @@ -111,7 +121,7 @@ func (s *Service) Open() error { continue } - n := NewNodeProcessor(nodeID, s.pathforNode(nodeID), s.shardWriter, s.MetaClient) + n := s.createProcessor(nodeID) if err := n.Open(); err != nil { return err } @@ -190,7 +200,7 @@ func (s *Service) WriteShard(shardID, ownerID uint64, points []models.Point) err processor, ok = s.processors[ownerID] if !ok { - processor = NewNodeProcessor(ownerID, s.pathforNode(ownerID), s.shardWriter, s.MetaClient) + processor = s.createProcessor(ownerID) if err := processor.Open(); err != nil { return err } diff --git a/x/cyclic_buffer.go b/x/cyclic_buffer.go new file mode 100644 index 0000000..1f34525 --- /dev/null +++ b/x/cyclic_buffer.go @@ -0,0 +1,93 @@ +package x + +// CyclicBuffer is a buffer which refills cyclically after writes. +// It can be compared to specific []byte efficiently. +// Well, it's NOT THREAD SAFE. +type CyclicBuffer struct { + buf []byte + pos int + cnt int +} + +func NewCyclicBuffer(capacity int) *CyclicBuffer { + return &CyclicBuffer{ + buf: make([]byte, capacity), + } +} + +func (c *CyclicBuffer) forward() { + c.pos++ + if c.pos >= len(c.buf) { + c.pos = 0 + } +} + +func (c *CyclicBuffer) write(data []byte, offset int) { + l := len(data) + for i := offset; i < l; i++ { + c.buf[c.pos] = data[i] + c.forward() + } +} + +// Dump dumps internal data into given slice and returns write byte count +// ATTENTION: the given buffer should not smaller than buffer capacity +func (c *CyclicBuffer) Dump(data []byte) int { + l := len(c.buf) + if len(data) < l { + return 0 + } + posRead := c.pos + posWrite := 0 + skip := len(c.buf) - c.cnt + for i := 0; i < len(c.buf); i++ { + if skip > 0 { + skip-- + goto NEXT + } + data[posWrite] = c.buf[posRead] + posWrite++ + NEXT: + posRead++ + if posRead >= l { + posRead %= l + } + } + return posWrite +} + +func (c *CyclicBuffer) Write(data []byte) { + sz := len(data) + if sz < 1 { + return + } + c.cnt += sz + offset := sz - len(c.buf) + if offset < 0 { + offset = 0 + } + c.write(data, offset) +} + +// Compare returns true when matched +func (c *CyclicBuffer) Compare(data []byte) bool { + l0 := len(c.buf) + l1 := len(data) + if l0 != l1 { + return false + } + pos := c.pos + for i := 0; i < l1; i++ { + if data[i] != c.buf[pos] { + return false + } + pos++ + if pos >= l0 { + pos %= l0 + } + if pos == c.pos { + break + } + } + return true +} diff --git a/x/cyclic_buffer_test.go b/x/cyclic_buffer_test.go new file mode 100644 index 0000000..f91f433 --- /dev/null +++ b/x/cyclic_buffer_test.go @@ -0,0 +1,30 @@ +package x + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestCyclicBuffer(t *testing.T) { + buf := NewCyclicBuffer(4) + buf1 := []byte{0, 0, 0, 0} + buf.Write([]byte{}) + assert.False(t, buf.Compare([]byte{})) + assert.False(t, buf.Compare([]byte{0, 0, 0})) + assert.True(t, buf.Compare([]byte{0, 0, 0, 0})) + assert.Equal(t, 0, buf.Dump(buf1)) + assert.Equal(t, []byte{0, 0, 0, 0}, buf1) + buf.Write([]byte{1}) + assert.True(t, buf.Compare([]byte{0, 0, 0, 1})) + buf.Write([]byte{22}) + assert.True(t, buf.Compare([]byte{0, 0, 1, 22})) + assert.Equal(t, 2, buf.Dump(buf1)) + assert.Equal(t, []byte{1, 22}, buf1[:2]) + buf.Write([]byte{12, 33, 11}) + assert.True(t, buf.Compare([]byte{22, 12, 33, 11})) + buf.Write([]byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14}) + assert.Equal(t, 4, buf.Dump(buf1)) + assert.Equal(t, []byte{11, 12, 13, 14}, buf1) + assert.True(t, buf.Compare([]byte{11, 12, 13, 14})) +} diff --git a/x/math.go b/x/math.go index 5d0fd57..e0f39fa 100644 --- a/x/math.go +++ b/x/math.go @@ -1,5 +1,11 @@ package x +import ( + "encoding/binary" + "math/rand" + "time" +) + func Max(args ...int) int { result := args[0] for i := 1; i < len(args); i++ { @@ -19,3 +25,27 @@ func Min(args ...int) int { } return result } + +func fillOnce(data []byte, pos int) int { + var buf [8]byte + binary.BigEndian.PutUint64(buf[:], rand.Uint64()) + cnt := 0 + for pos+cnt < len(data) && cnt < len(buf) { + data[pos+cnt] = buf[cnt%8] + cnt++ + } + return cnt +} + +func RandBytes(n int) []byte { + if n < 1 { + return []byte{} + } + data := make([]byte, n) + pos := 0 + rand.Seed(time.Now().UnixNano()) + for pos < n { + pos += fillOnce(data, pos) + } + return data +} diff --git a/x/math_test.go b/x/math_test.go new file mode 100644 index 0000000..1de28e4 --- /dev/null +++ b/x/math_test.go @@ -0,0 +1,25 @@ +package x + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestMax(t *testing.T) { + assert.Equal(t, 0, Max(-1, 0, -33, 0, -99)) + assert.Equal(t, 3, Max(-1, 0, -33, 3, -99)) +} + +func TestMin(t *testing.T) { + assert.Equal(t, -99, Min(-1, 0, -33, 0, -99)) + assert.Equal(t, -33, Min(-1, 0, -33, 3, 99)) +} + +func TestRandBytes(t *testing.T) { + assert.Equal(t, 0, len(RandBytes(0))) + assert.Equal(t, 2, len(RandBytes(2))) + assert.Equal(t, 7, len(RandBytes(7))) + assert.Equal(t, 8, len(RandBytes(8))) + assert.Equal(t, 17, len(RandBytes(17))) +} diff --git a/x/pool.go b/x/pool.go index 4068e6a..1f0bd9c 100644 --- a/x/pool.go +++ b/x/pool.go @@ -254,6 +254,7 @@ func (c *boundedPool) wrapConn(conn net.Conn) *pooledConn { // net.Conn's Close() method. type pooledConn struct { net.Conn + mu sync.Mutex c *boundedPool lastUse time.Time unusable bool @@ -261,18 +262,32 @@ type pooledConn struct { // Close() puts the given connects back to the pool instead of closing it. func (p *pooledConn) Close() error { + p.mu.Lock() + defer p.mu.Unlock() + if p.Conn == nil { + return nil + } + defer func() { + p.Conn = nil + }() if p.unusable { if p.Conn != nil { return p.Conn.Close() } return nil } - p.lastUse = time.Now() - return p.c.put(p) + return p.c.put(&pooledConn{ + Conn: p.Conn, + c: p.c, + lastUse: time.Now(), + unusable: false, + }) } // MarkUnusable() marks the connection not usable any more, to let the pool close it instead of returning it to pool. func (p *pooledConn) MarkUnusable() { + p.mu.Lock() + defer p.mu.Unlock() if p.unusable { return } From 7a0debe7216f4bee030a71cfc70797d2ee26514c Mon Sep 17 00:00:00 2001 From: Jason Joo Date: Mon, 21 Sep 2020 12:58:08 +0800 Subject: [PATCH 10/46] Polish logging of metad --- cmd/influxd-ctl/action/action.go | 2 - cmd/influxd-ctl/command/command.go | 19 +------- cmd/influxd-ctl/main.go | 1 - cmd/metad/main.go | 46 +++++++++++++------ raftmeta/apply.go | 73 ++++++++++++++++-------------- raftmeta/config.go | 8 +++- raftmeta/meta_service.go | 12 +++-- raftmeta/node.go | 8 ++-- raftmeta/transport.go | 4 +- services/controller/service.go | 26 ++++++++--- services/hh/queue.go | 4 ++ 11 files changed, 115 insertions(+), 88 deletions(-) diff --git a/cmd/influxd-ctl/action/action.go b/cmd/influxd-ctl/action/action.go index 8b2dfbb..5dd3826 100644 --- a/cmd/influxd-ctl/action/action.go +++ b/cmd/influxd-ctl/action/action.go @@ -128,8 +128,6 @@ func GetShard(addr, shard string) error { return err } if resp.Code != 0 { - color.Set(color.Bold) - color.Red("Error: ", resp.Msg, "\n") return errors.New(resp.Msg) } diff --git a/cmd/influxd-ctl/command/command.go b/cmd/influxd-ctl/command/command.go index 430c1d7..eef9f23 100644 --- a/cmd/influxd-ctl/command/command.go +++ b/cmd/influxd-ctl/command/command.go @@ -17,23 +17,6 @@ var ( DataNodeAddress string ) -func DatabaseCommand() *cli.Command { - return &cli.Command{ - Name: "database", - Usage: "Database info", - Subcommands: []*cli.Command{ - { - Name: "list", - Usage: "list all databases", - Action: func(ctx *cli.Context) error { - - return nil - }, - }, - }, - } -} - func NodeCommand() *cli.Command { return &cli.Command{ Name: "node", @@ -107,7 +90,7 @@ func ShardCommand() *cli.Command { return errors.New("Please specify shard id") } if err := action.GetShard(DataNodeAddress, ctx.Args().Get(0)); err != nil { - fmt.Println(err) + fmt.Println(err.Error()) } return nil }, diff --git a/cmd/influxd-ctl/main.go b/cmd/influxd-ctl/main.go index 9628354..92b268e 100644 --- a/cmd/influxd-ctl/main.go +++ b/cmd/influxd-ctl/main.go @@ -13,7 +13,6 @@ func main() { app.Usage = "Maintain the data nodes in cluster" app.Commands = []*cli.Command{ - command.DatabaseCommand(), command.NodeCommand(), command.ShardCommand(), } diff --git a/cmd/metad/main.go b/cmd/metad/main.go index 26817e8..365bfa7 100644 --- a/cmd/metad/main.go +++ b/cmd/metad/main.go @@ -17,10 +17,36 @@ import ( "gopkg.in/natefinch/lumberjack.v2" ) +func initialLogging(config *raftmeta.Config) (*zap.Logger, error) { + cfg := logger.NewConfig() + switch strings.ToLower(config.LogLevel) { + case "info": + cfg.Level = zap.InfoLevel + case "warn", "warning": + cfg.Level = zap.WarnLevel + case "debug": + cfg.Level = zap.DebugLevel + case "fatal": + cfg.Level = zap.FatalLevel + case "panic": + cfg.Level = zap.PanicLevel + } + if config.LogDir != "" { + dir := strings.TrimRight(config.LogDir, string(filepath.Separator)) + return cfg.New(&lumberjack.Logger{ + Filename: filepath.Join(dir, "metad.log"), + MaxSize: 100, + MaxBackups: 5, + Compress: true, + }) + } else { + return cfg.New(os.Stderr) + } +} + func main() { f := flag.NewFlagSet("metad", flag.ExitOnError) configFile := f.String("config", "", "Specify config file") - logDir := f.String("logdir", "", "Log to specified directory") f.Parse(os.Args[1:]) config := raftmeta.NewConfig() @@ -40,22 +66,14 @@ func main() { RetentionAutoCreate: config.RetentionAutoCreate, LoggingEnabled: true, }) - - var log *zap.Logger - if *logDir != "" { - dir := strings.TrimRight(*logDir, string(filepath.Separator)) - log = logger.New(&lumberjack.Logger{ - Filename: filepath.Join(dir, "metad.log"), - MaxSize: 100, - MaxBackups: 5, - Compress: true, - }) - } else { - log = logger.New(os.Stderr) + log, err := initialLogging(&config) + if err != nil { + fmt.Fprintln(os.Stderr, "Error to initialize logging", err) + return } metaCli.WithLogger(log) - err := metaCli.Open() + err = metaCli.Open() x.Check(err) node := raftmeta.NewRaftNode(config) diff --git a/raftmeta/apply.go b/raftmeta/apply.go index e7bb881..7fb5235 100644 --- a/raftmeta/apply.go +++ b/raftmeta/apply.go @@ -4,11 +4,12 @@ import ( "encoding/json" "errors" "fmt" + "time" + "github.com/angopher/chronus/raftmeta/internal" "github.com/angopher/chronus/x" "github.com/influxdata/influxdb/services/meta" "go.uber.org/zap" - "time" ) func (s *RaftNode) applyCommitted(proposal *internal.Proposal, index uint64) error { @@ -17,7 +18,7 @@ func (s *RaftNode) applyCommitted(proposal *internal.Proposal, index uint64) err s.ApplyCallBack(proposal, index) } msgName, _ := internal.MessageTypeName[proposal.Type] - s.Logger.Info("applyCommitted ", zap.String("type", msgName)) + s.Logger.Debug("applyCommitted ", zap.String("type", msgName)) pctx := s.props.pctx(proposal.Key) if pctx == nil { @@ -42,19 +43,19 @@ func (s *RaftNode) applyCommitted(proposal *internal.Proposal, index uint64) err var req DropDatabaseReq err := json.Unmarshal(proposal.Data, &req) x.Check(err) - s.Logger.Info(fmt.Sprintf("req %+v", req)) + s.Logger.Debug(fmt.Sprintf("req %+v", req)) return s.MetaCli.DropDatabase(req.Name) case internal.DropRetentionPolicy: var req DropRetentionPolicyReq err := json.Unmarshal(proposal.Data, &req) x.Check(err) - s.Logger.Info(fmt.Sprintf("req %+v", req)) + s.Logger.Debug(fmt.Sprintf("req %+v", req)) return s.MetaCli.DropRetentionPolicy(req.Database, req.Policy) case internal.CreateShardGroup: var req CreateShardGroupReq err := json.Unmarshal(proposal.Data, &req) x.Check(err) - s.Logger.Info(fmt.Sprintf("req %+v", req)) + s.Logger.Debug(fmt.Sprintf("req %+v", req)) sg, err := s.MetaCli.CreateShardGroup(req.Database, req.Policy, time.Unix(req.Timestamp, 0)) if err == nil && pctx.retData != nil { if sg != nil { @@ -68,7 +69,7 @@ func (s *RaftNode) applyCommitted(proposal *internal.Proposal, index uint64) err var req CreateDataNodeReq err := json.Unmarshal(proposal.Data, &req) x.Check(err) - s.Logger.Info(fmt.Sprintf("req %+v", req)) + s.Logger.Debug(fmt.Sprintf("req %+v", req)) ni, err := s.MetaCli.CreateDataNode(req.HttpAddr, req.TcpAddr) if err == nil && pctx.retData != nil { x.AssertTrue(ni != nil) @@ -79,13 +80,13 @@ func (s *RaftNode) applyCommitted(proposal *internal.Proposal, index uint64) err var req DeleteDataNodeReq err := json.Unmarshal(proposal.Data, &req) x.Check(err) - s.Logger.Info(fmt.Sprintf("req %+v", req)) + s.Logger.Debug(fmt.Sprintf("req %+v", req)) return s.MetaCli.DeleteDataNode(req.Id) case internal.CreateRetentionPolicy: var req CreateRetentionPolicyReq err := json.Unmarshal(proposal.Data, &req) x.Check(err) - s.Logger.Info(fmt.Sprintf("req %+v", req)) + s.Logger.Debug(fmt.Sprintf("req %+v", req)) var duration *time.Duration if req.Rps.Duration > 0 { @@ -109,7 +110,7 @@ func (s *RaftNode) applyCommitted(proposal *internal.Proposal, index uint64) err var req UpdateRetentionPolicyReq err := json.Unmarshal(proposal.Data, &req) x.Check(err) - s.Logger.Info(fmt.Sprintf("req %+v", req)) + s.Logger.Debug(fmt.Sprintf("req %+v", req)) var duration *time.Duration if req.Rps.Duration > 0 { @@ -138,7 +139,7 @@ func (s *RaftNode) applyCommitted(proposal *internal.Proposal, index uint64) err var req CreateDatabaseWithRetentionPolicyReq err := json.Unmarshal(proposal.Data, &req) x.Check(err) - s.Logger.Info(fmt.Sprintf("req %+v", req)) + s.Logger.Debug(fmt.Sprintf("req %+v", req)) var duration *time.Duration if req.Rps.Duration > 0 { @@ -162,7 +163,7 @@ func (s *RaftNode) applyCommitted(proposal *internal.Proposal, index uint64) err var req CreateUserReq err := json.Unmarshal(proposal.Data, &req) x.Check(err) - s.Logger.Info(fmt.Sprintf("req %+v", req)) + s.Logger.Debug(fmt.Sprintf("req %+v", req)) user, err := s.MetaCli.CreateUser(req.Name, req.Password, req.Admin) if err == nil && pctx.retData != nil { x.AssertTrue(user != nil) @@ -174,35 +175,35 @@ func (s *RaftNode) applyCommitted(proposal *internal.Proposal, index uint64) err var req DropUserReq err := json.Unmarshal(proposal.Data, &req) x.Check(err) - s.Logger.Info(fmt.Sprintf("req %+v", req)) + s.Logger.Debug(fmt.Sprintf("req %+v", req)) return s.MetaCli.DropUser(req.Name) case internal.UpdateUser: var req UpdateUserReq err := json.Unmarshal(proposal.Data, &req) x.Check(err) - s.Logger.Info(fmt.Sprintf("req %+v", req)) + s.Logger.Debug(fmt.Sprintf("req %+v", req)) return s.MetaCli.UpdateUser(req.Name, req.Password) case internal.SetPrivilege: var req SetPrivilegeReq err := json.Unmarshal(proposal.Data, &req) x.Check(err) - s.Logger.Info(fmt.Sprintf("req %+v", req)) + s.Logger.Debug(fmt.Sprintf("req %+v", req)) return s.MetaCli.SetPrivilege(req.UserName, req.Database, req.Privilege) case internal.SetAdminPrivilege: var req SetAdminPrivilegeReq err := json.Unmarshal(proposal.Data, &req) x.Check(err) - s.Logger.Info(fmt.Sprintf("req %+v", req)) + s.Logger.Debug(fmt.Sprintf("req %+v", req)) return s.MetaCli.SetAdminPrivilege(req.UserName, req.Admin) case internal.Authenticate: var req AuthenticateReq err := json.Unmarshal(proposal.Data, &req) x.Check(err) - s.Logger.Info(fmt.Sprintf("req %+v", req)) + s.Logger.Debug(fmt.Sprintf("req %+v", req)) user, err := s.MetaCli.Authenticate(req.UserName, req.Password) if err == nil { x.AssertTrue(user != nil) @@ -214,28 +215,28 @@ func (s *RaftNode) applyCommitted(proposal *internal.Proposal, index uint64) err var req AddShardOwnerReq err := json.Unmarshal(proposal.Data, &req) x.Check(err) - s.Logger.Info(fmt.Sprintf("add shard owner req %+v", req)) + s.Logger.Debug(fmt.Sprintf("add shard owner req %+v", req)) return s.MetaCli.AddShardOwner(req.ShardID, req.NodeID) case internal.RemoveShardOwner: var req RemoveShardOwnerReq err := json.Unmarshal(proposal.Data, &req) x.Check(err) - s.Logger.Info(fmt.Sprintf("remove shard owner req %+v", req)) + s.Logger.Debug(fmt.Sprintf("remove shard owner req %+v", req)) return s.MetaCli.RemoveShardOwner(req.ShardID, req.NodeID) case internal.DropShard: var req DropShardReq err := json.Unmarshal(proposal.Data, &req) x.Check(err) - s.Logger.Info(fmt.Sprintf("req %+v", req)) + s.Logger.Debug(fmt.Sprintf("req %+v", req)) return s.MetaCli.DropShard(req.Id) case internal.TruncateShardGroups: var req TruncateShardGroupsReq err := json.Unmarshal(proposal.Data, &req) x.Check(err) - s.Logger.Info(fmt.Sprintf("req %+v", req)) + s.Logger.Debug(fmt.Sprintf("req %+v", req)) return s.MetaCli.TruncateShardGroups(req.Time) case internal.PruneShardGroups: @@ -245,49 +246,49 @@ func (s *RaftNode) applyCommitted(proposal *internal.Proposal, index uint64) err var req DeleteShardGroupReq err := json.Unmarshal(proposal.Data, &req) x.Check(err) - s.Logger.Info(fmt.Sprintf("req %+v", req)) + s.Logger.Debug(fmt.Sprintf("req %+v", req)) return s.MetaCli.DeleteShardGroup(req.Database, req.Policy, req.Id) case internal.PrecreateShardGroups: var req PrecreateShardGroupsReq err := json.Unmarshal(proposal.Data, &req) x.Check(err) - s.Logger.Info(fmt.Sprintf("req %+v", req)) + s.Logger.Debug(fmt.Sprintf("req %+v", req)) return s.MetaCli.PrecreateShardGroups(req.From, req.To) case internal.CreateContinuousQuery: var req CreateContinuousQueryReq err := json.Unmarshal(proposal.Data, &req) x.Check(err) - s.Logger.Info(fmt.Sprintf("req %+v", req)) + s.Logger.Debug(fmt.Sprintf("req %+v", req)) return s.MetaCli.CreateContinuousQuery(req.Database, req.Name, req.Query) case internal.DropContinuousQuery: var req DropContinuousQueryReq err := json.Unmarshal(proposal.Data, &req) x.Check(err) - s.Logger.Info(fmt.Sprintf("req %+v", req)) + s.Logger.Debug(fmt.Sprintf("req %+v", req)) return s.MetaCli.DropContinuousQuery(req.Database, req.Name) case internal.CreateSubscription: var req CreateSubscriptionReq err := json.Unmarshal(proposal.Data, &req) x.Check(err) - s.Logger.Info(fmt.Sprintf("req %+v", req)) + s.Logger.Debug(fmt.Sprintf("req %+v", req)) return s.MetaCli.CreateSubscription(req.Database, req.Rp, req.Name, req.Mode, req.Destinations) case internal.DropSubscription: var req DropSubscriptionReq err := json.Unmarshal(proposal.Data, &req) x.Check(err) - s.Logger.Info(fmt.Sprintf("req %+v", req)) + s.Logger.Debug(fmt.Sprintf("req %+v", req)) return s.MetaCli.DropSubscription(req.Database, req.Rp, req.Name) case internal.AcquireLease: var req AcquireLeaseReq err := json.Unmarshal(proposal.Data, &req) x.Check(err) - s.Logger.Info(fmt.Sprintf("req %+v", req)) + s.Logger.Debug(fmt.Sprintf("req %+v", req)) lease, err := s.leases.Acquire(req.Name, req.NodeId) if err == nil && pctx.retData != nil { x.AssertTrue(lease != nil) @@ -314,7 +315,7 @@ func (s *RaftNode) applyCommitted(proposal *internal.Proposal, index uint64) err mcd := s.MetaCli.Data() //消除DeleteAt和TruncatedAt对checksum的影响 - for i, _ := range mcd.Databases { + for i := range mcd.Databases { db := &mcd.Databases[i] for j, _ := range db.RetentionPolicies { rp := &db.RetentionPolicies[j] @@ -331,27 +332,33 @@ func (s *RaftNode) applyCommitted(proposal *internal.Proposal, index uint64) err s.lastChecksum.checksum = x.Md5(data) s.lastChecksum.needVerify = true - detail := fmt.Sprintf("index:%d, checksum:%s, data:%+v", index, s.lastChecksum.checksum, mcd) - s.Logger.Info(fmt.Sprintf("create checksum costs:%s detail:%s", time.Now().Sub(start), detail)) + s.Logger.Debug( + fmt.Sprintf("create checksum costs:%s detail:%s", + time.Now().Sub(start), + fmt.Sprintf("index:%d, checksum:%s, data:%+v", + index, s.lastChecksum.checksum, mcd, + ), + ), + ) case internal.VerifyChecksumMsg: start := time.Now() var req internal.VerifyChecksum err := json.Unmarshal(proposal.Data, &req) x.Check(err) if req.NodeID == s.ID { - s.Logger.Info("ignore checksum. self trigger this verify") + s.Logger.Warn("ignore checksum. self trigger this verify") s.lastChecksum.needVerify = false return nil } if s.lastChecksum.index == 0 { //have no checksum only when restart - s.Logger.Info("ignore checksum. have no checksum", zap.Uint64("index", req.Index)) + s.Logger.Warn("ignore checksum. have no checksum", zap.Uint64("index", req.Index)) return nil } if s.lastChecksum.index != req.Index { - s.Logger.Warn("ingore checksum", zap.Uint64("last index:", s.lastChecksum.index), zap.Uint64("index", req.Index)) + s.Logger.Warn("ignore checksum", zap.Uint64("last index:", s.lastChecksum.index), zap.Uint64("index", req.Index)) return nil } diff --git a/raftmeta/config.go b/raftmeta/config.go index f6e39df..029d392 100644 --- a/raftmeta/config.go +++ b/raftmeta/config.go @@ -1,11 +1,12 @@ package raftmeta import ( + "io/ioutil" + "net" + "github.com/BurntSushi/toml" "golang.org/x/text/encoding/unicode" "golang.org/x/text/transform" - "io/ioutil" - "net" ) const ( @@ -37,6 +38,9 @@ type Config struct { SnapshotIntervalSec int `toml:"snapshot-interval"` ChecksumIntervalSec int `toml:"checksum-interval"` RetentionAutoCreate bool `toml:"retention-auto-create"` + + LogLevel string `toml:"log-level"` + LogDir string `toml:"log-dir"` } type Peer struct { diff --git a/raftmeta/meta_service.go b/raftmeta/meta_service.go index c877128..0d4077d 100644 --- a/raftmeta/meta_service.go +++ b/raftmeta/meta_service.go @@ -1359,17 +1359,19 @@ func (s *MetaService) AcquireLease(w http.ResponseWriter, r *http.Request) { err = s.ProposeAndWait(internal.AcquireLease, data, lease) if err != nil { resp.RetMsg = err.Error() - s.Logger.Error("AcquireLease fail", - zap.String("Name", req.Name), - zap.Uint64("NodeId", req.NodeId), - zap.Error(err)) + if !strings.Contains(err.Error(), "another node has the lease") { + s.Logger.Error("AcquireLease fail", + zap.String("Name", req.Name), + zap.Uint64("NodeId", req.NodeId), + zap.Error(err)) + } return } resp.Lease = *lease resp.RetCode = 0 resp.RetMsg = "ok" - s.Logger.Info("AcquireLease ok", + s.Logger.Debug("AcquireLease ok", zap.String("Name", req.Name), zap.Uint64("NodeId", req.NodeId)) } diff --git a/raftmeta/node.go b/raftmeta/node.go index 09fdcb0..ca1e327 100644 --- a/raftmeta/node.go +++ b/raftmeta/node.go @@ -371,7 +371,9 @@ func (s *RaftNode) Run() { if leader == s.ID { go func() { err := s.trigerSnapshot() - s.Logger.Error("calculateSnapshot fail", zap.Error(err)) + if err != nil { + s.Logger.Error("calculateSnapshot fail", zap.Error(err)) + } }() } case <-checkSumTicker.C: @@ -417,7 +419,7 @@ func (s *RaftNode) Run() { s.applyCh <- ew } for _, entry := range rd.CommittedEntries { - s.Logger.Info("process entry", zap.Uint64("term", entry.Term), zap.Uint64("index", entry.Index), zap.String("type", entry.Type.String())) + s.Logger.Debug("process entry", zap.Uint64("term", entry.Term), zap.Uint64("index", entry.Index), zap.String("type", entry.Type.String())) ew := &internal.EntryWrapper{Entry: entry, Restore: false} s.applyCh <- ew } @@ -512,7 +514,7 @@ func (s *RaftNode) processApplyCh() { x.Fatalf("Unable to unmarshal proposal: %v %q\n", err, e.Data) } err := s.applyCommitted(proposal, e.Index) - s.Logger.Info("Applied proposal", zap.String("key", proposal.Key), zap.Uint64("index", e.Index), zap.Error(err)) + s.Logger.Debug("Applied proposal", zap.String("key", proposal.Key), zap.Uint64("index", e.Index), zap.Error(err)) s.props.Done(proposal.Key, err) } diff --git a/raftmeta/transport.go b/raftmeta/transport.go index d36ca53..926d5b4 100644 --- a/raftmeta/transport.go +++ b/raftmeta/transport.go @@ -309,9 +309,7 @@ func (s *RaftNode) HandleMessage(w http.ResponseWriter, r *http.Request) { var msg raftpb.Message err = msg.Unmarshal(data) x.Check(err) - if msg.Type != raftpb.MsgHeartbeat && msg.Type != raftpb.MsgHeartbeatResp { - s.Logger.Info("recv message", zap.String("type", msg.Type.String())) - } + s.Logger.Debug("recv message", zap.String("type", msg.Type.String())) s.RecvRaftRPC(context.Background(), msg) } diff --git a/services/controller/service.go b/services/controller/service.go index 0b50aa4..47731a5 100644 --- a/services/controller/service.go +++ b/services/controller/service.go @@ -415,26 +415,38 @@ func (s *Service) shardsResponse(w io.Writer, rp *meta.RetentionPolicyInfo, e er } func (s *Service) handleShard(conn net.Conn) (string, string, *meta.ShardInfo, error) { - var req GetShardRequest - if err := s.readRequest(conn, &req); err != nil { - return "", "", nil, err + var ( + req GetShardRequest + err error + db, rp string + groupInfo *meta.ShardGroupInfo + ) + if err = s.readRequest(conn, &req); err != nil { + goto NOT_FOUND } if req.ShardID < 1 { - return "", "", nil, errors.New("ShardID should be specified") + err = errors.New("ShardID should be specified") + goto NOT_FOUND + } + db, rp, groupInfo = s.MetaClient.ShardOwner(req.ShardID) + fmt.Println("shardId:", req.ShardID, "=>", db, rp, groupInfo) + if db == "" || rp == "" || groupInfo == nil { + err = errors.New("Specified shard could not be found") + goto NOT_FOUND } - db, rp, groupInfo := s.MetaClient.ShardOwner(req.ShardID) for _, shard := range groupInfo.Shards { if shard.ID == req.ShardID { return db, rp, &shard, nil } } - return "", "", nil, errors.New("Specified shard could not be found") +NOT_FOUND: + return "", "", nil, err } func (s *Service) shardResponse(w io.Writer, db, rp string, shard *meta.ShardInfo, e error) { var resp ShardResponse setError(&resp.CommonResp, e) - if e != nil { + if e == nil { resp.ID = shard.ID resp.DB = db resp.Rp = rp diff --git a/services/hh/queue.go b/services/hh/queue.go index a21c948..0fbf199 100644 --- a/services/hh/queue.go +++ b/services/hh/queue.go @@ -621,6 +621,10 @@ func (l *segment) lastModified() (time.Time, error) { l.mu.RLock() defer l.mu.RUnlock() + if l.file == nil { + return time.Time{}, os.ErrNotExist + } + stats, err := os.Stat(l.file.Name()) if err != nil { return time.Time{}, err From 27bbfaeca0df6b4ea146e463a8bf24d7bdb419e4 Mon Sep 17 00:00:00 2001 From: Jason Joo Date: Mon, 21 Sep 2020 14:16:26 +0800 Subject: [PATCH 11/46] Polish log --- raftmeta/apply.go | 4 ++-- raftmeta/transport.go | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/raftmeta/apply.go b/raftmeta/apply.go index 7fb5235..04b8e15 100644 --- a/raftmeta/apply.go +++ b/raftmeta/apply.go @@ -317,9 +317,9 @@ func (s *RaftNode) applyCommitted(proposal *internal.Proposal, index uint64) err //消除DeleteAt和TruncatedAt对checksum的影响 for i := range mcd.Databases { db := &mcd.Databases[i] - for j, _ := range db.RetentionPolicies { + for j := range db.RetentionPolicies { rp := &db.RetentionPolicies[j] - for k, _ := range rp.ShardGroups { + for k := range rp.ShardGroups { sg := &rp.ShardGroups[k] sg.DeletedAt = time.Unix(0, 0) sg.TruncatedAt = time.Unix(0, 0) diff --git a/raftmeta/transport.go b/raftmeta/transport.go index 926d5b4..2b9959d 100644 --- a/raftmeta/transport.go +++ b/raftmeta/transport.go @@ -309,7 +309,9 @@ func (s *RaftNode) HandleMessage(w http.ResponseWriter, r *http.Request) { var msg raftpb.Message err = msg.Unmarshal(data) x.Check(err) - s.Logger.Debug("recv message", zap.String("type", msg.Type.String())) + if msg.Type != raftpb.MsgHeartbeat && msg.Type != raftpb.MsgHeartbeatResp { + s.Logger.Debug("recv message", zap.String("type", msg.Type.String())) + } s.RecvRaftRPC(context.Background(), msg) } From feeca1f2010654a53c41ec0db60a010d422657ed Mon Sep 17 00:00:00 2001 From: Jason Joo Date: Mon, 21 Sep 2020 16:55:44 +0800 Subject: [PATCH 12/46] Complete shard copying feature --- cmd/influxd-ctl/action/action.go | 40 ++++++++++++++++++++++++------ cmd/influxd-ctl/command/command.go | 2 -- cmd/influxd-ctl/main.go | 9 +++++++ coordinator/meta_client_impl.go | 7 +++++- raftmeta/meta_service.go | 1 + services/controller/service.go | 12 ++++++--- services/controller/shard.go | 10 +++++++- services/migrate/executor.go | 12 ++++----- services/migrate/manager.go | 23 +++++++++++------ 9 files changed, 88 insertions(+), 28 deletions(-) diff --git a/cmd/influxd-ctl/action/action.go b/cmd/influxd-ctl/action/action.go index 5dd3826..d0554a1 100644 --- a/cmd/influxd-ctl/action/action.go +++ b/cmd/influxd-ctl/action/action.go @@ -64,12 +64,14 @@ func ListShard(addr, db, rp string) error { reqTyp := byte(controller.RequestShards) req.Database = db req.RetentionPolicy = rp - if err := RequestAndWaitResp(addr, reqTyp, respTyp, req, &resp); err != nil { + nodes, err := getNodes(addr) + if err != nil { + return err + } + if err = RequestAndWaitResp(addr, reqTyp, respTyp, req, &resp); err != nil { return err } if resp.Code != 0 { - color.Set(color.Bold) - color.Red("Error: ", resp.Msg, "\n") return errors.New(resp.Msg) } @@ -104,7 +106,18 @@ func ListShard(addr, db, rp string) error { "\n", ) for _, shard := range g.Shards { - color.Cyan(" |--Shard: %d\tNodes: %v\n", shard.ID, shard.Nodes) + fmt.Print(color.CyanString(" |--Shard: %d\tNodes: [", shard.ID)) + first := true + for _, id := range shard.Nodes { + if n, ok := nodes[id]; ok { + if !first { + fmt.Print(color.CyanString(", ")) + } + fmt.Print(color.CyanString("%s", n.TcpAddr)) + first = false + } + } + fmt.Println(color.CyanString("]")) } if len(g.Shards) == 0 { fmt.Println("No shard") @@ -155,7 +168,7 @@ func CopyShardStatus(addr string) error { color.Set(color.Bold) color.Green("Running Copy Tasks:\n") for _, t := range resp.Tasks { - fmt.Print(t.ShardID, "\t", t.Database, "\t", t.Rp, "\t", t.CurrentSize, "/", t.TotalSize, "\t", t.Source, "=>", t.Destination, "\n") + fmt.Print(t.ShardID, "\t", t.Database, "\t", t.Rp, "\t", t.CurrentSize, "/", t.TotalSize, "\t", t.Source, "\n") } fmt.Println() return nil @@ -224,17 +237,28 @@ func RemoveDataNode(addr, removed_addr string) error { return nil } -func ShowDataNodes(addr string) error { +func getNodes(addr string) (map[uint64]controller.DataNode, error) { var resp controller.ShowDataNodesResponse respTyp := byte(controller.ResponseShowDataNodes) reqTyp := byte(controller.RequestShowDataNodes) if err := RequestAndWaitResp(addr, reqTyp, respTyp, struct{}{}, &resp); err != nil { - return err + return nil, err } + nodes := make(map[uint64]controller.DataNode) + for _, n := range resp.DataNodes { + nodes[n.ID] = n + } + return nodes, nil +} +func ShowDataNodes(addr string) error { color.Set(color.Bold) color.Green("Nodes:\n") - for _, n := range resp.DataNodes { + nodes, err := getNodes(addr) + if err != nil { + return err + } + for _, n := range nodes { fmt.Print(n.ID, "\thttp://", n.HttpAddr, "\ttcp://", n.TcpAddr, "\n") } fmt.Println() diff --git a/cmd/influxd-ctl/command/command.go b/cmd/influxd-ctl/command/command.go index eef9f23..568d025 100644 --- a/cmd/influxd-ctl/command/command.go +++ b/cmd/influxd-ctl/command/command.go @@ -66,8 +66,6 @@ func ShardCommand() *cli.Command { ), Action: func(ctx *cli.Context) error { if ctx.Args().Len() < 2 { - color.Red("Please specify database and retention policy\n") - fmt.Println() return errors.New("Please specify database and retention policy") } if err := action.ListShard(DataNodeAddress, ctx.Args().Get(0), ctx.Args().Get(1)); err != nil { diff --git a/cmd/influxd-ctl/main.go b/cmd/influxd-ctl/main.go index 92b268e..b156fed 100644 --- a/cmd/influxd-ctl/main.go +++ b/cmd/influxd-ctl/main.go @@ -1,9 +1,11 @@ package main import ( + "fmt" "os" "github.com/angopher/chronus/cmd/influxd-ctl/command" + "github.com/fatih/color" "github.com/urfave/cli/v2" ) @@ -11,6 +13,13 @@ func main() { app := &cli.App{} app.Name = "influxd-ctl" app.Usage = "Maintain the data nodes in cluster" + app.ExitErrHandler = func(ctx *cli.Context, err error) { + if err == nil { + return + } + color.Red(err.Error()) + fmt.Println() + } app.Commands = []*cli.Command{ command.NodeCommand(), diff --git a/coordinator/meta_client_impl.go b/coordinator/meta_client_impl.go index afc978a..af78b76 100644 --- a/coordinator/meta_client_impl.go +++ b/coordinator/meta_client_impl.go @@ -9,6 +9,7 @@ import ( "math/rand" "net" "net/http" + "os" "time" "github.com/influxdata/influxdb/services/meta" @@ -606,5 +607,9 @@ func RequestAndParseResponse(url string, data interface{}, resp interface{}) err return err } - return json.Unmarshal(resBody, resp) + err = json.Unmarshal(resBody, resp) + if err != nil { + fmt.Fprintln(os.Stderr, "Error response(unmarshal failed):", string(resBody)) + } + return err } diff --git a/raftmeta/meta_service.go b/raftmeta/meta_service.go index 0d4077d..30101b1 100644 --- a/raftmeta/meta_service.go +++ b/raftmeta/meta_service.go @@ -1448,6 +1448,7 @@ func initHttpHandler(s *MetaService) { http.HandleFunc(SET_ADMIN_PRIVILEGE, s.SetAdminPrivilege) http.HandleFunc(AUTHENTICATE_PATH, s.Authenticate) http.HandleFunc(DROP_SHARD_PATH, s.DropShard) + http.HandleFunc(ADD_SHARD_OWNER, s.AddShardOwner) http.HandleFunc(REMOVE_SHARD_OWNER, s.RemoveShardOwner) http.HandleFunc(TRUNCATE_SHARD_GROUPS_PATH, s.TruncateShardGroups) http.HandleFunc(PRUNE_SHARD_GROUPS_PATH, s.PruneShardGroups) diff --git a/services/controller/service.go b/services/controller/service.go index 47731a5..ba38c50 100644 --- a/services/controller/service.go +++ b/services/controller/service.go @@ -70,6 +70,7 @@ func (s *Service) Open() error { s.Logger.Info("Starting controller service") s.wg.Add(1) + s.migrateManager.Start() go s.serve() return nil } @@ -81,6 +82,7 @@ func (s *Service) Close() error { return err } } + s.migrateManager.Close() s.wg.Wait() return nil } @@ -88,6 +90,7 @@ func (s *Service) Close() error { // WithLogger sets the logger on the service. func (s *Service) WithLogger(log *zap.Logger) { s.Logger = log.With(zap.String("service", "controller")) + s.migrateManager.WithLogger(log.With(zap.String("service", "migrate_manager"))) } // serve serves snapshot requests from the listener. @@ -104,7 +107,7 @@ func (s *Service) serve() { s.Logger.Info("Error accepting snapshot request", zap.Error(err)) continue } - s.Logger.Info("accept new conn.") + s.Logger.Debug("accept new conn.") // Handle connection in separate goroutine. s.wg.Add(1) @@ -202,8 +205,7 @@ func (s *Service) handleCopyShard(conn net.Conn) error { return err } - s.copyShard(req.SourceNodeAddr, req.ShardID) - return nil + return s.copyShard(req.SourceNodeAddr, req.ShardID) } func (s *Service) copyShardResponse(w io.Writer, e error) { @@ -320,6 +322,10 @@ func (s *Service) handleRemoveShard(conn net.Conn) error { } if s.Node.ID == ni.ID { + shard := s.TSDBStore.Shard(req.ShardID) + if shard == nil { + return errors.New("Shard not found") + } if err := s.TSDBStore.DeleteShard(req.ShardID); err != nil { s.Logger.Error("DeleteShard fail.", zap.Error(err)) return err diff --git a/services/controller/shard.go b/services/controller/shard.go index 972ef7f..2cd4004 100644 --- a/services/controller/shard.go +++ b/services/controller/shard.go @@ -8,6 +8,7 @@ import ( "github.com/angopher/chronus/services/migrate" "github.com/angopher/chronus/x" + "go.uber.org/zap" ) func (s *Service) copyShard(sourceAddr string, shardId uint64) error { @@ -50,8 +51,15 @@ func (s *Service) copyShard(sourceAddr string, shardId uint64) error { err = s.TSDBStore.CreateShard(task.Database, task.Retention, task.ShardId, true) if err != nil { + s.Logger.Warn("Failed to load shard into memory", zap.Error(err)) return err } - return s.MetaClient.AddShardOwner(task.ShardId, s.Node.ID) + err = s.MetaClient.AddShardOwner(task.ShardId, s.Node.ID) + if err != nil { + s.Logger.Warn("Failed to add as owner", zap.Error(err)) + return err + } + s.Logger.Info("Successfully add as owner", zap.Uint64("shard", task.ShardId)) + return err } diff --git a/services/migrate/executor.go b/services/migrate/executor.go index 58b15d6..d59013c 100644 --- a/services/migrate/executor.go +++ b/services/migrate/executor.go @@ -67,9 +67,8 @@ func (m *Manager) execute(t *Task) (err error) { return fmt.Errorf("Migration for shard %d failed after retries", t.ShardId) } -func backupFromRemote(t *Task, logger *zap.Logger) (*os.File, error) { +func backupFromRemote(t *Task, logger *zap.SugaredLogger) (*os.File, error) { // prepare local tmp file - suger := logger.Sugar() tmpFilePath := filepath.Join(t.TmpStorePath, fmt.Sprint("shard_", t.ShardId)) tmpFile, err := os.Create(tmpFilePath) if err != nil { @@ -82,6 +81,7 @@ func backupFromRemote(t *Task, logger *zap.Logger) (*os.File, error) { return tmpFile, err } defer conn.Close() + logger.Info("Connected to ", t.SrcHost) req := &snapshotter.Request{ Type: snapshotter.RequestShardBackup, @@ -118,17 +118,16 @@ func backupFromRemote(t *Task, logger *zap.Logger) (*os.File, error) { // limiter in post way t.Limiter.WaitN(ctx, n) if t.ProgressLimiter.Allow() { - suger.Infof("Mirating shard %d: %d copied", t.ShardId, t.Copied) + logger.Infof("Migrating shard %d: %d copied", t.ShardId, t.Copied) } } - suger.Infof("Transfer Shard %d completely", t.ShardId) + logger.Infof("Transfer Shard %d completely with %d bytes", t.ShardId, t.Copied) // XXX: Verify return tmpFile, nil } func (m *Manager) replicate(t *Task) error { - sugar := m.logger.Sugar() if x.Exists(t.DstStorePath) != x.NotExisted { t.error(errors.New("Destination shard directory has already existed")) return nil @@ -139,6 +138,7 @@ func (m *Manager) replicate(t *Task) error { } // transfer to local + m.logger.Infof("start to execute task copying %d from %s", t.ShardId, t.SrcHost) tmpFile, err := backupFromRemote(t, m.logger) defer func() { // remove tmp file @@ -154,7 +154,7 @@ func (m *Manager) replicate(t *Task) error { os.MkdirAll(t.DstStorePath, os.FileMode(0755)) err = restoreFromTar(tmpFile.Name(), t.DstStorePath) if err != nil { - sugar.Errorf("Unpack from backup failed for shard %d: %v", t.ShardId, err) + m.logger.Errorf("Unpack from backup failed for shard %d: %v", t.ShardId, err) // remove incorrect data os.RemoveAll(t.DstStorePath) return err diff --git a/services/migrate/manager.go b/services/migrate/manager.go index 0a046ca..2091053 100644 --- a/services/migrate/manager.go +++ b/services/migrate/manager.go @@ -47,15 +47,20 @@ type Task struct { func (t *Task) succ() { t.Finished = true - t.C <- nil - t.Closer.Close() + close(t.C) + if t.Closer != nil { + t.Closer.Close() + } } func (t *Task) error(err error) { t.Error = err t.Finished = true t.C <- err - t.Closer.Close() + close(t.C) + if t.Closer != nil { + t.Closer.Close() + } } // Manager is the container of tasks running or queued @@ -67,7 +72,7 @@ type Manager struct { taskMap map[uint64]*Task taskQueue []*Task parallel int - logger *zap.Logger + logger *zap.SugaredLogger shouldStop bool } @@ -84,6 +89,7 @@ func NewManager(parallel int) *Manager { taskMap: make(map[uint64]*Task), taskQueue: make([]*Task, 0, TASK_PARALLEL_MAX), cond: sync.NewCond(&sync.Mutex{}), + logger: zap.NewNop().Sugar(), } return m } @@ -92,6 +98,7 @@ func (m *Manager) Start() { for i := 0; i < m.parallel; i++ { go m.loop() } + m.logger.Infof("%d migrators started", m.parallel) } func (m *Manager) loop() { @@ -119,7 +126,7 @@ func (m *Manager) Close() { } func (m *Manager) WithLogger(log *zap.Logger) { - m.logger = log.With(zap.String("service", "migrate.Manager")) + m.logger = log.With(zap.String("service", "migrate.Manager")).Sugar() } func (m *Manager) pop() *Task { @@ -140,10 +147,12 @@ func (m *Manager) Add(task *Task) error { if _, ok := m.taskMap[task.ShardId]; ok { return ErrTaskDuplicated } - task.C = make(chan error, 1) + if task.C == nil { + task.C = make(chan error, 1) + } // Currently limit to 5MB/s, 10MB/s in burst task.Limiter = rate.NewLimiter(5*1024*1024, 10*1024*1024) - task.ProgressLimiter = rate.NewLimiter(0.05, 10) // 1 log every 20 seconds + task.ProgressLimiter = rate.NewLimiter(0.05, 1) // 1 log every 20 seconds m.taskMap[task.ShardId] = task m.taskQueue = append(m.taskQueue, task) m.cond.Signal() From 9f4eac247e259f640291760af9ca889b5d55bfc0 Mon Sep 17 00:00:00 2001 From: Jason Joo Date: Tue, 22 Sep 2020 15:30:33 +0800 Subject: [PATCH 13/46] Introduce dump and restore for disaster recovery for metad Signed-off-by: Jason Joo --- cmd/metad/main.go | 31 +++++++++++++++++++ raftmeta/node.go | 78 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 109 insertions(+) diff --git a/cmd/metad/main.go b/cmd/metad/main.go index 365bfa7..3808fea 100644 --- a/cmd/metad/main.go +++ b/cmd/metad/main.go @@ -11,6 +11,7 @@ import ( "github.com/angopher/chronus/raftmeta" imeta "github.com/angopher/chronus/services/meta" "github.com/angopher/chronus/x" + "github.com/coreos/etcd/raft/raftpb" "github.com/influxdata/influxdb/logger" "github.com/influxdata/influxdb/services/meta" "go.uber.org/zap" @@ -47,6 +48,8 @@ func initialLogging(config *raftmeta.Config) (*zap.Logger, error) { func main() { f := flag.NewFlagSet("metad", flag.ExitOnError) configFile := f.String("config", "", "Specify config file") + dumpFile := f.String("dump", "", "Boot and dump the snapshot to a file specified") + restoreFile := f.String("restore", "", "Boot and restore data from the snapshot specified") f.Parse(os.Args[1:]) config := raftmeta.NewConfig() @@ -80,6 +83,34 @@ func main() { node.MetaCli = metaCli node.WithLogger(log) + // dump only + if *dumpFile != "" { + err := node.Dump(*dumpFile) + if err != nil { + fmt.Println("Dump to file error:", err) + return + } + fmt.Println("Dumped to", *dumpFile) + return + } + + // restore + if *restoreFile != "" { + // set conf state first + var ids []uint64 + for _, n := range node.Config.Peers { + ids = append(ids, n.RaftId) + } + node.SetConfState(&raftpb.ConfState{ + Nodes: ids, + }) + err = node.Restore(*restoreFile) + if err != nil { + node.Logger.Warn("Restore from file failed", zap.Error(err)) + return + } + } + t := raftmeta.NewTransport() t.WithLogger(log) diff --git a/raftmeta/node.go b/raftmeta/node.go index ca1e327..c15a7b2 100644 --- a/raftmeta/node.go +++ b/raftmeta/node.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "io" "math/rand" "os" "sync" @@ -229,6 +230,83 @@ func NewRaftNode(config Config) *RaftNode { } } +func (s *RaftNode) resetPeersInSnapshot(snapshot *raftpb.Snapshot) error { + var sndata internal.SnapshotData + err := json.Unmarshal(snapshot.Data, &sndata) + if err != nil { + return err + } + sndata.PeersAddr = make(map[uint64]string) + for _, p := range s.Config.Peers { + sndata.PeersAddr[p.RaftId] = p.Addr + } + snapshot.Data, err = json.Marshal(&sndata) + if err != nil { + return err + } + return nil +} + +func (s *RaftNode) Restore(filePath string) error { + f, err := os.OpenFile(filePath, os.O_RDONLY, 0) + if err != nil { + return err + } + defer f.Close() + info, err := os.Lstat(filePath) + if err != nil { + return err + } + s.Logger.Warn("Restore from snapfile", zap.String("file", filePath), zap.Int64("size", info.Size())) + snapdata := make([]byte, info.Size()) + _, err = io.ReadFull(f, snapdata) + if err != nil { + return err + } + snapshot := raftpb.Snapshot{} + err = json.Unmarshal(snapdata, &snapshot) + if err != nil { + return err + } + s.Logger.Warn(fmt.Sprintf( + "Term=%d, Index=%d", + snapshot.Metadata.Term, + snapshot.Metadata.Index, + )) + if s.RaftConfState == nil { + return errors.New("ConfState has not been set yet") + } + snapshot.Metadata.ConfState = *s.RaftConfState + s.Logger.Warn(fmt.Sprintf( + "Nodes=%v, Learners=%v", + snapshot.Metadata.ConfState.Nodes, + snapshot.Metadata.ConfState.Learners, + )) + err = s.resetPeersInSnapshot(&snapshot) + if err != nil { + return err + } + return s.Storage.Save(raftpb.HardState{}, []raftpb.Entry{}, snapshot) +} + +func (s *RaftNode) Dump(filePath string) error { + f, err := os.OpenFile(filePath, os.O_WRONLY|os.O_CREATE, 0644) + if err != nil { + return err + } + defer f.Close() + sp, err := s.Storage.Snapshot() + if err != nil { + return err + } + data, err := json.Marshal(&sp) + if err != nil { + return err + } + _, err = f.Write(data) + return err +} + func (s *RaftNode) WithLogger(log *zap.Logger) { s.Logger = log.With(zap.String("raftmeta", "RaftNode")) } From 51ec0cb59e8147fd7511a01e9b5851d94e4b9f19 Mon Sep 17 00:00:00 2001 From: Jason Joo Date: Mon, 5 Oct 2020 14:45:22 +0800 Subject: [PATCH 14/46] Polish logging for metad Signed-off-by: Jason Joo --- cmd/metad/main.go | 48 ++++------------ coordinator/cluster_meta_client.go | 6 +- go.mod | 1 + logging/config.go | 7 +++ logging/log.go | 92 ++++++++++++++++++++++++++++++ raftmeta/apply.go | 2 +- raftmeta/badger_logger_bridge.go | 31 ++++++++++ raftmeta/config.go | 5 +- raftmeta/meta_service.go | 15 +---- raftmeta/node.go | 40 ++++++------- raftmeta/raft_logger_bridge.go | 64 +++++++++++++++++++++ raftmeta/raft_test.go | 3 +- services/meta/client.go | 72 +++-------------------- 13 files changed, 244 insertions(+), 142 deletions(-) create mode 100644 logging/config.go create mode 100644 logging/log.go create mode 100644 raftmeta/badger_logger_bridge.go create mode 100644 raftmeta/raft_logger_bridge.go diff --git a/cmd/metad/main.go b/cmd/metad/main.go index 3808fea..df5aa80 100644 --- a/cmd/metad/main.go +++ b/cmd/metad/main.go @@ -4,47 +4,18 @@ import ( "flag" "fmt" "os" - "path/filepath" - "strings" "github.com/BurntSushi/toml" + "github.com/angopher/chronus/logging" "github.com/angopher/chronus/raftmeta" imeta "github.com/angopher/chronus/services/meta" "github.com/angopher/chronus/x" "github.com/coreos/etcd/raft/raftpb" - "github.com/influxdata/influxdb/logger" + "github.com/dgraph-io/badger" "github.com/influxdata/influxdb/services/meta" "go.uber.org/zap" - "gopkg.in/natefinch/lumberjack.v2" ) -func initialLogging(config *raftmeta.Config) (*zap.Logger, error) { - cfg := logger.NewConfig() - switch strings.ToLower(config.LogLevel) { - case "info": - cfg.Level = zap.InfoLevel - case "warn", "warning": - cfg.Level = zap.WarnLevel - case "debug": - cfg.Level = zap.DebugLevel - case "fatal": - cfg.Level = zap.FatalLevel - case "panic": - cfg.Level = zap.PanicLevel - } - if config.LogDir != "" { - dir := strings.TrimRight(config.LogDir, string(filepath.Separator)) - return cfg.New(&lumberjack.Logger{ - Filename: filepath.Join(dir, "metad.log"), - MaxSize: 100, - MaxBackups: 5, - Compress: true, - }) - } else { - return cfg.New(os.Stderr) - } -} - func main() { f := flag.NewFlagSet("metad", flag.ExitOnError) configFile := f.String("config", "", "Specify config file") @@ -63,25 +34,30 @@ func main() { return } - fmt.Printf("config:%+v\n", config) - metaCli := imeta.NewClient(&meta.Config{ RetentionAutoCreate: config.RetentionAutoCreate, LoggingEnabled: true, }) - log, err := initialLogging(&config) + log, err := logging.InitialLogging(&logging.Config{ + Format: config.LogFormat, + Level: config.LogLevel, + Dir: config.LogDir, + }) if err != nil { fmt.Fprintln(os.Stderr, "Error to initialize logging", err) return } + badger.SetLogger(raftmeta.NewBadgerLoggerBridge(log)) + + suger := log.Sugar() + suger.Debug("config: %+v", config) metaCli.WithLogger(log) err = metaCli.Open() x.Check(err) - node := raftmeta.NewRaftNode(config) + node := raftmeta.NewRaftNode(config, log) node.MetaCli = metaCli - node.WithLogger(log) // dump only if *dumpFile != "" { diff --git a/coordinator/cluster_meta_client.go b/coordinator/cluster_meta_client.go index 9d7ea20..b894885 100644 --- a/coordinator/cluster_meta_client.go +++ b/coordinator/cluster_meta_client.go @@ -100,10 +100,9 @@ func (me *ClusterMetaClient) syncLoop() { } func (me *ClusterMetaClient) Start() { + // sync first synchronously wait := me.WaitForDataChanged() - //sync data first and will signal changes - me.syncData() - go me.syncLoop() + go me.syncData() //wait sync meta data from meta server select { case <-time.After(5 * time.Second): @@ -111,6 +110,7 @@ func (me *ClusterMetaClient) Start() { panic("sync meta data failed") case <-wait: } + go me.syncLoop() } func (me *ClusterMetaClient) CreateDatabase(name string) (*meta.DatabaseInfo, error) { diff --git a/go.mod b/go.mod index 55dd242..5a13f9d 100644 --- a/go.mod +++ b/go.mod @@ -20,6 +20,7 @@ require ( github.com/influxdata/platform v0.0.0-20190117200541-d500d3cf5589 github.com/influxdata/roaring v0.4.12 // indirect github.com/influxdata/usage-client v0.0.0-20160829180054-6d3895376368 + github.com/jsternberg/zap-logfmt v1.2.0 github.com/klauspost/compress v1.4.1 // indirect github.com/klauspost/cpuid v1.2.0 // indirect github.com/klauspost/pgzip v1.2.1 // indirect diff --git a/logging/config.go b/logging/config.go new file mode 100644 index 0000000..ab2a9f2 --- /dev/null +++ b/logging/config.go @@ -0,0 +1,7 @@ +package logging + +type Config struct { + Format string + Level string + Dir string +} diff --git a/logging/log.go b/logging/log.go new file mode 100644 index 0000000..6a630d5 --- /dev/null +++ b/logging/log.go @@ -0,0 +1,92 @@ +package logging + +import ( + "fmt" + "io" + "os" + "path/filepath" + "strings" + "time" + + "github.com/influxdata/influxdb/pkg/snowflake" + zaplogfmt "github.com/jsternberg/zap-logfmt" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + "gopkg.in/natefinch/lumberjack.v2" +) + +var ( + gen = snowflake.New(0) +) + +const TimeFormat = "2006-01-02 15:04:05" + +func newEncoderConfig() zapcore.EncoderConfig { + config := zap.NewProductionEncoderConfig() + config.EncodeTime = func(ts time.Time, encoder zapcore.PrimitiveArrayEncoder) { + encoder.AppendString(ts.Local().Format(TimeFormat)) + } + config.EncodeDuration = func(d time.Duration, encoder zapcore.PrimitiveArrayEncoder) { + val := float64(d) / float64(time.Millisecond) + encoder.AppendString(fmt.Sprintf("%.3fms", val)) + } + config.LevelKey = "lvl" + return config +} + +func nextID() string { + return gen.NextString() +} + +func newEncoder(format string) (zapcore.Encoder, error) { + config := newEncoderConfig() + switch format { + case "json": + return zapcore.NewJSONEncoder(config), nil + case "console": + return zapcore.NewConsoleEncoder(config), nil + case "logfmt": + return zaplogfmt.NewEncoder(config), nil + default: + return nil, fmt.Errorf("unknown logging format: %s", format) + } +} + +func createWriter(c *Config) io.Writer { + if c.Dir != "" { + dir := strings.TrimRight(c.Dir, string(filepath.Separator)) + return &lumberjack.Logger{ + Filename: filepath.Join(dir, "metad.log"), + MaxSize: 100, + MaxBackups: 5, + Compress: true, + } + } else { + return os.Stderr + } +} + +func InitialLogging(c *Config) (*zap.Logger, error) { + encoder, err := newEncoder(c.Format) + if err != nil { + return nil, err + } + lvl := zap.InfoLevel + switch strings.ToLower(c.Level) { + case "info": + lvl = zap.InfoLevel + case "warn", "warning": + lvl = zap.WarnLevel + case "debug": + lvl = zap.DebugLevel + case "fatal": + lvl = zap.FatalLevel + case "panic": + lvl = zap.PanicLevel + } + return zap.New(zapcore.NewCore( + encoder, + zapcore.Lock(zapcore.AddSync(createWriter(c))), + lvl, + ), zap.Fields(zap.String("log_id", nextID()))), nil +} diff --git a/raftmeta/apply.go b/raftmeta/apply.go index 04b8e15..0cb92e7 100644 --- a/raftmeta/apply.go +++ b/raftmeta/apply.go @@ -346,7 +346,7 @@ func (s *RaftNode) applyCommitted(proposal *internal.Proposal, index uint64) err err := json.Unmarshal(proposal.Data, &req) x.Check(err) if req.NodeID == s.ID { - s.Logger.Warn("ignore checksum. self trigger this verify") + s.Logger.Debug("ignore checksum. self trigger this verify") s.lastChecksum.needVerify = false return nil } diff --git a/raftmeta/badger_logger_bridge.go b/raftmeta/badger_logger_bridge.go new file mode 100644 index 0000000..441e1d8 --- /dev/null +++ b/raftmeta/badger_logger_bridge.go @@ -0,0 +1,31 @@ +package raftmeta + +import ( + "strings" + + "github.com/dgraph-io/badger" + "go.uber.org/zap" +) + +type badgerLoggerBridge struct { + badger.Logger + suger *zap.SugaredLogger +} + +func NewBadgerLoggerBridge(logger *zap.Logger) *badgerLoggerBridge { + return &badgerLoggerBridge{ + suger: logger.Sugar(), + } +} + +func (b *badgerLoggerBridge) Errorf(format string, args ...interface{}) { + b.suger.Errorf(strings.Trim(format, "\n"), args) +} + +func (b *badgerLoggerBridge) Infof(format string, args ...interface{}) { + b.suger.Infof(strings.Trim(format, "\n"), args) +} + +func (b *badgerLoggerBridge) Warningf(format string, args ...interface{}) { + b.suger.Warnf(strings.Trim(format, "\n"), args) +} diff --git a/raftmeta/config.go b/raftmeta/config.go index 029d392..913c455 100644 --- a/raftmeta/config.go +++ b/raftmeta/config.go @@ -39,8 +39,9 @@ type Config struct { ChecksumIntervalSec int `toml:"checksum-interval"` RetentionAutoCreate bool `toml:"retention-auto-create"` - LogLevel string `toml:"log-level"` - LogDir string `toml:"log-dir"` + LogFormat string `toml:"log-format"` + LogLevel string `toml:"log-level"` + LogDir string `toml:"log-dir"` } type Peer struct { diff --git a/raftmeta/meta_service.go b/raftmeta/meta_service.go index 30101b1..26184ad 100644 --- a/raftmeta/meta_service.go +++ b/raftmeta/meta_service.go @@ -107,18 +107,7 @@ func (s *MetaService) ProposeAndWait(msgType int, data []byte, retData interface pr := &internal.Proposal{Type: msgType} pr.Data = data - resCh := make(chan error) - go func() { - err := s.Node.ProposeAndWait(ctx, pr, retData) - resCh <- err - }() - - var err error - select { - case err = <-resCh: - } - - return err + return s.Node.ProposeAndWait(ctx, pr, retData) } type CreateDatabaseReq struct { @@ -1413,7 +1402,7 @@ type PingResp struct { func (s *MetaService) Ping(w http.ResponseWriter, r *http.Request) { resp := new(PingResp) - resp.Index = s.cli.Data().Index + resp.Index = s.cli.DataIndex() resp.RetCode = 0 resp.RetMsg = "ok" WriteResp(w, &resp) diff --git a/raftmeta/node.go b/raftmeta/node.go index c15a7b2..541fdb2 100644 --- a/raftmeta/node.go +++ b/raftmeta/node.go @@ -178,13 +178,14 @@ type RaftNode struct { Logger *zap.Logger } -func NewRaftNode(config Config) *RaftNode { +func NewRaftNode(config Config, logger *zap.Logger) *RaftNode { c := &raft.Config{ ID: config.RaftId, ElectionTick: config.ElectionTick, HeartbeatTick: config.HeartbeatTick, MaxSizePerMsg: config.MaxSizePerMsg, MaxInflightMsgs: config.MaxInflightMsgs, + Logger: newRaftLoggerBridge(logger), } walDir := config.WalDir @@ -216,6 +217,7 @@ func NewRaftNode(config Config) *RaftNode { leases: meta.NewLeases(meta.DefaultLeaseDuration), ID: c.ID, RaftConfig: c, + Logger: logger.With(zap.String("raftmeta", "RaftNode")), Config: config, RaftCtx: rc, Storage: storage, @@ -307,10 +309,6 @@ func (s *RaftNode) Dump(filePath string) error { return err } -func (s *RaftNode) WithLogger(log *zap.Logger) { - s.Logger = log.With(zap.String("raftmeta", "RaftNode")) -} - // uniqueKey is meant to be unique across all the replicas. func (s *RaftNode) uniqueKey() string { return fmt.Sprintf("%02d-%d", s.ID, s.rand.Uint64()) @@ -515,7 +513,7 @@ func (s *RaftNode) Run() { //TODO:optimize func (s *RaftNode) triggerChecksum() { - s.Logger.Info("trigger check sum") + s.Logger.Info("trigger checksum") if s.lastChecksum.needVerify { go func() { @@ -647,12 +645,15 @@ func (s *RaftNode) ProposeConfChange(ctx context.Context, cc raftpb.ConfChange) } func (s *RaftNode) ProposeAndWait(ctx context.Context, proposal *internal.Proposal, retData interface{}) error { - cctx, cancel := context.WithTimeout(ctx, 5*time.Second) - defer cancel() - che := make(chan error, 1) + if _, ok := ctx.Deadline(); !ok { + // introduce timeout if needed + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, 5*time.Second) + defer cancel() + } pctx := &proposalCtx{ - ch: che, - ctx: cctx, + ch: make(chan error, 1), + ctx: ctx, retData: retData, } key := s.uniqueKey() @@ -667,7 +668,7 @@ func (s *RaftNode) ProposeAndWait(ctx context.Context, proposal *internal.Propos data, err := json.Marshal(proposal) x.Check(err) - if err = s.Propose(cctx, data); err != nil { + if err = s.Propose(ctx, data); err != nil { return x.Wrapf(err, "While proposing") } @@ -676,22 +677,21 @@ func (s *RaftNode) ProposeAndWait(ctx context.Context, proposal *internal.Propos } select { - case err = <-che: + case err = <-pctx.ch: // We arrived here by a call to n.props.Done(). if tr, ok := trace.FromContext(ctx); ok { tr.LazyPrintf("Done with error: %v", err) } return err case <-ctx.Done(): - if tr, ok := trace.FromContext(ctx); ok { - tr.LazyPrintf("External context timed out with error: %v.", ctx.Err()) + if ctx.Err() == context.DeadlineExceeded { + if tr, ok := trace.FromContext(ctx); ok { + tr.LazyPrintf("Propose timed out.") + } + return errInternalRetry } + return ctx.Err() - case <-cctx.Done(): - if tr, ok := trace.FromContext(ctx); ok { - tr.LazyPrintf("Internal context timed out with error: %v. Retrying...", cctx.Err()) - } - return errInternalRetry } } diff --git a/raftmeta/raft_logger_bridge.go b/raftmeta/raft_logger_bridge.go new file mode 100644 index 0000000..9be1d61 --- /dev/null +++ b/raftmeta/raft_logger_bridge.go @@ -0,0 +1,64 @@ +package raftmeta + +import ( + "fmt" + + "github.com/coreos/etcd/raft" + "go.uber.org/zap" +) + +type raftLoggerBridge struct { + raft.Logger + suger *zap.SugaredLogger +} + +func newRaftLoggerBridge(logger *zap.Logger) *raftLoggerBridge { + if logger == nil { + logger = zap.NewNop() + } + return &raftLoggerBridge{ + suger: logger.Sugar(), + } +} + +func (b *raftLoggerBridge) Debug(v ...interface{}) { + b.suger.Debug(v) +} +func (b *raftLoggerBridge) Debugf(format string, v ...interface{}) { + b.suger.Debug(fmt.Sprintf(format, v...)) +} + +func (b *raftLoggerBridge) Error(v ...interface{}) { + b.suger.Error(v) +} +func (b *raftLoggerBridge) Errorf(format string, v ...interface{}) { + b.suger.Error(fmt.Sprintf(format, v...)) +} + +func (b *raftLoggerBridge) Info(v ...interface{}) { + b.suger.Info(v) +} +func (b *raftLoggerBridge) Infof(format string, v ...interface{}) { + b.suger.Info(fmt.Sprintf(format, v...)) +} + +func (b *raftLoggerBridge) Warning(v ...interface{}) { + b.suger.Warn(v) +} +func (b *raftLoggerBridge) Warningf(format string, v ...interface{}) { + b.suger.Warn(fmt.Sprintf(format, v...)) +} + +func (b *raftLoggerBridge) Fatal(v ...interface{}) { + b.suger.Fatal(v) +} +func (b *raftLoggerBridge) Fatalf(format string, v ...interface{}) { + b.suger.Fatal(fmt.Sprintf(format, v...)) +} + +func (b *raftLoggerBridge) Panic(v ...interface{}) { + b.suger.Panic(v) +} +func (b *raftLoggerBridge) Panicf(format string, v ...interface{}) { + b.suger.Panic(fmt.Sprintf(format, v...)) +} diff --git a/raftmeta/raft_test.go b/raftmeta/raft_test.go index 8329160..6bcdd5d 100644 --- a/raftmeta/raft_test.go +++ b/raftmeta/raft_test.go @@ -126,10 +126,9 @@ func newService(config raftmeta.Config, t *fakeTransport, cb func(proposal *inte x.Check(err) log := logger.New(os.Stderr) - node := raftmeta.NewRaftNode(config) + node := raftmeta.NewRaftNode(config, log) node.MetaCli = metaCli node.ApplyCallBack = cb - node.WithLogger(log) node.Transport = t node.InitAndStartNode() diff --git a/services/meta/client.go b/services/meta/client.go index dea46ec..4352980 100644 --- a/services/meta/client.go +++ b/services/meta/client.go @@ -8,17 +8,13 @@ import ( "crypto/sha256" "errors" "io" - "io/ioutil" "net/http" - "os" - "path/filepath" "sort" "sync" "time" "github.com/influxdata/influxdb" "github.com/influxdata/influxdb/logger" - "github.com/influxdata/influxdb/pkg/file" "github.com/influxdata/influxdb/services/meta" "github.com/influxdata/influxql" "go.uber.org/zap" @@ -197,11 +193,12 @@ func (c *Client) DataNodeByTCPHost(tcpAddr string) (*meta.NodeInfo, error) { func (c *Client) DeleteDataNode(id uint64) error { c.mu.Lock() defer c.mu.Unlock() - err := c.cacheData.DeleteDataNode(id) + data := c.cacheData.Clone() + err := data.DeleteDataNode(id) if err != nil { return err } - if err := c.commit(c.cacheData); err != nil { + if err := c.commit(data); err != nil { return err } return nil @@ -1044,20 +1041,15 @@ func (c *Client) DropSubscription(database, rp, name string) error { // SetData overwrites the underlying data in the meta store. func (c *Client) SetData(data *Data) error { c.mu.Lock() + defer c.mu.Unlock() // reset the index so the commit will fire a change event c.cacheData.Index = 0 - // increment the index to force the changed channel to fire - d := data.Clone() - d.Index++ - - if err := c.commit(d); err != nil { + if err := c.commit(data.Clone()); err != nil { return err } - c.mu.Unlock() - return nil } @@ -1136,63 +1128,13 @@ func (c *Client) WithLogger(log *zap.Logger) { // snapshot saves the current meta data to disk. func snapshot(path string, data *Data) error { - //TODO: no need write snapshot to disk + // no need write snapshot to disk return nil - filename := filepath.Join(path, META_FILE) - tmpFile := filename + "tmp" - - f, err := os.Create(tmpFile) - if err != nil { - return err - } - defer f.Close() - - var d []byte - if b, err := data.MarshalBinary(); err != nil { - return err - } else { - d = b - } - - if _, err := f.Write(d); err != nil { - return err - } - - if err = f.Sync(); err != nil { - return err - } - - //close file handle before renaming to support Windows - if err = f.Close(); err != nil { - return err - } - - return file.RenameFile(tmpFile, filename) } // Load loads the current meta data from disk. func (c *Client) Load() error { - //TODO:no need load - return nil - file := filepath.Join(c.path, META_FILE) - - f, err := os.Open(file) - if err != nil { - if os.IsNotExist(err) { - return nil - } - return err - } - defer f.Close() - - data, err := ioutil.ReadAll(f) - if err != nil { - return err - } - - if err := c.cacheData.UnmarshalBinary(data); err != nil { - return err - } + // no need load return nil } From 6041c2b9d5c5020fa56cb7bd839a3310fd84a619 Mon Sep 17 00:00:00 2001 From: Jason Joo Date: Mon, 5 Oct 2020 18:09:56 +0800 Subject: [PATCH 15/46] Deal with short connections / Introduce reclaiming to storage of metad Signed-off-by: Jason Joo --- cmd/influxd/run/command.go | 25 +++++++++---------------- cmd/metad/main.go | 7 ++++--- coordinator/meta_client_impl.go | 33 ++++++++++++++++++--------------- logging/config.go | 7 ++++--- logging/log.go | 2 +- raftmeta/node.go | 24 +++++++++++++++++++++--- 6 files changed, 57 insertions(+), 41 deletions(-) diff --git a/cmd/influxd/run/command.go b/cmd/influxd/run/command.go index 61e9679..c1a2276 100644 --- a/cmd/influxd/run/command.go +++ b/cmd/influxd/run/command.go @@ -12,12 +12,11 @@ import ( "path/filepath" "runtime" "strconv" - "strings" "time" + "github.com/angopher/chronus/logging" "github.com/influxdata/influxdb/logger" "go.uber.org/zap" - "gopkg.in/natefinch/lumberjack.v2" ) const logo = ` @@ -89,20 +88,14 @@ func (cmd *Command) Run(args ...string) error { return fmt.Errorf("%s. To generate a valid configuration file run `influxd config > influxdb.generated.conf`", err) } - if options.LogDir != "" { - dir := strings.TrimRight(options.LogDir, string(filepath.Separator)) - cmd.Logger = logger.New(&lumberjack.Logger{ - Filename: filepath.Join(dir, "influxd.log"), - MaxSize: 100, - MaxBackups: 5, - Compress: true, - }) - } else { - cmd.Logger, _ = config.Logging.New(cmd.Stderr) - } - if cmd.Logger == nil { - // assign the default logger - cmd.Logger = logger.New(cmd.Stderr) + cmd.Logger, err = logging.InitialLogging(&logging.Config{ + Format: config.Logging.Format, + Level: config.Logging.Level.String(), + Dir: options.LogDir, + FileName: "influxd.log", + }) + if err != nil { + return fmt.Errorf("%s. Initialize logging failed", err) } // Attempt to run pprof on :6060 before startup if debug pprof enabled. diff --git a/cmd/metad/main.go b/cmd/metad/main.go index df5aa80..cd04f51 100644 --- a/cmd/metad/main.go +++ b/cmd/metad/main.go @@ -39,9 +39,10 @@ func main() { LoggingEnabled: true, }) log, err := logging.InitialLogging(&logging.Config{ - Format: config.LogFormat, - Level: config.LogLevel, - Dir: config.LogDir, + Format: config.LogFormat, + Level: config.LogLevel, + Dir: config.LogDir, + FileName: "metad.log", }) if err != nil { fmt.Fprintln(os.Stderr, "Error to initialize logging", err) diff --git a/coordinator/meta_client_impl.go b/coordinator/meta_client_impl.go index af78b76..1576bfc 100644 --- a/coordinator/meta_client_impl.go +++ b/coordinator/meta_client_impl.go @@ -19,6 +19,24 @@ import ( imeta "github.com/angopher/chronus/services/meta" ) +var ( + client = http.Client{ + Transport: &http.Transport{ + Dial: func(netw, addr string) (net.Conn, error) { + c, err := net.DialTimeout(netw, addr, time.Second) + if err != nil { + return nil, err + } + return c, nil + }, + MaxConnsPerHost: 500, + MaxIdleConns: 100, + MaxIdleConnsPerHost: 20, + IdleConnTimeout: 60 * time.Second, + }, + } +) + type MetaClientImpl struct { Addrs []string } @@ -580,21 +598,6 @@ func RequestAndParseResponse(url string, data interface{}, resp interface{}) err return err } req.Header.Set("Content-Type", "application/json") - req.Header.Set("Connection", "close") - - client := http.Client{ - Transport: &http.Transport{ - Dial: func(netw, addr string) (net.Conn, error) { - deadline := time.Now().Add(10 * time.Second) //TODO: timeout from config - c, err := net.DialTimeout(netw, addr, time.Second) - if err != nil { - return nil, err - } - c.SetDeadline(deadline) - return c, nil - }, - }, - } res, err := client.Do(req) if err != nil { diff --git a/logging/config.go b/logging/config.go index ab2a9f2..7f3f228 100644 --- a/logging/config.go +++ b/logging/config.go @@ -1,7 +1,8 @@ package logging type Config struct { - Format string - Level string - Dir string + Format string + Level string + Dir string + FileName string } diff --git a/logging/log.go b/logging/log.go index 6a630d5..57d0a1d 100644 --- a/logging/log.go +++ b/logging/log.go @@ -56,7 +56,7 @@ func createWriter(c *Config) io.Writer { if c.Dir != "" { dir := strings.TrimRight(c.Dir, string(filepath.Separator)) return &lumberjack.Logger{ - Filename: filepath.Join(dir, "metad.log"), + Filename: filepath.Join(dir, c.FileName), MaxSize: 100, MaxBackups: 5, Compress: true, diff --git a/raftmeta/node.go b/raftmeta/node.go index 541fdb2..54df79d 100644 --- a/raftmeta/node.go +++ b/raftmeta/node.go @@ -141,7 +141,8 @@ type RaftNode struct { Config Config //用于存储raft日志和snapshot - Storage *raftwal.DiskStorage + Storage *raftwal.DiskStorage + walStore *badger.DB //节点之间的通信模块 Transport interface { @@ -188,13 +189,13 @@ func NewRaftNode(config Config, logger *zap.Logger) *RaftNode { Logger: newRaftLoggerBridge(logger), } - walDir := config.WalDir - x.Checkf(os.MkdirAll(walDir, 0700), "Error while creating WAL dir.") + x.Checkf(os.MkdirAll(config.WalDir, 0700), "Error while creating WAL dir.") kvOpt := badger.DefaultOptions kvOpt.SyncWrites = true kvOpt.Dir = config.WalDir kvOpt.ValueDir = config.WalDir kvOpt.TableLoadingMode = options.MemoryMap + kvOpt.ValueLogFileSize = 64 << 20 walStore, err := badger.Open(kvOpt) x.Checkf(err, "Error while creating badger KV WAL store") @@ -221,6 +222,7 @@ func NewRaftNode(config Config, logger *zap.Logger) *RaftNode { Config: config, RaftCtx: rc, Storage: storage, + walStore: walStore, Done: make(chan struct{}), props: newProposals(), rand: rand.New(&lockedSource{src: rand.NewSource(time.Now().UnixNano())}), @@ -375,6 +377,20 @@ func (s *RaftNode) resetPeersFromConfig() { } } +func (s *RaftNode) reclaimDiskSpace() { + ticker := time.NewTicker(5 * time.Minute) + defer ticker.Stop() + + for { + select { + case <-ticker.C: + s.walStore.RunValueLogGC(0.1) + case <-s.Done: + return + } + } +} + func (s *RaftNode) InitAndStartNode() { peers := make([]raft.Peer, 0, len(s.Config.Peers)) for _, p := range s.Config.Peers { @@ -439,6 +455,8 @@ func (s *RaftNode) Run() { t := time.NewTicker(time.Duration(s.Config.TickTimeMs) * time.Millisecond) defer t.Stop() + go s.reclaimDiskSpace() + var leader uint64 for { From 8e50c41601aa7f105682da7d622188c975cdda7f Mon Sep 17 00:00:00 2001 From: Jason Joo Date: Mon, 5 Oct 2020 23:49:26 +0800 Subject: [PATCH 16/46] tweak a little on badger db Signed-off-by: Jason Joo --- raftmeta/meta_service.go | 2 ++ raftmeta/node.go | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/raftmeta/meta_service.go b/raftmeta/meta_service.go index 26184ad..9c844ae 100644 --- a/raftmeta/meta_service.go +++ b/raftmeta/meta_service.go @@ -9,6 +9,8 @@ import ( "strings" "time" + _ "net/http/pprof" + "github.com/angopher/chronus/raftmeta/internal" imeta "github.com/angopher/chronus/services/meta" "github.com/influxdata/influxdb/services/meta" diff --git a/raftmeta/node.go b/raftmeta/node.go index 54df79d..1f427d5 100644 --- a/raftmeta/node.go +++ b/raftmeta/node.go @@ -195,7 +195,9 @@ func NewRaftNode(config Config, logger *zap.Logger) *RaftNode { kvOpt.Dir = config.WalDir kvOpt.ValueDir = config.WalDir kvOpt.TableLoadingMode = options.MemoryMap - kvOpt.ValueLogFileSize = 64 << 20 + kvOpt.ValueLogFileSize = 8 << 20 + kvOpt.MaxTableSize = 8 << 20 + kvOpt.NumLevelZeroTables = 2 walStore, err := badger.Open(kvOpt) x.Checkf(err, "Error while creating badger KV WAL store") From 96aa3151851cd45f28e157f42659e99559f2b1b0 Mon Sep 17 00:00:00 2001 From: Jason Joo Date: Tue, 6 Oct 2020 09:43:41 +0800 Subject: [PATCH 17/46] Tweak badger db for metad --- raftmeta/node.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/raftmeta/node.go b/raftmeta/node.go index 1f427d5..056a292 100644 --- a/raftmeta/node.go +++ b/raftmeta/node.go @@ -198,6 +198,10 @@ func NewRaftNode(config Config, logger *zap.Logger) *RaftNode { kvOpt.ValueLogFileSize = 8 << 20 kvOpt.MaxTableSize = 8 << 20 kvOpt.NumLevelZeroTables = 2 + kvOpt.NumMemtables = 4 + kvOpt.LevelSizeMultiplier = 5 + kvOpt.LevelOneSize = 32 << 20 + kvOpt.NumCompactors = 2 walStore, err := badger.Open(kvOpt) x.Checkf(err, "Error while creating badger KV WAL store") From 6c1573b4fe07dced935d778735522bb7ee8f4670 Mon Sep 17 00:00:00 2001 From: Jason Joo Date: Sat, 10 Oct 2020 00:46:48 +0800 Subject: [PATCH 18/46] Fine the logging when checksum failed Signed-off-by: Jason Joo --- raftmeta/apply.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/raftmeta/apply.go b/raftmeta/apply.go index 0cb92e7..1fba100 100644 --- a/raftmeta/apply.go +++ b/raftmeta/apply.go @@ -363,7 +363,10 @@ func (s *RaftNode) applyCommitted(proposal *internal.Proposal, index uint64) err } s.Logger.Info("checksum", zap.Uint64("index", req.Index), zap.String("checksum", s.lastChecksum.checksum)) - x.AssertTruef(s.lastChecksum.checksum == req.Checksum, "verify checksum fail") + x.AssertTruef(s.lastChecksum.checksum == req.Checksum, "verify checksum fail, %s != %s, data=%+v", + s.lastChecksum.checksum, req.Checksum, + s.MetaCli.Data(), + ) s.Logger.Info(fmt.Sprintf("verify checksum success. costs %s", time.Now().Sub(start))) default: return fmt.Errorf("Unkown msg type:%d", proposal.Type) From 12238e680d05a3825139bcfc027340fe2eeea09e Mon Sep 17 00:00:00 2001 From: Jason Joo Date: Sat, 10 Oct 2020 02:08:46 +0800 Subject: [PATCH 19/46] Introduce freezed state to node in data store Signed-off-by: Jason Joo --- services/meta/data.go | 105 +++++++++++++++++++++++++++++++++---- services/meta/data_test.go | 60 +++++++++++++++++++++ services/meta/errors.go | 6 +++ 3 files changed, 161 insertions(+), 10 deletions(-) diff --git a/services/meta/data.go b/services/meta/data.go index 2632191..930d666 100644 --- a/services/meta/data.go +++ b/services/meta/data.go @@ -2,6 +2,7 @@ package meta import ( "encoding/json" + "errors" "fmt" "sort" "time" @@ -14,8 +15,9 @@ import ( // Data represents the top level collection of all metadata. type Data struct { meta.Data - MetaNodes []meta.NodeInfo - DataNodes []meta.NodeInfo + MetaNodes []meta.NodeInfo + DataNodes []meta.NodeInfo + FreezedDataNodes []uint64 // data nodes that can't create new shard on MaxNodeID uint64 } @@ -68,6 +70,65 @@ func (data *Data) CreateDataNode(host, tcpHost string) (uint64, error) { return existingID, nil } +func existInNodes(nodes []meta.NodeInfo, id uint64) *meta.NodeInfo { + for _, n := range nodes { + if n.ID == id { + return &n + } + } + return nil +} + +func getFreezed(freezed []uint64, id uint64) int { + for i, n := range freezed { + if n == id { + return i + } + } + return -1 +} + +func (data *Data) UnfreezeDataNode(id uint64) error { + if id == 0 { + return ErrNodeIDRequired + } + + if existInNodes(data.DataNodes, id) == nil { + return ErrNodeNotFound + } + + i := getFreezed(data.FreezedDataNodes, id) + if i == -1 { + return ErrNodeNotFreezed + } + + data.FreezedDataNodes = append(data.FreezedDataNodes[:i], data.FreezedDataNodes[i+1:]...) + + return nil +} + +func (data *Data) IsFreezeDataNode(id uint64) bool { + return getFreezed(data.FreezedDataNodes, id) > -1 +} + +func (data *Data) FreezeDataNode(id uint64) error { + if id == 0 { + return ErrNodeIDRequired + } + + if existInNodes(data.DataNodes, id) == nil { + return ErrNodeNotFound + } + + if getFreezed(data.FreezedDataNodes, id) > -1 { + return ErrNodeAlreadyFreezed + } + + data.FreezedDataNodes = append(data.FreezedDataNodes, id) + + return nil +} + // DeleteDataNode removes a node from the Meta store. // // If necessary, DeleteDataNode reassigns ownership of any shards that @@ -159,10 +220,16 @@ func (data *Data) DeleteDataNode(id uint64) error { } } } + + // Delete from freezed nodes if necessarily + if i := getFreezed(data.FreezedDataNodes, id); i > -1 { + data.FreezedDataNodes = append(data.FreezedDataNodes[:i], data.FreezedDataNodes[i+1:]...) + } + return nil } -func (data *Data) CloneNodes(src []meta.NodeInfo) []meta.NodeInfo { +func cloneNodes(src []meta.NodeInfo) []meta.NodeInfo { if len(src) == 0 { return []meta.NodeInfo{} } @@ -278,8 +345,10 @@ func (data *Data) Clone() *Data { other.Databases = data.CloneDatabases() other.Users = data.CloneUsers() - other.DataNodes = data.CloneNodes(data.DataNodes) - other.MetaNodes = data.CloneNodes(data.MetaNodes) + other.DataNodes = cloneNodes(data.DataNodes) + other.MetaNodes = cloneNodes(data.MetaNodes) + other.FreezedDataNodes = make([]uint64, len(data.FreezedDataNodes)) + copy(other.FreezedDataNodes, data.FreezedDataNodes) return &other } @@ -350,18 +419,34 @@ func (data *Data) CreateShardGroup(database, policy string, timestamp time.Time) return nil } + // Don't create shard on freezed nodes + availableNodes := make([]meta.NodeInfo, 0, len(data.DataNodes)) + freezedNodes := make(map[uint64]bool) + for _, n := range data.FreezedDataNodes { + freezedNodes[n] = true + } + for _, n := range data.DataNodes { + if freezedNodes[n.ID] { + continue + } + availableNodes = append(availableNodes, n) + } + // Require at least one replica but no more replicas than nodes. replicaN := rpi.ReplicaN if replicaN == 0 { replicaN = 1 - } else if replicaN > len(data.DataNodes) { - replicaN = len(data.DataNodes) + } else if replicaN > len(availableNodes) { + replicaN = len(availableNodes) + } + if replicaN < 1 { + return errors.New("No replica can be assigned") } // Determine shard count by node count divided by replication factor. // This will ensure nodes will get distributed across nodes evenly and // replicated the correct number of times. - shardN := len(data.DataNodes) / replicaN + shardN := len(availableNodes) / replicaN // Create the shard group. data.MaxShardGroupID++ @@ -383,11 +468,11 @@ func (data *Data) CreateShardGroup(database, policy string, timestamp time.Time) // Assign data nodes to shards via round robin. // Start from a repeatably "random" place in the node list. - nodeIndex := int(data.Index % uint64(len(data.DataNodes))) + nodeIndex := int(data.Index % uint64(len(availableNodes))) for i := range sgi.Shards { si := &sgi.Shards[i] for j := 0; j < replicaN; j++ { - nodeID := data.DataNodes[nodeIndex%len(data.DataNodes)].ID + nodeID := availableNodes[nodeIndex%len(availableNodes)].ID si.Owners = append(si.Owners, meta.ShardOwner{NodeID: nodeID}) nodeIndex++ } diff --git a/services/meta/data_test.go b/services/meta/data_test.go index 988c264..4d3060c 100644 --- a/services/meta/data_test.go +++ b/services/meta/data_test.go @@ -205,3 +205,63 @@ func TestDeleteDataNode(t *testing.T) { o := owners[0] assert.Equal(t, id2, o.NodeID) } + +func TestFreezeDataNode(t *testing.T) { + data := newData() + id1, id2 := initialTwoDataNodes(data) + replicaN := 2 + name := "testdb" + policy := "rp" + duration := time.Hour + data.CreateDatabase(name) + spec := meta.RetentionPolicySpec{ + Name: policy, + ReplicaN: &replicaN, + Duration: &duration, + ShardGroupDuration: time.Minute, + } + data.CreateRetentionPolicy(name, spec.NewRetentionPolicyInfo(), true) + data.CreateShardGroup(name, policy, time.Now()) + sg, err := data.ShardGroups(name, policy) + assert.Nil(t, err) + assert.Equal(t, 1, len(sg)) + assert.Equal(t, 2, len(sg[0].Shards[0].Owners)) + + assert.Equal(t, 0, len(data.FreezedDataNodes)) + + assert.NotNil(t, data.FreezeDataNode(3333)) + assert.Nil(t, data.FreezeDataNode(id1)) + assert.Nil(t, data.FreezeDataNode(id2)) + assert.Equal(t, []uint64{id1, id2}, data.FreezedDataNodes) + assert.True(t, data.IsFreezeDataNode(id1)) + assert.True(t, data.IsFreezeDataNode(id2)) + + assert.Nil(t, data.UnfreezeDataNode(id2)) + assert.Equal(t, []uint64{id1}, data.FreezedDataNodes) + + assert.Nil(t, data.UnfreezeDataNode(id1)) + assert.Equal(t, []uint64{}, data.FreezedDataNodes) + assert.Nil(t, data.FreezeDataNode(id1)) + assert.NotNil(t, data.FreezeDataNode(id1)) + assert.Equal(t, []uint64{id1}, data.FreezedDataNodes) + data.CreateShardGroup(name, policy, time.Now().Add(time.Hour)) + sg, err = data.ShardGroups(name, policy) + assert.Nil(t, err) + assert.Equal(t, 2, len(sg)) + assert.Equal(t, 1, len(sg[1].Shards[0].Owners)) + + // try delete freezed node + data.DeleteDataNode(1) + assert.Equal(t, []uint64{}, data.FreezedDataNodes) +} + +func TestClone(t *testing.T) { + data1 := newData() + id1, id2 := initialTwoDataNodes(data1) + data2 := data1.Clone() + data1.FreezeDataNode(id1) + data2.FreezeDataNode(id2) + assert.Equal(t, 1, len(data1.FreezedDataNodes)) + assert.Equal(t, 1, len(data2.FreezedDataNodes)) + assert.NotEqual(t, data1.FreezedDataNodes, data2.FreezedDataNodes) +} diff --git a/services/meta/errors.go b/services/meta/errors.go index 486fa57..0097f62 100644 --- a/services/meta/errors.go +++ b/services/meta/errors.go @@ -11,6 +11,12 @@ var ( // ErrNodeNotFound is returned when mutating a node that doesn't exist. ErrNodeNotFound = errors.New("node not found") + // ErrNodeAlreadyFreezed represents node has been already freezed + ErrNodeAlreadyFreezed = errors.New("node has been freezed before") + + // ErrNodeNotFreezed represents node is not freezed (is normal) + ErrNodeNotFreezed = errors.New("node hasn't been freezed") + // ErrNodesRequired is returned when at least one node is required for an operation. // This occurs when creating a shard group. ErrNodesRequired = errors.New("at least one node required") From 636b690f7682b9ab28009ebf0a8711692a82e844 Mon Sep 17 00:00:00 2001 From: Jason Joo Date: Mon, 12 Oct 2020 10:20:58 +0800 Subject: [PATCH 20/46] Fix bug breaking consensus of meta servers * Introduce freeze/unfreeze to node preventing new shard's creation temporarily * influxd-ctl supports viewing shards on specific node * Fix bug in DeleteDataNode operation which may break consensus between metad instances Signed-off-by: Jason Joo --- cmd/influxd-ctl/action/action.go | 63 ++++++++++++++++++- cmd/influxd-ctl/command/command.go | 57 ++++++++++++++++-- cmd/influxd/run/server.go | 2 +- cmd/metad-ctl/main.go | 9 +++ coordinator/cluster_meta_client.go | 19 ++++++ coordinator/meta_client_impl.go | 21 +++++++ raftmeta/apply.go | 12 +++- raftmeta/internal/message.go | 2 + raftmeta/meta_client.go | 6 +- raftmeta/meta_service.go | 43 ++++++++++++- raftmeta/path.go | 1 + services/controller/service.go | 97 +++++++++++++++++++++++++++++- services/meta/client.go | 44 ++++++++++++++ services/meta/data.go | 59 ++++++++++-------- services/meta/data_inner_test.go | 24 ++++++++ services/meta/data_test.go | 2 + 16 files changed, 423 insertions(+), 38 deletions(-) create mode 100644 services/meta/data_inner_test.go diff --git a/cmd/influxd-ctl/action/action.go b/cmd/influxd-ctl/action/action.go index d0554a1..d4acca8 100644 --- a/cmd/influxd-ctl/action/action.go +++ b/cmd/influxd-ctl/action/action.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "net" + "sort" "strconv" "time" @@ -57,6 +58,27 @@ func TruncateShards(delay string, addr string) error { return nil } +func ListShardOnNode(addr string, nodeId uint64) error { + var req controller.GetNodeShardsRequest + var resp controller.NodeShardsResponse + respTyp := byte(controller.ResponseNodeShards) + reqTyp := byte(controller.RequestNodeShards) + req.NodeID = nodeId + var err error + if err = RequestAndWaitResp(addr, reqTyp, respTyp, req, &resp); err != nil { + return err + } + if resp.Code != 0 { + return errors.New(resp.Msg) + } + + color.Set(color.Bold) + color.Green(fmt.Sprint("Shards on node ", req.NodeID, ":\n")) + fmt.Println(resp.Shards) + fmt.Println() + return nil +} + func ListShard(addr, db, rp string) error { var req controller.GetShardsRequest var resp controller.ShardsResponse @@ -237,6 +259,33 @@ func RemoveDataNode(addr, removed_addr string) error { return nil } +func freezeDataNode(addr, freezed_addr string, freeze bool) error { + req := &controller.FreezeDataNodeRequest{ + DataNodeAddr: freezed_addr, + Freeze: freeze, + } + + var resp controller.FreezeDataNodeResponse + respTyp := byte(controller.ResponseFreezeDataNode) + reqTyp := byte(controller.RequestFreezeDataNode) + if err := RequestAndWaitResp(addr, reqTyp, respTyp, req, &resp); err != nil { + return err + } + + color.Set(color.Bold) + color.Green("Result: ") + fmt.Println(resp.Msg) + return nil +} + +func FreezeDataNode(addr, freezed_addr string) error { + return freezeDataNode(addr, freezed_addr, true) +} + +func UnfreezeDataNode(addr, freezed_addr string) error { + return freezeDataNode(addr, freezed_addr, false) +} + func getNodes(addr string) (map[uint64]controller.DataNode, error) { var resp controller.ShowDataNodesResponse respTyp := byte(controller.ResponseShowDataNodes) @@ -258,8 +307,20 @@ func ShowDataNodes(addr string) error { if err != nil { return err } + ids := make([]uint64, 0, len(nodes)) for _, n := range nodes { - fmt.Print(n.ID, "\thttp://", n.HttpAddr, "\ttcp://", n.TcpAddr, "\n") + ids = append(ids, n.ID) + } + sort.Slice(ids, func(i, j int) bool { + return ids[i] < ids[j] + }) + for _, id := range ids { + n := nodes[id] + fmt.Print(n.ID, "\thttp://", n.HttpAddr, "\ttcp://", n.TcpAddr) + if n.Freezed { + fmt.Print("\t(freezed)") + } + fmt.Print("\n") } fmt.Println() return nil diff --git a/cmd/influxd-ctl/command/command.go b/cmd/influxd-ctl/command/command.go index 568d025..ef8eaff 100644 --- a/cmd/influxd-ctl/command/command.go +++ b/cmd/influxd-ctl/command/command.go @@ -3,9 +3,9 @@ package command import ( "errors" "fmt" + "strconv" "github.com/angopher/chronus/cmd/influxd-ctl/action" - "github.com/fatih/color" "github.com/urfave/cli/v2" ) @@ -32,13 +32,39 @@ func NodeCommand() *cli.Command { return nil }, }, { + Name: "freeze", + ArgsUsage: "freeze ", + Usage: "freeze specified node preventing creating new shard on", + Action: func(ctx *cli.Context) error { + if ctx.Args().Len() < 1 { + return errors.New("Please specify node addr to be freezed") + } + if err := action.FreezeDataNode(DataNodeAddress, ctx.Args().Get(0)); err != nil { + fmt.Println(err) + } + + return nil + }, + }, { + Name: "unfreeze", + ArgsUsage: "unfreeze ", + Usage: "unfreeze specified node", + Action: func(ctx *cli.Context) error { + if ctx.Args().Len() < 1 { + return errors.New("Please specify node addr to unfreeze") + } + if err := action.UnfreezeDataNode(DataNodeAddress, ctx.Args().Get(0)); err != nil { + fmt.Println(err) + } + + return nil + }, + }, { Name: "remove", ArgsUsage: "remove ", Usage: "remove specified node from cluster", Action: func(ctx *cli.Context) error { if ctx.Args().Len() < 1 { - fmt.Println("Please specify node addr to be removed from cluster") - fmt.Println() return errors.New("Please specify node addr to be removed from cluster") } if err := action.RemoveDataNode(DataNodeAddress, ctx.Args().Get(0)); err != nil { @@ -59,7 +85,7 @@ func ShardCommand() *cli.Command { Subcommands: []*cli.Command{ { Name: "list", - ArgsUsage: "copy ", + ArgsUsage: "list ", Usage: "show all shards of specified retention policy", Description: fmt.Sprint( "List all shards in specified retention policy.", @@ -73,6 +99,27 @@ func ShardCommand() *cli.Command { } return nil + }, + }, { + Name: "node", + ArgsUsage: "node ", + Usage: "show all shards' id on specified node", + Description: fmt.Sprint( + "List all shards on node.", + ), + Action: func(ctx *cli.Context) error { + if ctx.Args().Len() < 1 { + return errors.New("Please specify node id") + } + nodeId, _ := strconv.ParseUint(ctx.Args().Get(0), 10, 64) + if nodeId < 1 { + return errors.New("Please specify node id") + } + if err := action.ListShardOnNode(DataNodeAddress, nodeId); err != nil { + fmt.Println(err) + } + return nil + }, }, { Name: "info", @@ -83,8 +130,6 @@ func ShardCommand() *cli.Command { ), Action: func(ctx *cli.Context) error { if ctx.Args().Len() < 1 { - color.Red("Please specify shard id\n") - fmt.Println() return errors.New("Please specify shard id") } if err := action.GetShard(DataNodeAddress, ctx.Args().Get(0)); err != nil { diff --git a/cmd/influxd/run/server.go b/cmd/influxd/run/server.go index a707b43..75ed1b1 100644 --- a/cmd/influxd/run/server.go +++ b/cmd/influxd/run/server.go @@ -211,7 +211,7 @@ func NewServer(c *Config, buildInfo *BuildInfo, logger *zap.Logger) (*Server, er // If we've already created a data node for our id, we're done n, err := s.ClusterMetaClient.DataNode(nodeID) if err != nil { - s.Logger.Warn("Node id from store can't be used, try to create new", zap.Error(err)) + s.Logger.Warn(fmt.Sprintf("Node id(%d) from store can't be used, try to create new", nodeID), zap.Error(err)) n, err = s.ClusterMetaClient.CreateDataNode(s.httpAPIAddr, s.tcpAddr) if err != nil { s.Logger.Warn(fmt.Sprint("Unable to create data node. err: ", err.Error())) diff --git a/cmd/metad-ctl/main.go b/cmd/metad-ctl/main.go index 6948983..18a53cf 100644 --- a/cmd/metad-ctl/main.go +++ b/cmd/metad-ctl/main.go @@ -1,9 +1,11 @@ package main import ( + "fmt" "os" "github.com/angopher/chronus/cmd/metad-ctl/cmds" + "github.com/fatih/color" "github.com/urfave/cli/v2" ) @@ -11,6 +13,13 @@ func main() { app := &cli.App{} app.Name = "metad-ctl" app.Usage = "Maintain the metad cluster" + app.ExitErrHandler = func(ctx *cli.Context, err error) { + if err == nil { + return + } + color.Red(err.Error()) + fmt.Println() + } app.Commands = []*cli.Command{ cmds.StatusCommand(), diff --git a/coordinator/cluster_meta_client.go b/coordinator/cluster_meta_client.go index b894885..ec29e64 100644 --- a/coordinator/cluster_meta_client.go +++ b/coordinator/cluster_meta_client.go @@ -126,6 +126,25 @@ func (me *ClusterMetaClient) DeleteDataNode(id uint64) error { } return me.cache.DeleteDataNode(id) } + +func (me *ClusterMetaClient) IsDataNodeFreezed(id uint64) bool { + return me.cache.IsDataNodeFreezed(id) +} + +func (me *ClusterMetaClient) FreezeDataNode(id uint64) error { + if err := me.metaCli.FreezeDataNode(id); err != nil { + return err + } + return me.cache.FreezeDataNode(id) +} + +func (me *ClusterMetaClient) UnfreezeDataNode(id uint64) error { + if err := me.metaCli.UnfreezeDataNode(id); err != nil { + return err + } + return me.cache.UnfreezeDataNode(id) +} + func (me *ClusterMetaClient) Database(name string) *meta.DatabaseInfo { return me.cache.Database(name) } diff --git a/coordinator/meta_client_impl.go b/coordinator/meta_client_impl.go index 1576bfc..8b3c16d 100644 --- a/coordinator/meta_client_impl.go +++ b/coordinator/meta_client_impl.go @@ -134,6 +134,27 @@ func (me *MetaClientImpl) DeleteDataNode(id uint64) error { return nil } +func (me *MetaClientImpl) freezeDataNode(id uint64, freeze bool) error { + req := raftmeta.FreezeDataNodeReq{Id: id, Freeze: freeze} + var resp raftmeta.FreezeDataNodeResp + err := RequestAndParseResponse(me.Url(raftmeta.FREEZE_DATA_NODE_PATH), &req, &resp) + if err != nil { + return err + } + + if resp.RetCode != 0 { + return errors.New(resp.RetMsg) + } + + return nil +} +func (me *MetaClientImpl) FreezeDataNode(id uint64) error { + return me.freezeDataNode(id, true) +} +func (me *MetaClientImpl) UnfreezeDataNode(id uint64) error { + return me.freezeDataNode(id, false) +} + func (me *MetaClientImpl) AddShardOwner(shardID, nodeID uint64) error { req := raftmeta.AddShardOwnerReq{ ShardID: shardID, diff --git a/raftmeta/apply.go b/raftmeta/apply.go index 1fba100..2644e36 100644 --- a/raftmeta/apply.go +++ b/raftmeta/apply.go @@ -363,11 +363,21 @@ func (s *RaftNode) applyCommitted(proposal *internal.Proposal, index uint64) err } s.Logger.Info("checksum", zap.Uint64("index", req.Index), zap.String("checksum", s.lastChecksum.checksum)) - x.AssertTruef(s.lastChecksum.checksum == req.Checksum, "verify checksum fail, %s != %s, data=%+v", + x.AssertTruef(s.lastChecksum.checksum == req.Checksum, "verify checksum fail, local %s != %s, data=%+v", s.lastChecksum.checksum, req.Checksum, s.MetaCli.Data(), ) s.Logger.Info(fmt.Sprintf("verify checksum success. costs %s", time.Now().Sub(start))) + case internal.FreezeDataNode: + var req FreezeDataNodeReq + err := json.Unmarshal(proposal.Data, &req) + x.Check(err) + s.Logger.Debug(fmt.Sprintf("req %+v", req)) + if req.Freeze { + return s.MetaCli.FreezeDataNode(req.Id) + } else { + return s.MetaCli.UnfreezeDataNode(req.Id) + } default: return fmt.Errorf("Unkown msg type:%d", proposal.Type) } diff --git a/raftmeta/internal/message.go b/raftmeta/internal/message.go index e75bf3f..38dc813 100644 --- a/raftmeta/internal/message.go +++ b/raftmeta/internal/message.go @@ -36,6 +36,7 @@ const ( VerifyChecksumMsg = 29 AddShardOwner = 30 RemoveShardOwner = 31 + FreezeDataNode = 32 ) var MessageTypeName = map[int]string{ @@ -70,6 +71,7 @@ var MessageTypeName = map[int]string{ 29: "VerifyChecksumMsg", 30: "AddShardOwner", 31: "RemoveShardOwner", + 32: "FreezeDataNode", } type Proposal struct { diff --git a/raftmeta/meta_client.go b/raftmeta/meta_client.go index ab0cbeb..c4776df 100644 --- a/raftmeta/meta_client.go +++ b/raftmeta/meta_client.go @@ -1,10 +1,11 @@ package raftmeta import ( + "time" + imeta "github.com/angopher/chronus/services/meta" "github.com/influxdata/influxdb/services/meta" "github.com/influxdata/influxql" - "time" ) type MetaClient interface { @@ -20,6 +21,9 @@ type MetaClient interface { CreateUser(name, password string, admin bool) (meta.User, error) CreateDataNode(httpAddr, tcpAddr string) (*meta.NodeInfo, error) DeleteDataNode(id uint64) error + IsDataNodeFreezed(id uint64) bool + FreezeDataNode(id uint64) error + UnfreezeDataNode(id uint64) error Authenticate(username, password string) (meta.User, error) PruneShardGroups() error DeleteShardGroup(database, policy string, id uint64) error diff --git a/raftmeta/meta_service.go b/raftmeta/meta_service.go index 9c844ae..1553621 100644 --- a/raftmeta/meta_service.go +++ b/raftmeta/meta_service.go @@ -380,7 +380,7 @@ func (s *MetaService) DeleteDataNode(w http.ResponseWriter, r *http.Request) { resp.RetCode = 0 resp.RetMsg = "ok" - s.Logger.Info("DeleteDataNode ok", zap.Uint64("id", req.Id)) + s.Logger.Info(fmt.Sprintf("DeleteDataNode ok, id=%d", req.Id)) } type RetentionPolicySpec struct { @@ -1397,6 +1397,46 @@ func (s *MetaService) Data(w http.ResponseWriter, r *http.Request) { resp.RetMsg = "ok" } +type FreezeDataNodeReq struct { + Id uint64 + Freeze bool +} +type FreezeDataNodeResp struct { + CommonResp +} + +func (s *MetaService) FreezeDataNode(w http.ResponseWriter, r *http.Request) { + resp := new(FreezeDataNodeResp) + resp.RetCode = -1 + resp.RetMsg = "fail" + defer WriteResp(w, &resp) + + data, err := ioutil.ReadAll(r.Body) + if err != nil { + resp.RetMsg = err.Error() + s.Logger.Error("FreezeDataNode fail", zap.Error(err)) + return + } + + var req FreezeDataNodeReq + if err := json.Unmarshal(data, &req); err != nil { + resp.RetMsg = err.Error() + s.Logger.Error("FreezeDataNode fail", zap.Error(err)) + return + } + + err = s.ProposeAndWait(internal.FreezeDataNode, data, nil) + if err != nil { + resp.RetMsg = err.Error() + s.Logger.Error(fmt.Sprintf("FreezeDataNode fail, id=%d, freeze=%t", req.Id, req.Freeze), zap.Error(err)) + return + } + + resp.RetCode = 0 + resp.RetMsg = "ok" + s.Logger.Info(fmt.Sprintf("FreezeDataNode ok, id=%d, freeze=%t", req.Id, req.Freeze)) +} + type PingResp struct { CommonResp Index uint64 @@ -1430,6 +1470,7 @@ func initHttpHandler(s *MetaService) { http.HandleFunc(DROP_RETENTION_POLICY_PATH, s.DropRetentionPolicy) http.HandleFunc(DELETE_DATA_NODE_PATH, s.DeleteDataNode) + http.HandleFunc(FREEZE_DATA_NODE_PATH, s.FreezeDataNode) http.HandleFunc(CREATE_RETENTION_POLICY_PATH, s.CreateRetentionPolicy) http.HandleFunc(UPDATE_RETENTION_POLICY_PATH, s.UpdateRetentionPolicy) http.HandleFunc(CREATE_USER_PATH, s.CreateUser) diff --git a/raftmeta/path.go b/raftmeta/path.go index 1547417..ceb1dc3 100644 --- a/raftmeta/path.go +++ b/raftmeta/path.go @@ -7,6 +7,7 @@ const ( CREATE_DATA_NODE_PATH = "/create_data_node" DROP_RETENTION_POLICY_PATH = "/drop_retention_policy" DELETE_DATA_NODE_PATH = "/delete_data_node" + FREEZE_DATA_NODE_PATH = "/freeze_data_node" CREATE_RETENTION_POLICY_PATH = "/create_retention_policy" UPDATE_RETENTION_POLICY_PATH = "/update_retention_policy" CREATE_USER_PATH = "/create_user" diff --git a/services/controller/service.go b/services/controller/service.go index ba38c50..a702301 100644 --- a/services/controller/service.go +++ b/services/controller/service.go @@ -7,6 +7,7 @@ import ( "fmt" "io" "net" + "sort" "strings" "sync" "time" @@ -34,10 +35,14 @@ type Service struct { MetaClient interface { TruncateShardGroups(t time.Time) error DeleteDataNode(id uint64) error + IsDataNodeFreezed(id uint64) bool + FreezeDataNode(id uint64) error + UnfreezeDataNode(id uint64) error DataNodeByTCPHost(addr string) (*meta.NodeInfo, error) RemoveShardOwner(shardID, nodeID uint64) error DataNodes() ([]meta.NodeInfo, error) RetentionPolicy(database, name string) (*meta.RetentionPolicyInfo, error) + Databases() []meta.DatabaseInfo ShardOwner(shardID uint64) (database, policy string, sgi *meta.ShardGroupInfo) AddShardOwner(shardID, nodeID uint64) error @@ -154,9 +159,15 @@ func (s *Service) handleConn(conn net.Conn) error { case RequestShards: groupInfo, err := s.handleShards(conn) s.shardsResponse(conn, groupInfo, err) + case RequestNodeShards: + shards, err := s.handleNodeShards(conn) + s.nodeShardsResponse(conn, shards, err) case RequestShard: db, rp, info, err := s.handleShard(conn) s.shardResponse(conn, db, rp, info, err) + case RequestFreezeDataNode: + err = s.handleFreezeDataNode(conn) + s.freezeDataNodeResponse(conn, err) } return nil @@ -369,6 +380,59 @@ func (s *Service) removeDataNodeResponse(w io.Writer, e error) { s.writeResponse(w, ResponseRemoveDataNode, &resp) } +func (s *Service) handleFreezeDataNode(conn net.Conn) error { + var req FreezeDataNodeRequest + if err := s.readRequest(conn, &req); err != nil { + return err + } + + ni, err := s.MetaClient.DataNodeByTCPHost(req.DataNodeAddr) + if err != nil { + return err + } else if ni == nil { + return fmt.Errorf("not find data node by addr:%s", req.DataNodeAddr) + } + + if req.Freeze { + return s.MetaClient.FreezeDataNode(ni.ID) + } else { + return s.MetaClient.UnfreezeDataNode(ni.ID) + } +} + +func (s *Service) freezeDataNodeResponse(w io.Writer, e error) { + // Build response. + var resp FreezeDataNodeResponse + setError(&resp.CommonResp, e) + s.writeResponse(w, ResponseFreezeDataNode, &resp) +} + +func (s *Service) handleNodeShards(conn net.Conn) ([]uint64, error) { + var req GetNodeShardsRequest + if err := s.readRequest(conn, &req); err != nil { + return nil, err + } + if req.NodeID < 1 { + return nil, errors.New("Node ID should not be empty") + } + + shards := make([]uint64, 0) + dbs := s.MetaClient.Databases() + for _, db := range dbs { + infoList := db.ShardInfos() + for _, info := range infoList { + if info.OwnedBy(req.NodeID) { + shards = append(shards, info.ID) + } + } + } + sort.Slice(shards, func(i, j int) bool { + return shards[i] < shards[j] + }) + + return shards, nil +} + func (s *Service) handleShards(conn net.Conn) (*meta.RetentionPolicyInfo, error) { var req GetShardsRequest if err := s.readRequest(conn, &req); err != nil { @@ -420,6 +484,15 @@ func (s *Service) shardsResponse(w io.Writer, rp *meta.RetentionPolicyInfo, e er s.writeResponse(w, ResponseShards, &resp) } +func (s *Service) nodeShardsResponse(w io.Writer, shards []uint64, e error) { + // Build response. + var resp NodeShardsResponse + setError(&resp.CommonResp, e) + resp.Shards = shards + + s.writeResponse(w, ResponseNodeShards, &resp) +} + func (s *Service) handleShard(conn net.Conn) (string, string, *meta.ShardInfo, error) { var ( req GetShardRequest @@ -469,7 +542,7 @@ func (s *Service) handleShowDataNodes() ([]DataNode, error) { var dataNodes []DataNode for _, n := range nodes { - dataNodes = append(dataNodes, DataNode{ID: n.ID, TcpAddr: n.TCPHost, HttpAddr: n.Host}) + dataNodes = append(dataNodes, DataNode{ID: n.ID, TcpAddr: n.TCPHost, HttpAddr: n.Host, Freezed: s.MetaClient.IsDataNodeFreezed(n.ID)}) } return dataNodes, nil } @@ -543,6 +616,10 @@ type GetShardsRequest struct { Database, RetentionPolicy string } +type GetNodeShardsRequest struct { + NodeID uint64 +} + type GetShardRequest struct { ShardID uint64 } @@ -564,10 +641,19 @@ type RemoveDataNodeResponse struct { CommonResp } +type FreezeDataNodeRequest struct { + DataNodeAddr string `json:"data_node_addr"` + Freeze bool `json:"freeze"` +} +type FreezeDataNodeResponse struct { + CommonResp +} + type DataNode struct { ID uint64 `json:"id"` TcpAddr string `json:"tcp_addr"` HttpAddr string `json:"http_addr"` + Freezed bool `json:"freezed"` } type ShardGroup struct { @@ -593,6 +679,11 @@ type ShardsResponse struct { Groups []ShardGroup `json:"groups"` } +type NodeShardsResponse struct { + CommonResp + Shards []uint64 `json:"shards"` +} + type ShardResponse struct { CommonResp ID uint64 `json:"id"` @@ -621,6 +712,8 @@ const ( RequestShowDataNodes RequestShards RequestShard + RequestFreezeDataNode + RequestNodeShards ) type ResponseType byte @@ -636,4 +729,6 @@ const ( ResponseShowDataNodes ResponseShards ResponseShard + ResponseFreezeDataNode + ResponseNodeShards ) diff --git a/services/meta/client.go b/services/meta/client.go index 4352980..ecad42b 100644 --- a/services/meta/client.go +++ b/services/meta/client.go @@ -866,6 +866,50 @@ func createShardGroup(data *Data, database, policy string, timestamp time.Time) return sgi, nil } +// IsDataNodeFreezed returns whether the node has been freezed +func (c *Client) IsDataNodeFreezed(id uint64) bool { + c.mu.Lock() + defer c.mu.Unlock() + + return c.cacheData.IsFreezeDataNode(id) +} + +// FreezeDataNode freezes specific node for new shard's creation +func (c *Client) FreezeDataNode(id uint64) error { + c.mu.Lock() + defer c.mu.Unlock() + + data := c.cacheData.Clone() + + if err := data.FreezeDataNode(id); err != nil { + return err + } + + if err := c.commit(data); err != nil { + return err + } + + return nil +} + +// UnfreezeDataNode restores specific node for new shard's creation +func (c *Client) UnfreezeDataNode(id uint64) error { + c.mu.Lock() + defer c.mu.Unlock() + + data := c.cacheData.Clone() + + if err := data.UnfreezeDataNode(id); err != nil { + return err + } + + if err := c.commit(data); err != nil { + return err + } + + return nil +} + // DeleteShardGroup removes a shard group from a database and retention policy by id. func (c *Client) DeleteShardGroup(database, policy string, id uint64) error { c.mu.Lock() diff --git a/services/meta/data.go b/services/meta/data.go index 930d666..0a47eae 100644 --- a/services/meta/data.go +++ b/services/meta/data.go @@ -158,7 +158,7 @@ func (data *Data) DeleteDataNode(id uint64) error { for ri, rp := range d.RetentionPolicies { for sgi, sg := range rp.ShardGroups { var ( - nodeOwnerFreqs = make(map[int]int) + nodeOwnerFreqs = make(map[uint64]int) orphanedShards []meta.ShardInfo ) // Look through all shards in the shard group and @@ -174,7 +174,7 @@ func (data *Data) DeleteDataNode(id uint64) error { if owner.NodeID == id { nodeIdx = i } - nodeOwnerFreqs[int(owner.NodeID)]++ + nodeOwnerFreqs[owner.NodeID]++ } if nodeIdx > -1 { @@ -200,18 +200,19 @@ func (data *Data) DeleteDataNode(id uint64) error { // Reassign any orphaned shards. Delete the node we're // dropping from the list of potential new owners. - delete(nodeOwnerFreqs, int(id)) + delete(nodeOwnerFreqs, id) for _, orphan := range orphanedShards { - newOwnerID, err := newShardOwner(orphan, nodeOwnerFreqs) - if err != nil { - return err + newOwnerID := newShardOwner(nodeOwnerFreqs) + if newOwnerID == 0 { + return errors.New(fmt.Sprint("No node can be reassigned to ", orphan.ID)) } for si, s := range sg.Shards { if s.ID == orphan.ID { sg.Shards[si].Owners = append(sg.Shards[si].Owners, meta.ShardOwner{NodeID: newOwnerID}) data.Databases[di].RetentionPolicies[ri].ShardGroups[sgi].Shards = sg.Shards + nodeOwnerFreqs[newOwnerID]++ break } } @@ -245,26 +246,29 @@ func cloneNodes(src []meta.NodeInfo) []meta.NodeInfo { // that currently owns the fewest number of shards. If multiple nodes // own the same (fewest) number of shards, then one of those nodes // becomes the new shard owner. -func newShardOwner(s meta.ShardInfo, ownerFreqs map[int]int) (uint64, error) { - var ( - minId = -1 - minFreq int - ) - - for id, freq := range ownerFreqs { - if minId == -1 || freq < minFreq { - minId, minFreq = int(id), freq - } +// ATTENTION: This method should guarantee that the result is stable between +// different metad instances. +func newShardOwner(freqs map[uint64]int) uint64 { + if len(freqs) < 1 { + return 0 } - if minId < 0 { - return 0, fmt.Errorf("cannot reassign shard %d due to lack of data nodes", s.ID) + type item struct { + id uint64 + freq int + } + arr := make([]item, 0, len(freqs)) + for id, freq := range freqs { + arr = append(arr, item{id, freq}) } + sort.Slice(arr, func(i, j int) bool { + if arr[i].freq != arr[j].freq { + return arr[i].freq < arr[j].freq + } + return arr[i].id < arr[j].id + }) - // Update the shard owner frequencies and set the new owner on the - // shard. - ownerFreqs[minId]++ - return uint64(minId), nil + return arr[0].id } // MetaNode returns a node by id. @@ -354,10 +358,11 @@ func (data *Data) Clone() *Data { } type DataJson struct { - Data []byte - MetaNodes []meta.NodeInfo - DataNodes []meta.NodeInfo - MaxNodeID uint64 + Data []byte + MetaNodes []meta.NodeInfo + DataNodes []meta.NodeInfo + MaxNodeID uint64 + FreezedDataNodes []uint64 } func (data *Data) marshal() ([]byte, error) { @@ -365,6 +370,7 @@ func (data *Data) marshal() ([]byte, error) { js.MetaNodes = data.MetaNodes js.DataNodes = data.DataNodes js.MaxNodeID = data.MaxNodeID + js.FreezedDataNodes = data.FreezedDataNodes var err error js.Data, err = data.Data.MarshalBinary() if err != nil { @@ -385,6 +391,7 @@ func (data *Data) unmarshal(buf []byte) error { data.MetaNodes = js.MetaNodes data.DataNodes = js.DataNodes data.MaxNodeID = js.MaxNodeID + data.FreezedDataNodes = js.FreezedDataNodes return data.Data.UnmarshalBinary(js.Data) } diff --git a/services/meta/data_inner_test.go b/services/meta/data_inner_test.go new file mode 100644 index 0000000..f0b0a64 --- /dev/null +++ b/services/meta/data_inner_test.go @@ -0,0 +1,24 @@ +package meta + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestNewShardOwner(t *testing.T) { + assert.Equal(t, uint64(0), newShardOwner(nil)) + assert.Equal(t, uint64(0), newShardOwner(map[uint64]int{})) + assert.Equal(t, uint64(1), newShardOwner(map[uint64]int{ + 5: 1, + 7: 1, + 1: 1, + 3: 1, + })) + assert.Equal(t, uint64(5), newShardOwner(map[uint64]int{ + 5: 1, + 7: 2, + 1: 2, + 3: 2, + })) +} diff --git a/services/meta/data_test.go b/services/meta/data_test.go index 4d3060c..c5bd8b4 100644 --- a/services/meta/data_test.go +++ b/services/meta/data_test.go @@ -261,7 +261,9 @@ func TestClone(t *testing.T) { data2 := data1.Clone() data1.FreezeDataNode(id1) data2.FreezeDataNode(id2) + data3 := data1.Clone() assert.Equal(t, 1, len(data1.FreezedDataNodes)) assert.Equal(t, 1, len(data2.FreezedDataNodes)) assert.NotEqual(t, data1.FreezedDataNodes, data2.FreezedDataNodes) + assert.Equal(t, data3.FreezedDataNodes, data1.FreezedDataNodes) } From f9d578408383ffc1c6aac8778f060e87861a8373 Mon Sep 17 00:00:00 2001 From: Jason Joo Date: Mon, 12 Oct 2020 13:12:43 +0800 Subject: [PATCH 21/46] Upgrade to latest stable influxdb * influxdb to 1.8.3 * badger to v2.2007.2 for better performance * etcd to 3.3.25 Signed-off-by: Jason Joo --- cmd/influxd/run/server.go | 2 +- cmd/metad-ctl/cmds/update.go | 4 - cmd/metad/main.go | 4 +- go.mod | 42 +- go.sum | 713 +++++++++++++++++++------------ raftmeta/badger_logger_bridge.go | 6 +- raftmeta/internal/message.go | 2 +- raftmeta/linearizable_read.go | 3 +- raftmeta/node.go | 15 +- raftmeta/raft_logger_bridge.go | 2 +- raftmeta/raft_test.go | 4 +- raftmeta/transport.go | 4 +- 12 files changed, 484 insertions(+), 317 deletions(-) diff --git a/cmd/influxd/run/server.go b/cmd/influxd/run/server.go index 75ed1b1..481daeb 100644 --- a/cmd/influxd/run/server.go +++ b/cmd/influxd/run/server.go @@ -30,9 +30,9 @@ import ( "github.com/influxdata/influxdb/services/storage" "github.com/influxdata/influxdb/services/subscriber" "github.com/influxdata/influxdb/services/udp" + "github.com/influxdata/influxdb/storage/reads" "github.com/influxdata/influxdb/tcp" "github.com/influxdata/influxdb/tsdb" - "github.com/influxdata/platform/storage/reads" client "github.com/influxdata/usage-client/v1" "go.uber.org/zap" diff --git a/cmd/metad-ctl/cmds/update.go b/cmd/metad-ctl/cmds/update.go index 4765057..1a3ef17 100644 --- a/cmd/metad-ctl/cmds/update.go +++ b/cmd/metad-ctl/cmds/update.go @@ -4,7 +4,6 @@ import ( "errors" "fmt" "net/url" - "os" "github.com/angopher/chronus/cmd/metad-ctl/util" "github.com/fatih/color" @@ -72,7 +71,6 @@ func clusterAdd(ctx *cli.Context) (err error) { return nil ERR: - fmt.Fprint(os.Stderr, "ERR: ", err.Error(), "\n\n") return err } @@ -107,7 +105,6 @@ func clusterUpdate(ctx *cli.Context) (err error) { return nil ERR: - fmt.Fprint(os.Stderr, "ERR: ", err.Error(), "\n\n") return err } @@ -142,6 +139,5 @@ func clusterRemove(ctx *cli.Context) (err error) { return nil ERR: - fmt.Fprint(os.Stderr, "ERR: ", err.Error(), "\n\n") return err } diff --git a/cmd/metad/main.go b/cmd/metad/main.go index cd04f51..b04ebd0 100644 --- a/cmd/metad/main.go +++ b/cmd/metad/main.go @@ -10,9 +10,8 @@ import ( "github.com/angopher/chronus/raftmeta" imeta "github.com/angopher/chronus/services/meta" "github.com/angopher/chronus/x" - "github.com/coreos/etcd/raft/raftpb" - "github.com/dgraph-io/badger" "github.com/influxdata/influxdb/services/meta" + "go.etcd.io/etcd/raft/raftpb" "go.uber.org/zap" ) @@ -48,7 +47,6 @@ func main() { fmt.Fprintln(os.Stderr, "Error to initialize logging", err) return } - badger.SetLogger(raftmeta.NewBadgerLoggerBridge(log)) suger := log.Sugar() suger.Debug("config: %+v", config) diff --git a/go.mod b/go.mod index 5a13f9d..9a2136c 100644 --- a/go.mod +++ b/go.mod @@ -1,44 +1,34 @@ module github.com/angopher/chronus require ( - collectd.org v0.3.0 // indirect - github.com/AndreasBriese/bbloom v0.0.0-20180913140656-343706a395b7 // indirect github.com/BurntSushi/toml v0.3.1 - github.com/bmizerany/pat v0.0.0-20170815010413-6226ea591a40 // indirect - github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd // indirect - github.com/coreos/etcd v3.3.12+incompatible - github.com/dgraph-io/badger v1.5.5-0.20181126210712-49a49e321746 - github.com/dgraph-io/dgo v0.0.0-20190201214300-d5a1729ba705 // indirect - github.com/dgraph-io/dgraph v1.0.11 - github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f // indirect + github.com/dgraph-io/badger/v2 v2.2007.2 + github.com/dgraph-io/dgraph v1.2.7 github.com/fatih/color v1.7.0 - github.com/gogo/protobuf v1.2.1 - github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db - github.com/google/go-cmp v0.2.0 - github.com/influxdata/influxdb v1.7.4 - github.com/influxdata/influxql v0.0.0-20180925231337-1cbfca8e56b6 - github.com/influxdata/platform v0.0.0-20190117200541-d500d3cf5589 - github.com/influxdata/roaring v0.4.12 // indirect + github.com/gogo/protobuf v1.3.1 + github.com/golang/snappy v0.0.1 + github.com/google/go-cmp v0.4.0 + github.com/influxdata/influxdb v1.8.3 + github.com/influxdata/influxql v1.1.1-0.20200828144457-65d3ef77d385 github.com/influxdata/usage-client v0.0.0-20160829180054-6d3895376368 github.com/jsternberg/zap-logfmt v1.2.0 github.com/klauspost/compress v1.4.1 // indirect github.com/klauspost/cpuid v1.2.0 // indirect github.com/klauspost/pgzip v1.2.1 // indirect - github.com/pkg/errors v0.8.1 - github.com/pkg/profile v1.2.1 // indirect + github.com/pkg/errors v0.9.1 github.com/stretchr/testify v1.5.1 // test github.com/urfave/cli/v2 v2.2.0 + github.com/willf/bitset v1.1.9 // indirect github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca // indirect - go.opencensus.io v0.19.0 // indirect + go.etcd.io/etcd v3.3.25+incompatible go.uber.org/zap v1.15.0 - golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529 - golang.org/x/net v0.0.0-20190620200207-3b0461eec859 - golang.org/x/text v0.3.0 - golang.org/x/time v0.0.0-20181108054448-85acf8d2951c - gopkg.in/fatih/pool.v2 v2.0.0 + golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 + golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553 + golang.org/x/text v0.3.2 + golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 gopkg.in/natefinch/lumberjack.v2 v2.0.0 - labix.org/v2/mgo v0.0.0-20140701140051-000000000287 // indirect - launchpad.net/gocheck v0.0.0-20140225173054-000000000087 // indirect ) +replace github.com/coreos/etcd v3.3.25+incompatible => go.etcd.io/etcd v3.3.25+incompatible + go 1.13 diff --git a/go.sum b/go.sum index 88e341e..0202d65 100644 --- a/go.sum +++ b/go.sum @@ -1,246 +1,282 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.43.0/go.mod h1:BOSR3VbTLkk6FDC/TcffxP4NF/FFBGA5ku+jvKOP7pg= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.51.0 h1:PvKAVQWCtlGUSlZkGW3QLelKaWq7KYv/MW1EboG8bfM= +cloud.google.com/go v0.51.0/go.mod h1:hWtGJ6gnXH+KgDv+V0zFGDvpi07n3z8ZNj3T1RW0Gcw= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0 h1:sAbMqjY1PEQKZBWfbu6Y6bsupJ9c4QdHnzg/VvYTLcE= +cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/bigtable v1.2.0 h1:F4cCmA4nuV84V5zYQ3MKY+M1Cw1avHDuf3S/LcZPA9c= +cloud.google.com/go/bigtable v1.2.0/go.mod h1:JcVAOl45lrTmQfLj7T6TxyMzIN/3FGGcFm+2xVAli2o= +cloud.google.com/go/datastore v1.0.0 h1:Kt+gOPPp2LEPWp8CSfxhsM8ik9CcyE/gYu+0r+RnZvM= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/pubsub v1.1.0 h1:9/vpR43S4aJaROxqQHQ3nH9lfyKKV0dC3vOmnw8ebQQ= +cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0 h1:RPUcBvDeYgQFMfQu1eBMq6piD1SXmLH+vK3qjewZPus= +cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= collectd.org v0.3.0 h1:iNBHGw1VvPJxH2B6RiFWFZ+vsjo1lCdRszBeOuwGi00= collectd.org v0.3.0/go.mod h1:A/8DzQBkF6abtvrT2j/AU/4tiBgJWYyh0y/oB/4MlWE= -git.apache.org/thrift.git v0.0.0-20181218151757-9b75e4fe745a/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg= -github.com/AndreasBriese/bbloom v0.0.0-20180913140656-343706a395b7 h1:PqzgE6kAMi81xWQA2QIVxjWkFHptGgC547vchpUbtFo= -github.com/AndreasBriese/bbloom v0.0.0-20180913140656-343706a395b7/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= -github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= +contrib.go.opencensus.io/exporter/jaeger v0.1.0 h1:WNc9HbA38xEQmsI40Tjd/MNU/g8byN2Of7lwIjv0Jdc= +contrib.go.opencensus.io/exporter/jaeger v0.1.0/go.mod h1:VYianECmuFPwU37O699Vc1GOcy+y8kOsfaxHRImmjbA= +contrib.go.opencensus.io/exporter/prometheus v0.1.0 h1:SByaIoWwNgMdPSgl5sMqM2KDE5H/ukPWBRo314xiDvg= +contrib.go.opencensus.io/exporter/prometheus v0.1.0/go.mod h1:cGFniUXGZlKRjzOyuZJ6mgB+PgBcCIa79kEKR8YCW+A= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/99designs/gqlgen v0.10.1/go.mod h1:IviubpnyI4gbBcj8IcxSSc/Q/+af5riwCmJmwF0uaPE= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/DataDog/datadog-go v0.0.0-20180822151419-281ae9f2d895/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= -github.com/Jeffail/gabs v1.1.1/go.mod h1:6xMvQMK4k33lb7GUUpaAPh6nKMmemQeg5d4gn7/bOXc= -github.com/Masterminds/semver v1.4.2 h1:WBLTQ37jOCzSLtXNdoo8bNM8876KhNqOKvrlGITgsTc= -github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= -github.com/Masterminds/sprig v2.16.0+incompatible h1:QZbMUPxRQ50EKAq3LFMnxddMu88/EUUG3qmxwtDmPsY= -github.com/Masterminds/sprig v2.16.0+incompatible/go.mod h1:y6hNFY5UBTIWBxnzTeuNhlNS5hqE0NB0E6fgfo2Br3o= -github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA= -github.com/NYTimes/gziphandler v1.0.1/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/DATA-DOG/go-sqlmock v1.3.2/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= +github.com/DATA-DOG/go-sqlmock v1.3.3 h1:CWUqKXe0s8A2z6qCgkP4Kru7wC11YoAnoupUKFDnH08= +github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= +github.com/DataDog/datadog-go v0.0.0-20190425163447-40bafcb5f6c1 h1:fSu93OUqfEkoQJBkTsxFB1e0oESqabS45iRX880e7Xw= +github.com/DataDog/datadog-go v0.0.0-20190425163447-40bafcb5f6c1/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= +github.com/DataDog/opencensus-go-exporter-datadog v0.0.0-20190503082300-0f32ad59ab08 h1:5btKvK+N+FpW0EEgvxq7LWcUEwIRLsL4IwIo0u+Qlhs= +github.com/DataDog/opencensus-go-exporter-datadog v0.0.0-20190503082300-0f32ad59ab08/go.mod h1:gMGUEe16aZh0QN941HgDjwrdjU4iTthPoz2/AtDRADE= +github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= +github.com/DataDog/zstd v1.4.5 h1:EndNeuB0l9syBZhut0wns3gV1hL8zX8LIu6ZiVHWLIQ= +github.com/DataDog/zstd v1.4.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5/go.mod h1:lmUJ/7eu/Q8D7ML55dXQrVaamCz2vxCfdQBasLZfHKk= github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= -github.com/RoaringBitmap/roaring v0.4.16 h1:NholfewybRLOwACgfqfzn/N5xa6keKNs4fP00t0cwLo= -github.com/RoaringBitmap/roaring v0.4.16/go.mod h1:8khRDP4HmeXns4xIj9oGrKSz7XTQiJx2zgh7AcNke4w= -github.com/SAP/go-hdb v0.13.1/go.mod h1:etBT+FAi1t5k3K3tf5vQTnosgYmhDkRi8jEnQqCnxF0= -github.com/SermoDigital/jose v0.9.1/go.mod h1:ARgCUhI1MHQH+ONky/PAtmVHQrP5JlGY0F3poXOp/fA= -github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7/go.mod h1:6zEj6s6u/ghQa61ZWa/C2Aw3RkjiTBOix7dkqa1VLIs= -github.com/alecthomas/kingpin v2.2.6+incompatible h1:5svnBTFgJjZvGKyYBtMB0+m5wvrbUHiqye8wRJMlnYI= -github.com/alecthomas/kingpin v2.2.6+incompatible/go.mod h1:59OFYbFVLKQKq+mqrL6Rw5bR0c3ACQaawgXx0QYndlE= +github.com/OneOfOne/xxhash v1.2.5 h1:zl/OfRA6nftbBK9qTohYBJ5xvw6C/oNKizR7cZGl3cI= +github.com/OneOfOne/xxhash v1.2.5/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q= +github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= +github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= +github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM= +github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc h1:cAKDfWh5VpdgMhJosfJnn5/FoN2SRZ4p7fJNX58YPaU= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf h1:qet1QNfXsQxTZqLG4oE62mJzwPIB8+Tee4RNCL9ulrY= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883 h1:bvNMNQO63//z+xNgfBlViaCIJKLlCJ6/fmUseuG0wVQ= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= -github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= -github.com/aokoli/goutils v1.0.1 h1:7fpzNGoJ3VA8qcrm++XEE1QUe0mIwNeLa02Nwq7RDkg= -github.com/aokoli/goutils v1.0.1/go.mod h1:SijmP0QR8LtwsmDs8Yii5Z/S4trXFGFC2oO5g9DP+DQ= -github.com/apache/arrow/go/arrow v0.0.0-20181031164735-a56c009257a7/go.mod h1:GjvccvtI06FGFvRU1In/maF7tKp3h7GBV9Sexo5rNPM= -github.com/apache/arrow/go/arrow v0.0.0-20181217213538-e9ed591db9cb h1:p6xQwsjxRtuIrUDjGAFuro04BO0GNJ9V2troYRY8kmQ= -github.com/apache/arrow/go/arrow v0.0.0-20181217213538-e9ed591db9cb/go.mod h1:GjvccvtI06FGFvRU1In/maF7tKp3h7GBV9Sexo5rNPM= -github.com/apex/log v1.1.0 h1:J5rld6WVFi6NxA6m8GJ1LJqu3+GiTFIt3mYv27gdQWI= -github.com/apex/log v1.1.0/go.mod h1:yA770aXIDQrhVOIGurT/pVdfCpSq1GQV/auzMN5fzvY= -github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da h1:8GUt8eRujhVEGZFFEjBj46YV4rDjvGrNxb0KMWYkL2I= -github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= -github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= -github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= -github.com/aws/aws-sdk-go v1.15.59/go.mod h1:E3/ieXAlvM0XWO57iftYVDLLvQ824smPP3ATZkfNZeM= -github.com/aws/aws-sdk-go v1.15.64 h1:xI5HhxebTF+jVqVOraUDqI3kr24n+yTvslwZCo3OhGA= -github.com/aws/aws-sdk-go v1.15.64/go.mod h1:E3/ieXAlvM0XWO57iftYVDLLvQ824smPP3ATZkfNZeM= -github.com/benbjohnson/tmpl v1.0.0 h1:T5QPGJD0W6JJxyEEAlVnX3co/IkUrfHen1/42nlgAHo= -github.com/benbjohnson/tmpl v1.0.0/go.mod h1:igT620JFIi44B6awvU9IsDhR77IXWtFigTLil/RPdps= +github.com/apache/arrow/go/arrow v0.0.0-20191024131854-af6fa24be0db h1:nxAtV4VajJDhKysp2kdcJZsq8Ss1xSA0vZTkVHHJd0E= +github.com/apache/arrow/go/arrow v0.0.0-20191024131854-af6fa24be0db/go.mod h1:VTxUBvSJ3s3eHAg65PNgrsn5BtqCRPdmyXh6rAfdxN0= +github.com/apache/thrift v0.12.0 h1:pODnxUFNcjP9UTLZGTdeh+j16A8lJbRvD3rOtrk/7bs= +github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= -github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932/go.mod h1:NOuUCSz6Q9T7+igc/hlvDOUdtWKryOrtFyIVABv/p7k= -github.com/blakesmith/ar v0.0.0-20150311145944-8bd4349a67f2 h1:oMCHnXa6CCCafdPDbMh/lWRhRByN0VFLvv+g+ayx1SI= -github.com/blakesmith/ar v0.0.0-20150311145944-8bd4349a67f2/go.mod h1:PkYb9DJNAwrSvRx5DYA+gUcOIgTGVMNkfSCbZM8cWpI= -github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= +github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/blevesearch/bleve v0.0.0-20181114232033-e1f5e6cdcd76/go.mod h1:Y2lmIkzV6mcNfAnAdOd+ZxHkHchhBfU/xroGIp61wfw= +github.com/blevesearch/go-porterstemmer v1.0.2/go.mod h1:haWQqFT3RdOGz7PJuM3or/pWNJS1pKkoZJWCkWu0DVA= +github.com/blevesearch/segment v0.0.0-20160915185041-762005e7a34f/go.mod h1:IInt5XRvpiGE09KOk9mmCMLjHhydIhNPKPPFLFBB7L8= +github.com/blevesearch/snowballstem v0.0.0-20180110192139-26b06a2c243d/go.mod h1:cdytUvf6FKWA9NpXJihYdZq8TN2AiQ5HOS0UZUz0C9g= github.com/bmizerany/pat v0.0.0-20170815010413-6226ea591a40 h1:y4B3+GPxKlrigF1ha5FFErxK+sr6sWxQovRMzwMhejo= github.com/bmizerany/pat v0.0.0-20170815010413-6226ea591a40/go.mod h1:8rLXio+WjiTceGBHIoTvn60HIbs7Hm7bcHjyrSqYB9c= github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4= github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= -github.com/bouk/httprouter v0.0.0-20160817010721-ee8b3818a7f5/go.mod h1:CDReaxg1cmLrtcasZy43l4EYPAknXLiQSrb7tLw5zXM= +github.com/c-bata/go-prompt v0.2.2 h1:uyKRz6Z6DUyj49QVijyM339UJV9yhbr70gESwbNU3e0= github.com/c-bata/go-prompt v0.2.2/go.mod h1:VzqtzE2ksDBcdln8G7mk2RX9QyGjH+OVqOCSiVIqS34= -github.com/caarlos0/ctrlc v1.0.0 h1:2DtF8GSIcajgffDFJzyG15vO+1PuBWOMUdFut7NnXhw= -github.com/caarlos0/ctrlc v1.0.0/go.mod h1:CdXpj4rmq0q/1Eb44M9zi2nKB0QraNKuRGYGrrHhcQw= -github.com/campoy/unique v0.0.0-20180121183637-88950e537e7e h1:V9a67dfYqPLAvzk5hMQOXYJlZ4SLIXgyKIE+ZiHzgGQ= -github.com/campoy/unique v0.0.0-20180121183637-88950e537e7e/go.mod h1:9IOqJGCPMSc6E5ydlp5NIonxObaeu/Iub/X03EKPVYo= -github.com/cenkalti/backoff v2.0.0+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= +github.com/cenkalti/backoff v2.1.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= -github.com/circonus-labs/circonus-gometrics v2.2.5+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= -github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd h1:qMd81Ts1T2OTKmB4acZcyKaMtRnY5Y44NuXGX2GFJ1w= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= -github.com/containerd/continuity v0.0.0-20181027224239-bea7585dbfac/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= -github.com/coreos/bbolt v1.3.1-coreos.6 h1:uTXKg9gY70s9jMAKdfljFQcuh4e/BXOM+V+d00KFj3A= -github.com/coreos/bbolt v1.3.1-coreos.6/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= -github.com/coreos/etcd v3.3.12+incompatible h1:pAWNwdf7QiT1zfaWyqCtNZQWCLByQyA3JrSQyuYAqnQ= -github.com/coreos/etcd v3.3.12+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/containerd/continuity v0.0.0-20181203112020-004b46473808/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= +github.com/coreos/etcd v3.3.10+incompatible h1:jFneRYjIvLMLhDLCzuTuU4rSJUjRplcJQ7pD7MnhC04= +github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/cpuguy83/go-md2man v1.0.10 h1:BSKMNlYxDvnunlTymqtgONjNnaRV1sTpcovwwjF22jk= +github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/d4l3k/messagediff v1.2.1/go.mod h1:Oozbb1TVXFac9FtSIxHBMnBCq2qeH/2KkEQxENCrlLo= +github.com/dave/jennifer v1.2.0/go.mod h1:fIb+770HOpJ2fmN9EPPKOqm1vMGhB+TwXKMZhrIygKg= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/denisenkom/go-mssqldb v0.0.0-20181014144952-4e0d7dc8888f/go.mod h1:xN/JuLBIz4bjkxNmByTiV1IbhfnYb6oo99phBn4Eqhc= -github.com/dgraph-io/badger v1.5.5-0.20181126210712-49a49e321746 h1:cd1s0THXu7KWnykg7V07VSiTde/g38d8PJBTXBaq0mk= -github.com/dgraph-io/badger v1.5.5-0.20181126210712-49a49e321746/go.mod h1:VZxzAIRPHRVNRKRo6AXrX9BJegn6il06VMTZVJYCIjQ= -github.com/dgraph-io/dgo v0.0.0-20190201214300-d5a1729ba705 h1:f2knGDNK8Q5y9D8rKdACl7qviC4jonzug90y4TNaRxU= -github.com/dgraph-io/dgo v0.0.0-20190201214300-d5a1729ba705/go.mod h1:/3OtwVvGnHiyXL9bWTjZaHQk4cp8rakX6jVBFoykWaA= -github.com/dgraph-io/dgraph v1.0.11 h1:An8I4+pG7e6n3evXXZsB4E6f3PUc9+0ynBRVCivb8f4= -github.com/dgraph-io/dgraph v1.0.11/go.mod h1:Qx1vy3ocQ5/Q66iKt6KcFADD3VUncMHuVaNF0sgr9js= +github.com/dgraph-io/badger/v2 v2.2007.2-0.20200827131741-d5a25b83fbf4 h1:DUDFTVgqZysKplH39/ya0aI4+zGm91L9QttXgITT2YE= +github.com/dgraph-io/badger/v2 v2.2007.2-0.20200827131741-d5a25b83fbf4/go.mod h1:26P/7fbL4kUZVEVKLAKXkBXKOydDmM2p1e+NhhnBCAE= +github.com/dgraph-io/badger/v2 v2.2007.2 h1:EjjK0KqwaFMlPin1ajhP943VPENHJdEz1KLIegjaI3k= +github.com/dgraph-io/badger/v2 v2.2007.2/go.mod h1:26P/7fbL4kUZVEVKLAKXkBXKOydDmM2p1e+NhhnBCAE= +github.com/dgraph-io/dgo/v2 v2.1.1-0.20191127085444-c7a02678e8a6 h1:5leDFqGys055YO3TbghBhk/QdRPEwyLPdgsSJfiR20I= +github.com/dgraph-io/dgo/v2 v2.1.1-0.20191127085444-c7a02678e8a6/go.mod h1:LJCkLxm5fUMcU+yb8gHFjHt7ChgNuz3YnQQ6MQkmscI= +github.com/dgraph-io/dgraph v1.2.7 h1:umWwQ1wPpRJqhOx52IFJ4ixCU5//anPK6VbT+N67L5Y= +github.com/dgraph-io/dgraph v1.2.7/go.mod h1:q0S+BGN+D8UUDHbK+oECqCU27NIzWwNLJ5vDw4+oAhs= +github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= +github.com/dgraph-io/ristretto v0.0.4-0.20200904131139-4dec2770af66 h1:ectpJv2tGhTudyk0JhqE/53o/ObH30u5yt/yThsAn3I= +github.com/dgraph-io/ristretto v0.0.4-0.20200904131139-4dec2770af66/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-bitstream v0.0.0-20180413035011-3522498ce2c8 h1:akOQj8IVgoeFfBTzGOEQakCYshWD6RNo1M5pivFXt70= github.com/dgryski/go-bitstream v0.0.0-20180413035011-3522498ce2c8/go.mod h1:VMaSuZ+SZcx/wljOQKvp5srsbCiKDEb6K2wC4+PiBmQ= -github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f h1:dDxpBYafY/GYpcl+LS4Bn3ziLPuEdGRkRjYAbSlWxSA= -github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= -github.com/docker/distribution v2.6.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/docker v0.0.0-20180422163414-57142e89befe/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= +github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 h1:fAjc9m62+UWV/WAFKLNi6ZS0675eEUC9y3AlwSbQu1Y= +github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= +github.com/dgryski/go-groupvarint v0.0.0-20190318181831-5ce5df8ca4e1/go.mod h1:MlkUQveSLEDbIgq2r1e++tSf0zfzU9mQpa9Qkczl+9Y= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/duosecurity/duo_api_golang v0.0.0-20181024123116-92fea9203dbc/go.mod h1:UqXY1lYT/ERa4OEAywUqdok1T4RCRdArkhic1Opuavo= +github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/elazarl/go-bindata-assetfs v1.0.0/go.mod h1:v+YaWX3bdea5J/mo8dSETolEo7R71Vk1u8bnjau5yw4= -github.com/emirpasic/gods v1.9.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o= +github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= +github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= +github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= +github.com/eclipse/paho.mqtt.golang v1.2.0 h1:1F8mhG9+aO5/xpdtFkW4SxOJB67ukuDC3t2y2qayIX0= +github.com/eclipse/paho.mqtt.golang v1.2.0/go.mod h1:H9keYFcgq3Qr5OUJm/JZI/i6U7joQ8SYLhZwfeOo6Ts= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= -github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= +github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/getkin/kin-openapi v0.1.0/go.mod h1:+0ZtELZf+SlWH8ZdA/IeFb3L/PKOKJx8eGxAlUZ/sOU= -github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= +github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32/go.mod h1:GIjDIg/heH5DOkXY3YJ/wNhfHsQHoXGjl8G8amsYQ1I= github.com/glycerine/go-unsnap-stream v0.0.0-20180323001048-9f0cb55181dd h1:r04MMPyLHj/QwZuMJ5+7tJcBr1AQjpiAK/rZWRrQT7o= github.com/glycerine/go-unsnap-stream v0.0.0-20180323001048-9f0cb55181dd/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE= -github.com/glycerine/goconvey v0.0.0-20180728074245-46e3a41ad493 h1:OTanQnFt0bi5iLFSdbEVA/idR6Q2WhCm+deb7ir2CcM= -github.com/glycerine/goconvey v0.0.0-20180728074245-46e3a41ad493/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24= -github.com/go-ldap/ldap v2.5.1+incompatible/go.mod h1:qfd9rJvER9Q0/D/Sqn1DfHRoBp40uXYvFoEVrNEPqRc= -github.com/go-sql-driver/mysql v1.4.0 h1:7LxgVwFb2hIQtMm87NdgAVfXjnt4OePseqT1tKx+opk= -github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= -github.com/go-test/deep v1.0.1/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= -github.com/gocql/gocql v0.0.0-20181117210152-33c0e89ca93a/go.mod h1:4Fw1eo5iaEhDUs8XyuhSVCVy52Jq3L+/3GJgYkwc+/0= +github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31 h1:gclg6gY70GLy3PbkQ1AERPfmLMMagS60DKF78eWwLn8= +github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24= +github.com/go-chi/chi v3.3.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-ini/ini v1.39.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-sql-driver/mysql v0.0.0-20190330032241-c0f6b444ad8f/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= +github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA= +github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gofrs/uuid v3.3.0+incompatible h1:8K4tyRfvU1CYPgJsveYFQMhpFd/wXNM7iK6rR7UHz84= +github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/gogo/protobuf v1.0.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE= -github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= -github.com/golang/gddo v0.0.0-20181116215533-9bd4a3295021/go.mod h1:xEhNfoBDX1hzLm2Nf80qUvZ2sVwoMZ8d6IE2SrsQfh4= +github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= +github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= +github.com/golang/geo v0.0.0-20170810003146-31fb0106dc4a/go.mod h1:vgWZ7cu0fq0KY3PpEHsocXOWJpRtkcbKemU4IUw0M60= +github.com/golang/geo v0.0.0-20190916061304-5b978397cfec h1:lJwO/92dFXWeXOZdoGXgptLmNLwynMSHUmU6besqtiw= +github.com/golang/geo v0.0.0-20190916061304-5b978397cfec/go.mod h1:QZ0nwyI2jOfgRAoBvP+ab5aRr7c9x7lhGEJrKvBwjWI= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E= +github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7 h1:5ZkaAPbicIKTF2I64qf5Fh8Aa83Q/dnOafMYV0OMwjA= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/snappy v0.0.0-20170215233205-553a64147049/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.5 h1:F768QJ1E9tib+q5Sc8MkdJi1RxLTbRcTf8LJV56aRls= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db h1:woRePGFeVFfLKN/pOkfl+p/TAqKOfFu+7KPlMVpok/w= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/btree v0.0.0-20180124185431-e89373fe6b4a/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c h1:964Od4U6p2jUkFxvCydnIczKteheJEzHRToSGK3Bnlw= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/codesearch v1.0.0/go.mod h1:qCnXDFnak/trCmLaE50kgPte3AX9jSeruZexWEOivi0= +github.com/google/flatbuffers v1.11.0 h1:O7CEyB8Cb3/DmtxODGtLHcEvpr81Jm5qLg/hsHnxA2A= +github.com/google/flatbuffers v1.11.0/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY= -github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= -github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= -github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/uuid v1.0.0 h1:b4Gk+7WdP/d3HZH8EJsZpvV7EtDOgaZLtnaNGIu1adA= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/goreleaser/goreleaser v0.94.0 h1:2CFMxMTLODjYfNOx2sADNzpgCwH9ltMqvQYtj+ntK1Q= -github.com/goreleaser/goreleaser v0.94.0/go.mod h1:OjbYR2NhOI6AEUWCowMSBzo9nP1aRif3sYtx+rhp+Zo= -github.com/goreleaser/nfpm v0.9.7 h1:h8RQMDztu6cW7b0/s4PGbdeMYykAbJG0UMXaWG5uBMI= -github.com/goreleaser/nfpm v0.9.7/go.mod h1:F2yzin6cBAL9gb+mSiReuXdsfTrOQwDMsuSpULof+y4= -github.com/gotestyourself/gotestyourself v2.2.0+incompatible/go.mod h1:zZKM6oeNM8k+FRljX1mnzVYeS8wiGgQyvST1/GafPbY= -github.com/grpc-ecosystem/grpc-gateway v1.6.2/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= -github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed/go.mod h1:tMWxXQ9wFIaZeTI9F+hmhFiGpFmhOHzyShyFUhRm0H4= -github.com/hashicorp/consul v1.4.0/go.mod h1:mFrjN1mfidgJfYP1xrJCF+AfRhr6Eaqhb2+sfyn/OOI= -github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= -github.com/hashicorp/go-hclog v0.0.0-20180709165350-ff2cf002a8dd/go.mod h1:9bjs9uLqI8l75knNv3lV1kA55veR+WUPSiKIWcQHudI= -github.com/hashicorp/go-hclog v0.0.0-20181001195459-61d530d6c27f/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= -github.com/hashicorp/go-immutable-radix v1.0.0 h1:AKDB1HM5PWEA7i4nhcpwOrO2byshxBjXVn/J/3+z5/0= -github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= -github.com/hashicorp/go-memdb v0.0.0-20181108192425-032f93b25bec/go.mod h1:kbfItVoBJwCfKXDXN4YoAXjxcFVZ7MRrJzyTX6H4giE= -github.com/hashicorp/go-msgpack v0.0.0-20150518234257-fa3f63826f7c h1:BTAbnbegUIMB6xmQCwWE8yRzbA4XSpnZY5hvRJC188I= -github.com/hashicorp/go-msgpack v0.0.0-20150518234257-fa3f63826f7c/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= -github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= -github.com/hashicorp/go-plugin v0.0.0-20181030172320-54b6ff97d818/go.mod h1:Ft7ju2vWzhO0ETMKUVo12XmXmII6eSUS4rsPTkY/siA= -github.com/hashicorp/go-retryablehttp v0.5.0/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= -github.com/hashicorp/go-rootcerts v0.0.0-20160503143440-6bb64b370b90/go.mod h1:o4zcYY1e0GEZI6eSEr+43QDYmuGglw1qSO6qdHUHCgg= -github.com/hashicorp/go-sockaddr v0.0.0-20180320115054-6d291a969b86/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= -github.com/hashicorp/go-uuid v1.0.0 h1:RS8zrF7PhGwyNPOtxSClXXj9HA8feRnJzgnI1RJCSnM= -github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go-version v1.0.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/gorilla/context v0.0.0-20160226214623-1ea25387ff6f/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= +github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= +github.com/gorilla/mux v1.6.1/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/gorilla/websocket v1.2.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v1.4.1/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= github.com/hashicorp/golang-lru v0.5.0 h1:CL2msUPvZTLb5O648aiLNJw3hnBxN2+1Jq8rCOH9wdo= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/hashicorp/memberlist v0.1.0/go.mod h1:ncdBp14cuox2iFOq3kDiquKU6fqsTBc3W6JvZwjxxsE= -github.com/hashicorp/raft v1.0.0 h1:htBVktAOtGs4Le5Z7K8SF5H2+oWsQFYVmOgH5loro7Y= -github.com/hashicorp/raft v1.0.0/go.mod h1:DVSAWItjLjTOkVbSpWQ0j0kUADIvDaCtBxIcbNAQLkI= -github.com/hashicorp/serf v0.8.1/go.mod h1:h/Ru6tmZazX7WO/GDmwdpS975F019L4t5ng5IgwbNrE= -github.com/hashicorp/vault v0.11.5/go.mod h1:KfSyffbKxoVyspOdlaGVjIuwLobi07qD1bAbosPMpP0= -github.com/hashicorp/vault-plugin-secrets-kv v0.0.0-20181106190520-2236f141171e/go.mod h1:VJHHT2SC1tAPrfENQeBhLlb5FbZoKZM+oC/ROmEftz0= -github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM= -github.com/hashicorp/yamux v0.0.0-20181012175058-2f1d1f20f75d/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/huandu/xstrings v1.0.0 h1:pO2K/gKgKaat5LdpAhxhluX2GPQMaI3W5FUz/I/UnWk= -github.com/huandu/xstrings v1.0.0/go.mod h1:4qWG/gcEcfX4z/mBDHJ++3ReCw9ibxbsNJbcucJdbSo= -github.com/imdario/mergo v0.3.4/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= -github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28= -github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/influxdata/flux v0.13.0 h1:iWNTxfR8m8LFYNAjSXAYMdxMKHqsBxd3xHlZ7rpl1KA= -github.com/influxdata/flux v0.13.0/go.mod h1:81jeDcHVn1rN5uj9aQ81S72Q8ol8If7N0zM0G8TnxTE= -github.com/influxdata/influxdb v1.7.4 h1:Ufqfn5xFixUXXj5Fgmhfa9RSke2R2AOvUOXfxgp9SCA= -github.com/influxdata/influxdb v1.7.4/go.mod h1:qZna6X/4elxqT3yI9iZYdZrWWdeFOOprn86kgg4+IzY= -github.com/influxdata/influxdb v1.8.0 h1:/X+G+i3udzHVxpBMuXdPZcUbkIE0ouT+6U+CzQTsOys= -github.com/influxdata/influxql v0.0.0-20180925231337-1cbfca8e56b6 h1:CFx+pP90q/qg3spoiZjf8donE4WpAdjeJfPOcoNqkWo= -github.com/influxdata/influxql v0.0.0-20180925231337-1cbfca8e56b6/go.mod h1:KpVI7okXjK6PRi3Z5B+mtKZli+R1DnZgb3N+tzevNgo= +github.com/influxdata/flux v0.65.1 h1:77BcVUCzvN5HMm8+j9PRBQ4iZcu98Dl4Y9rf+J5vhnc= +github.com/influxdata/flux v0.65.1/go.mod h1:J754/zds0vvpfwuq7Gc2wRdVwEodfpCFM7mYlOw2LqY= +github.com/influxdata/influxdb v1.8.3 h1:WEypI1BQFTT4teLM+1qkEcvUi0dAvopAI/ir0vAiBg8= +github.com/influxdata/influxdb v1.8.3/go.mod h1:JugdFhsvvI8gadxOI6noqNeeBHvWNTbfYGtiAn+2jhI= +github.com/influxdata/influxql v1.1.1-0.20200828144457-65d3ef77d385 h1:ED4e5Cc3z5vSN2Tz2GkOHN7vs4Sxe2yds6CXvDnvZFE= +github.com/influxdata/influxql v1.1.1-0.20200828144457-65d3ef77d385/go.mod h1:gHp9y86a/pxhjJ+zMjNXiQAA197Xk9wLxaz+fGG+kWk= github.com/influxdata/line-protocol v0.0.0-20180522152040-32c6aa80de5e h1:/o3vQtpWJhvnIbXley4/jwzzqNeigJK9z+LZcJZ9zfM= github.com/influxdata/line-protocol v0.0.0-20180522152040-32c6aa80de5e/go.mod h1:4kt73NQhadE3daL3WhR5EJ/J2ocX0PZzwxQ0gXJ7oFE= -github.com/influxdata/platform v0.0.0-20190117200541-d500d3cf5589 h1:oN2MMxbnMD/XIlyXbSczQqN5vcrCMFuRsiQafSt2c1E= -github.com/influxdata/platform v0.0.0-20190117200541-d500d3cf5589/go.mod h1:YVhys+JOY4wmXtJvdtkzLhS2K/r/px/vPc+EAddK+pg= -github.com/influxdata/roaring v0.4.12 h1:3DzTjKHcXFs4P3D7xRLpCqVrfK6eFRQT0c8BG99M3Ms= -github.com/influxdata/roaring v0.4.12/go.mod h1:bSgUQ7q5ZLSO+bKBGqJiCBGAl+9DxyW63zLTujjUlOE= +github.com/influxdata/promql/v2 v2.12.0/go.mod h1:fxOPu+DY0bqCTCECchSRtWfc+0X19ybifQhZoQNF5D8= +github.com/influxdata/roaring v0.4.13-0.20180809181101-fc520f41fab6 h1:UzJnB7VRL4PSkUJHwsyzseGOmrO/r4yA+AuxGJxiZmA= +github.com/influxdata/roaring v0.4.13-0.20180809181101-fc520f41fab6/go.mod h1:bSgUQ7q5ZLSO+bKBGqJiCBGAl+9DxyW63zLTujjUlOE= github.com/influxdata/tdigest v0.0.0-20181121200506-bf2b5ad3c0a9 h1:MHTrDWmQpHq/hkq+7cw9oYAt2PqUw52TZazRA0N7PGE= github.com/influxdata/tdigest v0.0.0-20181121200506-bf2b5ad3c0a9/go.mod h1:Js0mqiSBE6Ffsg94weZZ2c+v/ciT8QRHFOap7EKDrR0= github.com/influxdata/usage-client v0.0.0-20160829180054-6d3895376368 h1:+TUUmaFa4YD1Q+7bH9o5NCHQGPMqZCYJiNW6lIIS9z4= github.com/influxdata/usage-client v0.0.0-20160829180054-6d3895376368/go.mod h1:Wbbw6tYNvwa5dlB6304Sd+82Z3f7PmVZHVKU637d4po= -github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= -github.com/jefferai/jsonx v0.0.0-20160721235117-9cc31c3135ee/go.mod h1:N0t2vlmpe8nyZB5ouIbJQPDSR+mH6oe7xHB9VZHSUzM= -github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= -github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= -github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= -github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1 h1:6QPYqodiu3GuPL+7mfx+NwDdp2eTkp9IfEUpgAwUN0o= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/jsternberg/zap-logfmt v1.0.0/go.mod h1:uvPs/4X51zdkcm5jXl5SYoN+4RK21K8mysFmDaM/h+o= github.com/jsternberg/zap-logfmt v1.2.0 h1:1v+PK4/B48cy8cfQbxL4FmmNZrjnIMr2BsnyEmXqv2o= github.com/jsternberg/zap-logfmt v1.2.0/go.mod h1:kz+1CUmCutPWABnNkOu9hOHKdT2q3TDYCcsFy9hpqb0= -github.com/jtolds/gls v4.2.1+incompatible h1:fSuqC+Gmlu6l/ZYAoZzx2pyucC8Xza35fpRVWLVmUEE= -github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= github.com/jwilder/encoding v0.0.0-20170811194829-b4e1701a28ef h1:2jNeR4YUziVtswNP9sEFAI913cVrzH85T+8Q6LpYbT0= github.com/jwilder/encoding v0.0.0-20170811194829-b4e1701a28ef/go.mod h1:Ct9fl0F6iIOGgxJ5npU/IUOhOhqlVrGjyIZc8/MagT0= -github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k= -github.com/kevinburke/go-bindata v3.11.0+incompatible h1:RcC+GJNmrBHbGaOpQ9MBD8z22rdzlIm0esDRDkyxd4s= -github.com/kevinburke/go-bindata v3.11.0+incompatible/go.mod h1:/pEEZ72flUW2p0yi30bslSp9YqD9pysLxunQDdb2CPM= -github.com/kevinburke/ssh_config v0.0.0-20180830205328-81db2a75821e/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= -github.com/keybase/go-crypto v0.0.0-20181031135447-f919bfda4fc1/go.mod h1:ghbZscTyKdM07+Fw3KSi0hcJm+AlEUWj8QLlPtijN/M= -github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= +github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.4.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.4.1 h1:8VMb5+0wMgdBykOV96DwNwKFQ+WTI4pzYURP99CcB9E= github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/cpuid v0.0.0-20170728055534-ae7887de9fa5/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid v1.2.0 h1:NMpwD2G9JSFOE1/TJjGSo5zG7Yb2bTe7eq1jH+irmeE= github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= +github.com/klauspost/crc32 v0.0.0-20161016154125-cb6bfca970f6/go.mod h1:+ZoRqAPRLkC4NPOvfYeR5KNOrY6TD+/sAC3HXPZgDYg= +github.com/klauspost/pgzip v1.0.2-0.20170402124221-0bf5dcad4ada/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/klauspost/pgzip v1.2.1 h1:oIPZROsWuPHpOdMVWLuJZXwgjhrW8r1yEX8UqMyeNHM= github.com/klauspost/pgzip v1.2.1/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.0.0/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= @@ -252,54 +288,46 @@ github.com/mattn/go-colorable v0.0.9 h1:UVL0vNpWh04HeJXV0KLcaT7r06gOH2l4OW6ddYRU github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-isatty v0.0.4 h1:bnP0vzxcAdeI1zdubAl5PjU6zsERjGZb7raWodagDYs= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= +github.com/mattn/go-runewidth v0.0.3 h1:a+kO+98RDGEfo6asOGMmpodZq4FNtnGP54yps8BzLR4= github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= +github.com/mattn/go-sqlite3 v1.11.0 h1:LDdKkqtYlom37fkvqs8rMPFKAMe8+SgjbwZ6ex1/A/Q= +github.com/mattn/go-sqlite3 v1.11.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= +github.com/mattn/go-tty v0.0.0-20180907095812-13ff1204f104 h1:d8RFOZ2IiFtFWBcKEHAFYJcPTf0wY5q0exFNJZVWa1U= github.com/mattn/go-tty v0.0.0-20180907095812-13ff1204f104/go.mod h1:XPvLUNfbS4fJH25nqRHfWLMa1ONC8Amw+mIA639KxkE= -github.com/mattn/go-zglob v0.0.0-20171230104132-4959821b4817/go.mod h1:9fxibJccNxU2cnpIKLRRFA7zX7qhkJIQWBb449FYHOo= -github.com/mattn/go-zglob v0.0.0-20180803001819-2ea3427bfa53 h1:tGfIHhDghvEnneeRhODvGYOt305TPwingKt6p90F4MU= -github.com/mattn/go-zglob v0.0.0-20180803001819-2ea3427bfa53/go.mod h1:9fxibJccNxU2cnpIKLRRFA7zX7qhkJIQWBb449FYHOo= +github.com/matttproud/golang_protobuf_extensions v1.0.0/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/miekg/dns v1.1.1/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= -github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= -github.com/mitchellh/go-homedir v1.0.0 h1:vKb8ShqSby24Yrqr/yDYkuFz8d0WUjys40rvnGC8aR0= -github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/go-testing-interface v0.0.0-20171004221916-a61a99592b77/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= -github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= -github.com/mitchellh/mapstructure v1.0.0/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/minio/minio-go v0.0.0-20181109183348-774475480ffe/go.mod h1:7guKYtitv8dktvNUGrhzmNlA5wrAABTQXCoesZdFQO8= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/mapstructure v0.0.0-20180203102830-a4e142e9c047/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= -github.com/mna/pigeon v1.0.1-0.20180808201053-bb0192cfc2ae h1:mQO+oxi0kpii/TX+ltfTCFuYkOjEn53JhaOObiMuvnk= -github.com/mna/pigeon v1.0.1-0.20180808201053-bb0192cfc2ae/go.mod h1:Iym28+kJVnC1hfQvv5MUtI6AiFFzvQjHcvI4RFTG/04= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae h1:VeRdUYdCw49yizlSbMEn2SZ+gT+3IUKx8BqxyQdz+BY= github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg= -github.com/nats-io/gnatsd v1.3.0 h1:+5d80klu3QaJgNbdavVBjWJP7cHd11U2CLnRTFM9ICI= -github.com/nats-io/gnatsd v1.3.0/go.mod h1:nqco77VO78hLCJpIcVfygDP2rPGfsEHkGTUk94uh5DQ= -github.com/nats-io/go-nats v1.6.0 h1:FznPwMfrVwGnSCh7JTXyJDRW0TIkD4Tr+M1LPJt9T70= -github.com/nats-io/go-nats v1.6.0/go.mod h1:+t7RHT5ApZebkrQdnn6AhQJmhJJiKAvJUio1PiiCtj0= -github.com/nats-io/go-nats-streaming v0.4.0 h1:00wOBnTKzZGvQOFRSxj18kUm4X2TvXzv8LS0skZegPc= -github.com/nats-io/go-nats-streaming v0.4.0/go.mod h1:gfq4R3c9sKAINOpelo0gn/b9QDMBZnmrttcsNF+lqyo= -github.com/nats-io/nats-streaming-server v0.11.2 h1:UCqZbfXUKs9Ejw7KiNaFZEbbiVbK7uA8jbK2TsdGbqg= -github.com/nats-io/nats-streaming-server v0.11.2/go.mod h1:RyqtDJZvMZO66YmyjIYdIvS69zu/wDAkyNWa8PIUa5c= -github.com/nats-io/nuid v1.0.0 h1:44QGdhbiANq8ZCbUkdn6W5bqtg+mHuDE4wOUuxxndFs= -github.com/nats-io/nuid v1.0.0/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= -github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v1.4.2/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= github.com/opencontainers/runc v0.1.1/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= +github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74= github.com/opentracing/opentracing-go v1.0.2 h1:3jA2P6O1F9UOrWVpwrIo17pu01KWvNWg4X946/Y5Zwg= github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= -github.com/openzipkin/zipkin-go v0.1.3/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8= -github.com/ory/dockertest v3.3.2+incompatible/go.mod h1:1vX4m9wsvi00u5bseYwXaSnhNrne+V0E6LAcBILJdPs= -github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c h1:Lgl0gzECD8GnQ5QCWA8o6BtfL6mDH5rQgM4/fX3avOs= -github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= -github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= -github.com/pelletier/go-buffruneio v0.2.0/go.mod h1:JkE26KsDizTr40EUHkXVtNPvgGtbSNq5BcowyYOWdKo= +github.com/opentracing/opentracing-go v1.0.3-0.20180606204148-bd9c31933947 h1:oFoBvyA9Xh7MJd5dtfgocpsfjZUjh50IHPlDB0tILBs= +github.com/opentracing/opentracing-go v1.0.3-0.20180606204148-bd9c31933947/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= +github.com/ory/dockertest v3.3.4+incompatible/go.mod h1:1vX4m9wsvi00u5bseYwXaSnhNrne+V0E6LAcBILJdPs= +github.com/paulbellamy/ratecounter v0.2.0/go.mod h1:Hfx1hDpSGoqxkVVpBi/IlYD7kChlfo5C6hzIHwPqfFE= +github.com/paulmach/go.geojson v0.0.0-20170327170536-40612a87147b/go.mod h1:YaKx1hKpWF+T2oj2lFJPsW/t1Q5e1jQI61eoQSTwpIs= github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/peterh/liner v1.0.1-0.20180619022028-8c1271fcf47f/go.mod h1:xIteQHvHuaLYG9IFj6mSxM0fCKrs34IrEQUhOYuGPHc= github.com/philhofer/fwd v1.0.0 h1:UbZqGr5Y38ApvM/V/jEljVxwocdweyH+vmYvRPBnbqQ= github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= github.com/pierrec/lz4 v2.0.5+incompatible h1:2xWsjqPFWcplujydGg4WmhC/6fZqK42wMM8aXeqhl0I= @@ -308,60 +336,88 @@ github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/profile v1.2.1 h1:F++O52m40owAmADcojzM+9gyjmMOY/T4oYJkgFDH8RE= github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA= +github.com/pkg/term v0.0.0-20180730021639-bffc007b7fd5 h1:tFwafIEMf0B7NlcxV/zJ6leBIa81D3hgGSgsE5hCkOQ= github.com/pkg/term v0.0.0-20180730021639-bffc007b7fd5/go.mod h1:eCbImbZ95eXtAUIbLAuAVnBnwf83mjf6QIVH8SHYwqQ= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v0.0.0-20171201122222-661e31bf844d/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v0.9.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v0.9.2 h1:awm861/B8OKDd2I/6o1dy3ra4BamzKhYOiGItCeZ740= +github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM= +github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= +github.com/prometheus/client_golang v1.0.0 h1:vrDKnkGzuGvhNAL56c7DBz29ZL+KxnoR0x7enabFceM= +github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_model v0.0.0-20170216185247-6f3806018612/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 h1:idejC8f05m9MGOsuEi1ATq9shN03HrxNkD/luQvxCv8= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= -github.com/prometheus/common v0.0.0-20181020173914-7e9e6cabbd39/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 h1:gQz4mCbXsO+nc9n1hCxHcGA3Zx3Eo+UHZoInFGUIXNM= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.0.0-20180518154759-7600349dcfe1/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= -github.com/prometheus/common v0.0.0-20181218105931-67670fe90761 h1:z6tvbDJ5OLJ48FFmnksv04a78maSTRBUIhkdHYV5Y98= -github.com/prometheus/common v0.0.0-20181218105931-67670fe90761/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.6.0 h1:kRhiuYSXR3+uv2IbVbZhUxK5zVD/2pp3Gd2PpvPkpEo= +github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= +github.com/prometheus/procfs v0.0.0-20180612222113-7d6f385de8be/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a h1:9a8MnZMP0X2nLJdBg+pBmGgkJlSaKC2KaQmTCk1XDtE= github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190517135640-51af30a78b0e/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.2 h1:6LJUbpNm42llc4HRCuvApCSWB/WfhuNo9K98Q9sNGfs= +github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= +github.com/retailnext/hllpp v1.0.1-0.20180308014038-101a6d2f8b52/go.mod h1:RDpi1RftBQPUCDRw6SmxeaREsAaRKnOclghuzp/WRzc= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rs/cors v1.6.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= +github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNueLj0oo= +github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/ryanuber/go-glob v0.0.0-20170128012129-256dc444b735/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc= -github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= -github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= -github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/segmentio/kafka-go v0.1.0 h1:IXCHG+sXPNiIR5pC/vTEItZduPKu4cnpr85YgxpxlW0= github.com/segmentio/kafka-go v0.1.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfPOCvTvk+EJo= +github.com/segmentio/kafka-go v0.2.0 h1:HtCSf6B4gN/87yc5qTl7WsxPKQIIGXLPPM1bMCPOsoY= +github.com/segmentio/kafka-go v0.2.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfPOCvTvk+EJo= github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= +github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= +github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= +github.com/shurcooL/httpfs v0.0.0-20171119174359-809beceb2371/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg= github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/shurcooL/vfsgen v0.0.0-20180121065927-ffb13db8def0/go.mod h1:TrYk7fJVaAttu97ZZKrO9UbRa8izdowaMIZcxYMbVaw= +github.com/sirupsen/logrus v1.0.5/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= -github.com/smartystreets/goconvey v0.0.0-20180222194500-ef6db91d284a h1:JSvGDIbmil4Ui/dDdFBExb7/cmkNjyX5F97oglmvCDo= -github.com/smartystreets/goconvey v0.0.0-20180222194500-ef6db91d284a/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s= +github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= +github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= +github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= -github.com/spf13/cast v1.2.0 h1:HHl1DSRbEQN2i8tJmtS6ViPyHx35+p51amrdsiTCrkg= -github.com/spf13/cast v1.2.0/go.mod h1:r2rcYCSwa1IExKTDiTfzaxqT2FNHs8hODu4LnUfgKEg= +github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.3 h1:ZlrZ4XsMRm04Fr5pSFxBgfND2EBVa1nLpiy1stUsX/8= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= +github.com/spf13/cobra v0.0.5 h1:f0B+LkLX6DtmRH1isoNA9VTtNUK9K8xYd28JNNfOv/s= +github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= -github.com/spf13/pflag v1.0.2/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/viper v1.2.1 h1:bIcUwXqLseLF3BDAZduuNfekWG87ibtFxi59Bq+oI9M= -github.com/spf13/viper v1.2.1/go.mod h1:P4AexN0a+C9tGAnUFNwDMYYZv3pjFuvmeiMyKRaNVlI= -github.com/src-d/gcfg v1.4.0/go.mod h1:p/UMsR43ujA89BJY9duynAwIpvqEujIH/jFlfL7jWoI= -github.com/stevvooe/resumable v0.0.0-20180830230917-22b14a53ba50/go.mod h1:1pdIZTAHUz+HDKDVZ++5xg/duPlhKAIzw9qy42CWYp4= +github.com/spf13/viper v1.3.2 h1:VUFqw5KcqRf7i70GOzW7N+Q7+gxVBkSSqiXB12+JQ4M= +github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.0/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= @@ -371,25 +427,41 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/tcnksm/go-input v0.0.0-20180404061846-548a7d7a8ee8/go.mod h1:IlWNj9v/13q7xFbaK4mbyzMNwrZLaWSHx/aibKIZuIg= -github.com/testcontainers/testcontainer-go v0.0.0-20181115231424-8e868ca12c0f/go.mod h1:SrG3IY071gtmZJjGbKO+POJ57a/MMESerYNWt6ZRtKs= +github.com/tinylib/msgp v0.0.0-20190103190839-ade0ca4ace05/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= github.com/tinylib/msgp v1.0.2 h1:DfdQrzQa7Yh2es9SuLkixqxuXS2SxsdYn0KbdrOGWD8= github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= -github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= -github.com/tylerb/graceful v1.2.15/go.mod h1:LPYTbOYmUTdabwRt0TGhLllQ0MUNbs0Y5q1WXJOI9II= -github.com/urfave/cli v1.22.4 h1:u7tSpNPPswAFymm8IehJhy4uJMlUuU/GmqSkvJ1InXA= +github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/twpayne/go-geom v1.0.5/go.mod h1:gO3i8BeAvZuihwwXcw8dIOWXebCzTmy3uvXj9dZG2RA= +github.com/twpayne/go-kml v1.0.0/go.mod h1:LlvLIQSfMqYk2O7Nx8vYAbSLv4K9rjMvLlEdUKWdjq0= +github.com/twpayne/go-polyline v1.0.0/go.mod h1:ICh24bcLYBX8CknfvNPKqoTbe+eg+MX1NPyJmSBo7pU= +github.com/ugorji/go v1.1.2/go.mod h1:hnLbHMwcvSihnDhEfx2/BzKp2xb0Y+ErdfYcrs9tkJQ= +github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= +github.com/ugorji/go/codec v0.0.0-20190204201341-e444a5086c43/go.mod h1:iT03XoTwV7xq/+UGwKO3UbC1nNNlopQiY61beSdrtOA= +github.com/urfave/cli v1.20.0 h1:fDqGv3UG/4jbVl/QkFwEdddtEDjh/5Ov6X+0B/3bPaw= +github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli/v2 v2.2.0 h1:JTTnM6wKzdA0Jqodd966MVj4vWbbquZykeX1sKbe2C4= github.com/urfave/cli/v2 v2.2.0/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ= +github.com/vektah/dataloaden v0.2.1-0.20190515034641-a19b9a6e7c9e/go.mod h1:/HUdMve7rvxZma+2ZELQeNh88+003LL7Pf/CZ089j8U= +github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw= +github.com/willf/bitset v0.0.0-20181014161241-71fa2377963f/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= +github.com/willf/bitset v1.1.3/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= github.com/willf/bitset v1.1.9 h1:GBtFynGY9ZWZmEC9sWuu41/7VBXPFCOAbCbqTflOg9c= github.com/willf/bitset v1.1.9/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= -github.com/xanzy/ssh-agent v0.2.0/go.mod h1:0NyE30eGUDliuLEHJgYte/zncp2zdTStcOnWhgSqHD8= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/xlab/treeprint v0.0.0-20180616005107-d6fb6747feb6/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg= github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca h1:1CFlNzQhALwjS9mBAUkycX616GzgsuYUOCHA5+HSlXI= github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg= -github.com/yudai/gojsondiff v1.0.0/go.mod h1:AY32+k2cwILAkW1fbgxQ5mUmMiZFgLIV+FBNExI05xg= -github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82/go.mod h1:lgjkn3NuSvDfVJdfcVVdX+jpBxNmX4rDAzaS45IcYoM= -github.com/yudai/pp v2.0.1+incompatible/go.mod h1:PuxR/8QJ7cyCkFp/aUDS+JY727OFEZkTdatxwunjIkc= -go.opencensus.io v0.19.0 h1:+jrnNy8MR4GZXvwF9PEuSyHxA4NaTf6601oNRwCSXq0= -go.opencensus.io v0.19.0/go.mod h1:AYeH0+ZxYyghG8diqaaIq/9P3VgCCt5GF2ldCY4dkFg= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.etcd.io/etcd v0.0.0-20190228193606-a943ad0ee4c9 h1:3QcOf2A2G8CYue5DY60PR20dsJlfTT/vdnXEdU3ba7c= +go.etcd.io/etcd v0.0.0-20190228193606-a943ad0ee4c9/go.mod h1:KSGwdbiFchh5KIC9My2+ZVl5/3ANcwohw50dpPwa2cw= +go.etcd.io/etcd v3.3.25+incompatible h1:V1RzkZJj9LqsJRy+TUBgpWSbZXITLB819lstuTFoZOY= +go.etcd.io/etcd v3.3.25+incompatible/go.mod h1:yaeTdrJi5lOmYerz05bd8+V7KubZs8YSFZfzsF9A6aI= +go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.2 h1:75k/FF0Q2YM8QYo07VPddOLBslDt1MZOdEslOHvmzAs= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.uber.org/atomic v1.3.2 h1:2Oa65PReHzfn29GpvgsYwloV9AVFHPDk8tYxt2c2tr4= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.6.0 h1:Ezj3JGmsOnG1MoRWQkPBsKLe9DwWD9QeXzTRzzldNVk= @@ -404,123 +476,230 @@ go.uber.org/zap v1.9.1 h1:XCJQEf3W6eZaVwhRBof6ImoYGJSITeKWsyeh3HFu/5o= go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.15.0 h1:ZZCA22JRF2gQE5FoNmhmrf7jeJJ2uhqDUNRYKm8dvmM= go.uber.org/zap v1.15.0/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= -golang.org/x/crypto v0.0.0-20180505025534-4ec37c66abab/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20180608092829-8ac0e0d97ce4/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9 h1:mKdxBk7AujPs8kU4m80U72y/zjbZ3UcXC7dClwKbUI0= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529 h1:iMGN4xG0cnqj3t+zOM8wUB0BiPKHEwSxEZCvzcbZuvk= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20181112044915-a3060d491354 h1:6UAgZ8309zQ9+1iWkHzfszFguqzOdHGyGkd1HmhJ+UE= -golang.org/x/exp v0.0.0-20181112044915-a3060d491354/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299 h1:zQpM52jfKHG6II1ISZY1ZcpygvuSFZpLwfluuF89XOg= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20181217174547-8f45f776aaf1/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20190930215403-16217165b5de h1:5hukYrvBGR8/eNkX5mdUezrA6JiaEZDtJb9Ei+1LlBs= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f h1:J5lckAjkw6qYlOZNj90mLYNTEKDvWeuc1yieZ8qUzUE= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181106065722-10aee1819953/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181217023233-e147a9138326 h1:iCzOf0xz39Tstp+Tu/WwyGjUXCk34QhQORRxBeXXTA4= -golang.org/x/net v0.0.0-20181217023233-e147a9138326/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553 h1:efeOvDhwQ29Dj3SdAV/MJf8oukgn+8D8WgaCaRMchF8= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890 h1:uESlIz09WIHT2I+pasSXcpLYqYK8wHcdCetU3VuMBJE= -golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 h1:YUO/7uOKsKeq9UokNS62b8FYywz3ker1l1vDZRCRefw= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180903190138-2b024373dcd9/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180906133057-8cf3aee42992/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181030150119-7e31e0c00fa0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181218192612-074acd46bca6 h1:MXtOG7w2ND9qNCUZSDBGll/SpVIq7ftozR9I8/JGBHY= -golang.org/x/sys v0.0.0-20181218192612-074acd46bca6/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190405154228-4b34438f7a67/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d h1:+R4KGOnez64A81RvjARKc4UT5/tI9ujCIVX+P5KiHuI= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191210023423-ac6580df4449/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200107162124-548cf772de50 h1:YvQ10rzcqWXLlJZ3XCUoO25savxmscf4+SC+ZqiCHhA= +golang.org/x/sys v0.0.0-20200107162124-548cf772de50/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c h1:fqgJT0MGcGpPgpWU7VRdRjuArfcOvC4AoJmILihzhDg= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20181219222714-6e267b5cc78e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20181221154417-3ad2d988d5e2/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190125232054-d66bd3c5d5a6/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190515012406-7d7faa4812bd/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5 h1:hKsoRgsbwY1NafxrwTs+k64bikrLBkAgPir1TNCj3Zs= golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200108203644-89082a384178 h1:f5gMxb6FbpY48csegk9UPd7IAHVrBD013CU7N4pWzoE= +golang.org/x/tools v0.0.0-20200108203644-89082a384178/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= gonum.org/v1/gonum v0.0.0-20181121035319-3f7ecaa7e8ca h1:PupagGYwj8+I4ubCxcmcBRk3VlUWtTg5huQpZR9flmE= gonum.org/v1/gonum v0.0.0-20181121035319-3f7ecaa7e8ca/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= +gonum.org/v1/gonum v0.6.0 h1:DJy6UzXbahnGUf1ujUNkh/NEtK14qMo2nvlBPs4U5yw= +gonum.org/v1/gonum v0.6.0/go.mod h1:9mxDZsDKxgMAuccQkewq682L+0eCu4dCN2yonUJTCLU= gonum.org/v1/netlib v0.0.0-20181029234149-ec6d1f5cefe6 h1:4WsZyVtkthqrHTbDCJfiTs8IWNYE4uvsSDgaV6xpp+o= gonum.org/v1/netlib v0.0.0-20181029234149-ec6d1f5cefe6/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= -google.golang.org/api v0.0.0-20181021000519-a2651947f503/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= -google.golang.org/api v0.0.0-20181220000619-583d854617af/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= +gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0 h1:OE9mWmgKkjJyEmDAAtGMPjXu+YNeGvK9VTSHY6+Qihc= +gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= +gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc= +google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= +google.golang.org/api v0.3.2/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.15.0 h1:yzlyyDW/J0w8yNFJIhiAJy4kq74S+1DOLdawELNxFMA= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.3.0 h1:FBSsiFRMz3LBeXIomRnVzrQwSDj4ibvcRexLG0LZGQk= -google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5 h1:tycE03LOZYQNhDpS27tcQdAzLCVMaj7QT2SXxebnpCM= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20180608181217-32ee49c4dd80/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20181016170114-94acd270e44e/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20181219182458-5a97ab628bfb h1:dQshZyyJ5W/Xk8myF4GKBak1pZW6EywJuQ8+44EQhGA= -google.golang.org/genproto v0.0.0-20181219182458-5a97ab628bfb/go.mod h1:7Ep/1NZk928CDR8SjdVbjWNpdIf6nzjE3BTgJDr2Atg= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190516172635-bb713bdc0e52/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= +google.golang.org/genproto v0.0.0-20190716160619-c506a9f90610/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200108215221-bd8f9a0ef82f h1:2wh8dWY8959cBGQvk1RD+/eQBgRYYDaZ+hT0/zsARoA= +google.golang.org/genproto v0.0.0-20200108215221-bd8f9a0ef82f/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= -google.golang.org/grpc v1.15.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= -google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= -google.golang.org/grpc v1.17.0 h1:TRJYBgMclJvGYn2rIMjj+h9KtMt5r1Ij7ODVRIZkwhk= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= -gopkg.in/asn1-ber.v1 v1.0.0-20170511165959-379148ca0225/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.26.0 h1:2dTRdpdFEEhJYQD8EMLB61nnrzSCTbG38PhqdhvOltg= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +gopkg.in/DataDog/dd-trace-go.v1 v1.13.1 h1:oTzOClfuudNhW9Skkp2jxjqYO92uDKXqKLbiuPA13Rk= +gopkg.in/DataDog/dd-trace-go.v1 v1.13.1/go.mod h1:DVp8HmDh8PuTu2Z0fVVlBsyWaC++fzwVCaGWylTe3tg= +gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/fatih/pool.v2 v2.0.0 h1:xIFeWtxifuQJGk/IEPKsTduEKcKvPmhoiVDGpC40nKg= -gopkg.in/fatih/pool.v2 v2.0.0/go.mod h1:8xVGeu1/2jr2wm5V9SPuMht2H5AEmf5aFMGSQixtjTY= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= -gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/ldap.v2 v2.5.1/go.mod h1:oI0cpe/D7HRtBQl8aTg+ZmzFUAvu4lsv3eLXMLGFxWk= -gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= +gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo= +gopkg.in/ini.v1 v1.48.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= -gopkg.in/robfig/cron.v2 v2.0.0-20150107220207-be2e0b0deed5/go.mod h1:hiOFpYm0ZJbusNj2ywpbrXowU3G8U6GIQzqn2mw1UIE= -gopkg.in/src-d/go-billy.v4 v4.2.1/go.mod h1:tm33zBoOwxjYHZIE+OV8bxTWFMJLrconzFMd38aARFk= -gopkg.in/src-d/go-git-fixtures.v3 v3.1.1/go.mod h1:dLBcvytrw/TYZsNTWCnkNF2DSIlzWYqTe3rJR56Ac7g= -gopkg.in/src-d/go-git.v4 v4.8.1/go.mod h1:Vtut8izDyrM8BUVQnzJ+YvmNcem2J89EmfZYCkLokZk= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= -gopkg.in/vmihailenco/msgpack.v2 v2.9.1 h1:kb0VV7NuIojvRfzwslQeP3yArBqJHW9tOl4t38VS1jM= -gopkg.in/vmihailenco/msgpack.v2 v2.9.1/go.mod h1:/3Dn1Npt9+MYyLpYYXjInO/5jvMLamn+AEGwNEOatn8= -gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= +gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20180920025451-e3ad64cb4ed3/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20181108184350-ae8f1f9103cc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXeM= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -honnef.co/go/tools v0.0.1-2020.1.4 h1:UoveltGrhghAA7ePc+e+QYDHXrBps2PqFZiHkGR/xK8= -labix.org/v2/mgo v0.0.0-20140701140051-000000000287 h1:L0cnkNl4TfAXzvdrqsYEmxOHOCv2p5I3taaReO8BWFs= -labix.org/v2/mgo v0.0.0-20140701140051-000000000287/go.mod h1:Lg7AYkt1uXJoR9oeSZ3W/8IXLdvOfIITgZnommstyz4= -launchpad.net/gocheck v0.0.0-20140225173054-000000000087 h1:Izowp2XBH6Ya6rv+hqbceQyw/gSGoXfH/UPoTGduL54= -launchpad.net/gocheck v0.0.0-20140225173054-000000000087/go.mod h1:hj7XX3B/0A+80Vse0e+BUHsHMTEhd0O4cpUHr/e/BUM= +rsc.io/binaryregexp v0.2.0 h1:HfqmD5MEmC0zvwBuF187nq9mdnXjXsSivRiXN7SmRkE= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= +sourcegraph.com/sourcegraph/appdash v0.0.0-20180110180208-2cc67fd64755/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU= +sourcegraph.com/sourcegraph/appdash-data v0.0.0-20151005221446-73f23eafcf67/go.mod h1:L5q+DGLGOQFpo1snNEkLOJT2d1YTW66rWNzatr3He1k= diff --git a/raftmeta/badger_logger_bridge.go b/raftmeta/badger_logger_bridge.go index 441e1d8..82fcaff 100644 --- a/raftmeta/badger_logger_bridge.go +++ b/raftmeta/badger_logger_bridge.go @@ -3,7 +3,7 @@ package raftmeta import ( "strings" - "github.com/dgraph-io/badger" + "github.com/dgraph-io/badger/v2" "go.uber.org/zap" ) @@ -29,3 +29,7 @@ func (b *badgerLoggerBridge) Infof(format string, args ...interface{}) { func (b *badgerLoggerBridge) Warningf(format string, args ...interface{}) { b.suger.Warnf(strings.Trim(format, "\n"), args) } + +func (b *badgerLoggerBridge) Debugf(format string, args ...interface{}) { + b.suger.Debugf(strings.Trim(format, "\n"), args) +} diff --git a/raftmeta/internal/message.go b/raftmeta/internal/message.go index 38dc813..1a55467 100644 --- a/raftmeta/internal/message.go +++ b/raftmeta/internal/message.go @@ -1,7 +1,7 @@ package internal import ( - "github.com/coreos/etcd/raft/raftpb" + "go.etcd.io/etcd/raft/raftpb" ) const ( diff --git a/raftmeta/linearizable_read.go b/raftmeta/linearizable_read.go index 9bddc39..12bceaf 100644 --- a/raftmeta/linearizable_read.go +++ b/raftmeta/linearizable_read.go @@ -6,9 +6,10 @@ import ( "encoding/binary" "errors" "fmt" - "github.com/coreos/etcd/raft" "sync" "time" + + "go.etcd.io/etcd/raft" ) type Linearizabler struct { diff --git a/raftmeta/node.go b/raftmeta/node.go index 056a292..133dc92 100644 --- a/raftmeta/node.go +++ b/raftmeta/node.go @@ -15,13 +15,13 @@ import ( "github.com/angopher/chronus/raftmeta/internal" imeta "github.com/angopher/chronus/services/meta" "github.com/angopher/chronus/x" - "github.com/coreos/etcd/pkg/wait" - "github.com/coreos/etcd/raft" - "github.com/coreos/etcd/raft/raftpb" - "github.com/dgraph-io/badger" - "github.com/dgraph-io/badger/options" + "github.com/dgraph-io/badger/v2" + "github.com/dgraph-io/badger/v2/options" "github.com/dgraph-io/dgraph/raftwal" "github.com/influxdata/influxdb/services/meta" + "go.etcd.io/etcd/pkg/wait" + "go.etcd.io/etcd/raft" + "go.etcd.io/etcd/raft/raftpb" "go.uber.org/zap" "golang.org/x/net/trace" ) @@ -190,10 +190,9 @@ func NewRaftNode(config Config, logger *zap.Logger) *RaftNode { } x.Checkf(os.MkdirAll(config.WalDir, 0700), "Error while creating WAL dir.") - kvOpt := badger.DefaultOptions + kvOpt := badger.DefaultOptions(config.WalDir) kvOpt.SyncWrites = true - kvOpt.Dir = config.WalDir - kvOpt.ValueDir = config.WalDir + kvOpt.Logger = NewBadgerLoggerBridge(logger) kvOpt.TableLoadingMode = options.MemoryMap kvOpt.ValueLogFileSize = 8 << 20 kvOpt.MaxTableSize = 8 << 20 diff --git a/raftmeta/raft_logger_bridge.go b/raftmeta/raft_logger_bridge.go index 9be1d61..a3c04c5 100644 --- a/raftmeta/raft_logger_bridge.go +++ b/raftmeta/raft_logger_bridge.go @@ -3,7 +3,7 @@ package raftmeta import ( "fmt" - "github.com/coreos/etcd/raft" + "go.etcd.io/etcd/raft" "go.uber.org/zap" ) diff --git a/raftmeta/raft_test.go b/raftmeta/raft_test.go index 6bcdd5d..177f8e2 100644 --- a/raftmeta/raft_test.go +++ b/raftmeta/raft_test.go @@ -7,10 +7,10 @@ import ( "os" "testing" - "github.com/coreos/etcd/raft" - "github.com/coreos/etcd/raft/raftpb" "github.com/influxdata/influxdb/logger" "github.com/influxdata/influxdb/services/meta" + "go.etcd.io/etcd/raft" + "go.etcd.io/etcd/raft/raftpb" "github.com/angopher/chronus/raftmeta" "github.com/angopher/chronus/raftmeta/internal" diff --git a/raftmeta/transport.go b/raftmeta/transport.go index 2b9959d..cbeea05 100644 --- a/raftmeta/transport.go +++ b/raftmeta/transport.go @@ -15,8 +15,8 @@ import ( "github.com/angopher/chronus/raftmeta/internal" "github.com/angopher/chronus/x" - "github.com/coreos/etcd/raft" - "github.com/coreos/etcd/raft/raftpb" + "go.etcd.io/etcd/raft" + "go.etcd.io/etcd/raft/raftpb" "go.uber.org/zap" "golang.org/x/time/rate" ) From 59ea0d0a9af0c07f7ec5310888e3a0493fd309b1 Mon Sep 17 00:00:00 2001 From: Jason Joo Date: Mon, 12 Oct 2020 16:03:27 +0800 Subject: [PATCH 22/46] Fix for nasty denpendency issue on etcd Signed-off-by: Jason Joo --- cmd/metad/main.go | 2 +- go.mod | 33 ++++++------- go.sum | 82 +++++++++++++++++++++++++------- raftmeta/badger_logger_bridge.go | 8 ++-- raftmeta/node.go | 4 +- 5 files changed, 86 insertions(+), 43 deletions(-) diff --git a/cmd/metad/main.go b/cmd/metad/main.go index b04ebd0..aa51dc9 100644 --- a/cmd/metad/main.go +++ b/cmd/metad/main.go @@ -77,7 +77,7 @@ func main() { ids = append(ids, n.RaftId) } node.SetConfState(&raftpb.ConfState{ - Nodes: ids, + Voters: ids, }) err = node.Restore(*restoreFile) if err != nil { diff --git a/go.mod b/go.mod index 9a2136c..19dcbc4 100644 --- a/go.mod +++ b/go.mod @@ -4,31 +4,28 @@ require ( github.com/BurntSushi/toml v0.3.1 github.com/dgraph-io/badger/v2 v2.2007.2 github.com/dgraph-io/dgraph v1.2.7 - github.com/fatih/color v1.7.0 + github.com/fatih/color v1.9.0 github.com/gogo/protobuf v1.3.1 - github.com/golang/snappy v0.0.1 - github.com/google/go-cmp v0.4.0 + github.com/golang/snappy v0.0.2 + github.com/google/go-cmp v0.5.2 github.com/influxdata/influxdb v1.8.3 github.com/influxdata/influxql v1.1.1-0.20200828144457-65d3ef77d385 github.com/influxdata/usage-client v0.0.0-20160829180054-6d3895376368 github.com/jsternberg/zap-logfmt v1.2.0 - github.com/klauspost/compress v1.4.1 // indirect - github.com/klauspost/cpuid v1.2.0 // indirect - github.com/klauspost/pgzip v1.2.1 // indirect + github.com/klauspost/compress v1.11.1 // indirect + github.com/klauspost/pgzip v1.2.5 // indirect github.com/pkg/errors v0.9.1 - github.com/stretchr/testify v1.5.1 // test + github.com/stretchr/testify v1.6.1 // test github.com/urfave/cli/v2 v2.2.0 - github.com/willf/bitset v1.1.9 // indirect - github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca // indirect - go.etcd.io/etcd v3.3.25+incompatible - go.uber.org/zap v1.15.0 - golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 - golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553 - golang.org/x/text v0.3.2 - golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 + github.com/willf/bitset v1.1.11 // indirect + github.com/xlab/treeprint v1.0.0 // indirect + go.etcd.io/etcd v0.0.0-20200824191128-ae9734ed278b // actual it's v3.4.13. we need it because of nasty dependency issue. + go.uber.org/zap v1.16.0 + golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0 + golang.org/x/net v0.0.0-20201010224723-4f7140c49acb + golang.org/x/text v0.3.3 + golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e gopkg.in/natefinch/lumberjack.v2 v2.0.0 ) -replace github.com/coreos/etcd v3.3.25+incompatible => go.etcd.io/etcd v3.3.25+incompatible - -go 1.13 +go 1.14 diff --git a/go.sum b/go.sum index 0202d65..4f14672 100644 --- a/go.sum +++ b/go.sum @@ -86,6 +86,8 @@ github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWR github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa h1:OaNxuTZr7kxeODyLWsRMC+OD03aFUH+mW6r2d+MWa5Y= +github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd h1:qMd81Ts1T2OTKmB4acZcyKaMtRnY5Y44NuXGX2GFJ1w= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/containerd/continuity v0.0.0-20181203112020-004b46473808/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= @@ -99,6 +101,7 @@ github.com/cpuguy83/go-md2man v1.0.10 h1:BSKMNlYxDvnunlTymqtgONjNnaRV1sTpcovwwjF github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/d4l3k/messagediff v1.2.1/go.mod h1:Oozbb1TVXFac9FtSIxHBMnBCq2qeH/2KkEQxENCrlLo= github.com/dave/jennifer v1.2.0/go.mod h1:fIb+770HOpJ2fmN9EPPKOqm1vMGhB+TwXKMZhrIygKg= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -135,11 +138,13 @@ github.com/eclipse/paho.mqtt.golang v1.2.0 h1:1F8mhG9+aO5/xpdtFkW4SxOJB67ukuDC3t github.com/eclipse/paho.mqtt.golang v1.2.0/go.mod h1:H9keYFcgq3Qr5OUJm/JZI/i6U7joQ8SYLhZwfeOo6Ts= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fatih/color v1.9.0 h1:8xPHl4/q1VyqGIPif1F+1V3Y3lSmrq01EabUW3CoW5s= +github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32/go.mod h1:GIjDIg/heH5DOkXY3YJ/wNhfHsQHoXGjl8G8amsYQ1I= github.com/glycerine/go-unsnap-stream v0.0.0-20180323001048-9f0cb55181dd h1:r04MMPyLHj/QwZuMJ5+7tJcBr1AQjpiAK/rZWRrQT7o= github.com/glycerine/go-unsnap-stream v0.0.0-20180323001048-9f0cb55181dd/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE= @@ -161,6 +166,7 @@ github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRx github.com/gogo/protobuf v1.0.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= @@ -187,6 +193,8 @@ github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db h1:woRePGFeVFfLKN/pO github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.2 h1:aeE13tS0IiQgFjYdoL8qN3K1N2bXXtI6Vi51/y7BpMw= +github.com/golang/snappy v0.0.2/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180124185431-e89373fe6b4a/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c h1:964Od4U6p2jUkFxvCydnIczKteheJEzHRToSGK3Bnlw= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -201,6 +209,9 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= @@ -222,6 +233,7 @@ github.com/gorilla/websocket v1.2.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoA github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.4.1/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= +github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/hashicorp/golang-lru v0.5.0 h1:CL2msUPvZTLb5O648aiLNJw3hnBxN2+1Jq8rCOH9wdo= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= @@ -248,6 +260,7 @@ github.com/influxdata/usage-client v0.0.0-20160829180054-6d3895376368 h1:+TUUmaF github.com/influxdata/usage-client v0.0.0-20160829180054-6d3895376368/go.mod h1:Wbbw6tYNvwa5dlB6304Sd+82Z3f7PmVZHVKU637d4po= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1 h1:6QPYqodiu3GuPL+7mfx+NwDdp2eTkp9IfEUpgAwUN0o= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= @@ -260,18 +273,17 @@ github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7V github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= github.com/jwilder/encoding v0.0.0-20170811194829-b4e1701a28ef h1:2jNeR4YUziVtswNP9sEFAI913cVrzH85T+8Q6LpYbT0= github.com/jwilder/encoding v0.0.0-20170811194829-b4e1701a28ef/go.mod h1:Ct9fl0F6iIOGgxJ5npU/IUOhOhqlVrGjyIZc8/MagT0= +github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.4.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/compress v1.4.1 h1:8VMb5+0wMgdBykOV96DwNwKFQ+WTI4pzYURP99CcB9E= -github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/compress v1.11.1 h1:bPb7nMRdOZYDrpPMTA3EInUQrdgoBinqUuSwlGdKDdE= +github.com/klauspost/compress v1.11.1/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/cpuid v0.0.0-20170728055534-ae7887de9fa5/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= -github.com/klauspost/cpuid v1.2.0 h1:NMpwD2G9JSFOE1/TJjGSo5zG7Yb2bTe7eq1jH+irmeE= -github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/crc32 v0.0.0-20161016154125-cb6bfca970f6/go.mod h1:+ZoRqAPRLkC4NPOvfYeR5KNOrY6TD+/sAC3HXPZgDYg= github.com/klauspost/pgzip v1.0.2-0.20170402124221-0bf5dcad4ada/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= -github.com/klauspost/pgzip v1.2.1 h1:oIPZROsWuPHpOdMVWLuJZXwgjhrW8r1yEX8UqMyeNHM= -github.com/klauspost/pgzip v1.2.1/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= +github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE= +github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= @@ -286,8 +298,13 @@ github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDe github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mattn/go-colorable v0.0.9 h1:UVL0vNpWh04HeJXV0KLcaT7r06gOH2l4OW6ddYRUIY4= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA= +github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-isatty v0.0.4 h1:bnP0vzxcAdeI1zdubAl5PjU6zsERjGZb7raWodagDYs= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.11 h1:FxPOTFNqGkuDUGi3H/qkUbQO4ZiBa2brKq5r0l8TGeM= +github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.3 h1:a+kO+98RDGEfo6asOGMmpodZq4FNtnGP54yps8BzLR4= github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= @@ -303,7 +320,9 @@ github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrk github.com/mitchellh/mapstructure v0.0.0-20180203102830-a4e142e9c047/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae h1:VeRdUYdCw49yizlSbMEn2SZ+gT+3IUKx8BqxyQdz+BY= github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg= @@ -372,6 +391,7 @@ github.com/prometheus/procfs v0.0.2 h1:6LJUbpNm42llc4HRCuvApCSWB/WfhuNo9K98Q9sNG github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/retailnext/hllpp v1.0.1-0.20180308014038-101a6d2f8b52/go.mod h1:RDpi1RftBQPUCDRw6SmxeaREsAaRKnOclghuzp/WRzc= +github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rs/cors v1.6.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNueLj0oo= @@ -393,6 +413,7 @@ github.com/shurcooL/vfsgen v0.0.0-20180121065927-ffb13db8def0/go.mod h1:TrYk7fJV github.com/sirupsen/logrus v1.0.5/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= @@ -425,8 +446,8 @@ github.com/stretchr/testify v1.2.1/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/tinylib/msgp v0.0.0-20190103190839-ade0ca4ace05/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= github.com/tinylib/msgp v1.0.2 h1:DfdQrzQa7Yh2es9SuLkixqxuXS2SxsdYn0KbdrOGWD8= github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= @@ -445,18 +466,18 @@ github.com/vektah/dataloaden v0.2.1-0.20190515034641-a19b9a6e7c9e/go.mod h1:/HUd github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw= github.com/willf/bitset v0.0.0-20181014161241-71fa2377963f/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= github.com/willf/bitset v1.1.3/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= -github.com/willf/bitset v1.1.9 h1:GBtFynGY9ZWZmEC9sWuu41/7VBXPFCOAbCbqTflOg9c= -github.com/willf/bitset v1.1.9/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= +github.com/willf/bitset v1.1.11 h1:N7Z7E9UvjW+sGsEl7k/SJrvY2reP1A07MrGuCjIOjRE= +github.com/willf/bitset v1.1.11/go.mod h1:83CECat5yLh5zVOf4P1ErAgKA5UDvKtgyUABdr3+MjI= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xlab/treeprint v0.0.0-20180616005107-d6fb6747feb6/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg= -github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca h1:1CFlNzQhALwjS9mBAUkycX616GzgsuYUOCHA5+HSlXI= -github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg= +github.com/xlab/treeprint v1.0.0 h1:J0TkWtiuYgtdlrkkrDLISYBQ92M+X5m4LrIIMKrbDTs= +github.com/xlab/treeprint v1.0.0/go.mod h1:IoImgRak9i3zJyuxOKUP1v4UZd1tMoKkq/Cimt1uhCg= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.etcd.io/etcd v0.0.0-20190228193606-a943ad0ee4c9 h1:3QcOf2A2G8CYue5DY60PR20dsJlfTT/vdnXEdU3ba7c= +go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/etcd v0.0.0-20190228193606-a943ad0ee4c9/go.mod h1:KSGwdbiFchh5KIC9My2+ZVl5/3ANcwohw50dpPwa2cw= -go.etcd.io/etcd v3.3.25+incompatible h1:V1RzkZJj9LqsJRy+TUBgpWSbZXITLB819lstuTFoZOY= -go.etcd.io/etcd v3.3.25+incompatible/go.mod h1:yaeTdrJi5lOmYerz05bd8+V7KubZs8YSFZfzsF9A6aI= +go.etcd.io/etcd v0.0.0-20200824191128-ae9734ed278b h1:QS2G6o7lP5jDfqsEdRAJM3J/5Ml5fpWbh9EUNpzKAVY= +go.etcd.io/etcd v0.0.0-20200824191128-ae9734ed278b/go.mod h1:yVHk9ub3CSBatqGNg7GRmsnfLWtoW60w4eDYfh7vHDg= go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= @@ -474,8 +495,9 @@ go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee h1:0mgffUl7nfd+FpvXMVz4IDEa go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v1.9.1 h1:XCJQEf3W6eZaVwhRBof6ImoYGJSITeKWsyeh3HFu/5o= go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -go.uber.org/zap v1.15.0 h1:ZZCA22JRF2gQE5FoNmhmrf7jeJJ2uhqDUNRYKm8dvmM= -go.uber.org/zap v1.15.0/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.16.0 h1:uFRZXykJGK9lLY4HtgSw44DnIcAM+kRBP7x5m+NpAOM= +go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= golang.org/x/crypto v0.0.0-20180608092829-8ac0e0d97ce4/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -486,6 +508,9 @@ golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0 h1:hb9wdF1z5waM+dSIICn1l0DkLVDT3hqhhQsDNUmHPRE= +golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -520,6 +545,7 @@ golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -532,8 +558,11 @@ golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553 h1:efeOvDhwQ29Dj3SdAV/MJf8oukgn+8D8WgaCaRMchF8= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201010224723-4f7140c49acb h1:mUVeFHoDKis5nxCAzoAi7E8Ghb86EXh/RK6wtvJIqRY= +golang.org/x/net v0.0.0-20201010224723-4f7140c49acb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -552,14 +581,17 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190405154228-4b34438f7a67/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d h1:+R4KGOnez64A81RvjARKc4UT5/tI9ujCIVX+P5KiHuI= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -567,21 +599,30 @@ golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191210023423-ac6580df4449/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200107162124-548cf772de50 h1:YvQ10rzcqWXLlJZ3XCUoO25savxmscf4+SC+ZqiCHhA= golang.org/x/sys v0.0.0-20200107162124-548cf772de50/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f h1:+Nyd8tzPX9R7BWHguqsrbFdRx3WQ/1ib8I44HXV5yTA= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c h1:fqgJT0MGcGpPgpWU7VRdRjuArfcOvC4AoJmILihzhDg= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 h1:SvFZT6jyqRaOeXpc5h/JSfZenJ2O330aBsf7JfSUXmQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e h1:EHBhcS0mlXEAVwNyO2dLfjToGsyY4j24pTs2ScHnX7s= +golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -685,12 +726,16 @@ gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKW gopkg.in/ini.v1 v1.48.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= +gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= @@ -701,5 +746,6 @@ honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt rsc.io/binaryregexp v0.2.0 h1:HfqmD5MEmC0zvwBuF187nq9mdnXjXsSivRiXN7SmRkE= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= +sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= sourcegraph.com/sourcegraph/appdash v0.0.0-20180110180208-2cc67fd64755/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU= sourcegraph.com/sourcegraph/appdash-data v0.0.0-20151005221446-73f23eafcf67/go.mod h1:L5q+DGLGOQFpo1snNEkLOJT2d1YTW66rWNzatr3He1k= diff --git a/raftmeta/badger_logger_bridge.go b/raftmeta/badger_logger_bridge.go index 82fcaff..fc2e0fb 100644 --- a/raftmeta/badger_logger_bridge.go +++ b/raftmeta/badger_logger_bridge.go @@ -19,17 +19,17 @@ func NewBadgerLoggerBridge(logger *zap.Logger) *badgerLoggerBridge { } func (b *badgerLoggerBridge) Errorf(format string, args ...interface{}) { - b.suger.Errorf(strings.Trim(format, "\n"), args) + b.suger.Errorf(strings.Trim(format, "\n"), args...) } func (b *badgerLoggerBridge) Infof(format string, args ...interface{}) { - b.suger.Infof(strings.Trim(format, "\n"), args) + b.suger.Infof(strings.Trim(format, "\n"), args...) } func (b *badgerLoggerBridge) Warningf(format string, args ...interface{}) { - b.suger.Warnf(strings.Trim(format, "\n"), args) + b.suger.Warnf(strings.Trim(format, "\n"), args...) } func (b *badgerLoggerBridge) Debugf(format string, args ...interface{}) { - b.suger.Debugf(strings.Trim(format, "\n"), args) + b.suger.Debugf(strings.Trim(format, "\n"), args...) } diff --git a/raftmeta/node.go b/raftmeta/node.go index 133dc92..0200ba8 100644 --- a/raftmeta/node.go +++ b/raftmeta/node.go @@ -287,8 +287,8 @@ func (s *RaftNode) Restore(filePath string) error { } snapshot.Metadata.ConfState = *s.RaftConfState s.Logger.Warn(fmt.Sprintf( - "Nodes=%v, Learners=%v", - snapshot.Metadata.ConfState.Nodes, + "Voters=%v, Learners=%v", + snapshot.Metadata.ConfState.Voters, snapshot.Metadata.ConfState.Learners, )) err = s.resetPeersInSnapshot(&snapshot) From 034cb70982364d00ae1e7ee97f8ae7c44866abcd Mon Sep 17 00:00:00 2001 From: Jason Joo Date: Tue, 13 Oct 2020 10:32:51 +0800 Subject: [PATCH 23/46] - Fix bug of adding new meta node - Add quick start Signed-off-by: Jason Joo --- README.md | 183 +++++++++++++++++++++++++++++--------- cmd/influxd/run/config.go | 7 ++ cmd/metad/main.go | 10 ++- coordinator/config.go | 2 +- logging/log.go | 2 +- raftmeta/config.go | 9 +- raftmeta/node.go | 6 +- raftmeta/transport.go | 4 + 8 files changed, 170 insertions(+), 53 deletions(-) diff --git a/README.md b/README.md index 896f287..8a9108e 100644 --- a/README.md +++ b/README.md @@ -1,78 +1,177 @@ -# chronus -chronus是基于influxdb1.7.x版本开发的分布式时序数据库，兼容官方的influxql。 +# Chronus + +Chronus is an opensourced version of implementation follow the +design of official influxdb-cluster. +It's free and compatible with original influxdb. ## Features -- 完全兼容influxdb的influxql协议 +- Compatible with influxql +- High availability +- Scalable online +- Eventual consistent +- Parallel computation +- No other infrastructure needed -- 高可用 +## Architecture -- 水平线性扩展 +![Cluster Sample](docs/architecture.png) -- 最终一致性 +### Meta Server -- MPP架构：大规模并行处理 +Keep the meta data including shards and its group, continuous +queries, data nodes, etc. . -- 不依赖第三方服务 +High availability of meta cluster is achieved through RAFT +(from etcd). So at least 3 meta servers are recommended. And +any odd number greater than one should work but the more the +poorer writing performance. So 3 or 5 is recommended finally. -## Architecture - +Meta server cluster can be managed through command line tool +`metad-ctl` in `cmd/metad-ctl`. + +### Data Server + +Data servers hold the data of shards, run continuous queries +(when aquired lock), accept queries of client and do inner +remote joins if necessary. + +Shards are managed by shard groups in logical and each shard +will be guaranteed to be owned by specified `replica` count of +data servers. Group is used to make writes distributed. + +## Compile + +```shell +cd cmd/influxd && go build && cd - +cd cmd/influxd-ctl && go build && cd - +cd cmd/metad && go build && cd - +cd cmd/metad-ctl && go build && cd - +``` + +You would get four binaries (metad, metad-ctl, influxd, +influxd-ctl) if successfully. You can put them into PATH or +package them into private repository. + +## Boot Meta Cluster +### Configuration -- meta server：负责元数据的存储和管理 +Sample configuration can be reached by executing `metad -sample`. +You should pay attention to following items: -- data server：负责数据的存储和管理，并向meta server获取元数据。所有涉及到元数据的修改，都会请求到meta server，由meta server执行修改，并同步给集群内所有的data server +- my-addr: in ip:port format which will be used in cluster's communication between nodes +- raft-id: should be a positive, unique number +- wal-dir: where the meta data stores in +- log-dir: where the logs store in. Logs will be splitted automatically and if you leave +it blank, standard output would be used +- log-level: in production `warn` is recommended -## compile +### Boot First Meta Node -- export GO111MODULE=on +Start with sample configuation: -- 需要梯子：export -GOPROXY=https://goproxy.io +```shell +metad -sample > metad.conf +metad -config metad.conf +``` -- mkdir -p workspace/src/github.com/angopher && export GOPATH=$PWD/workspace +Then a single node meta cluster is done. -- cd workspace/src/github.com/angopher +### Check Meta Cluster Status -- git clone https://github.com/angopher/chronus.git && cd chronus +Using: -- 编译influxd： cd cmd/influxd && go build && cd - +```shell +metad-ctl -s ip:port status +``` -- 编译metad：cd cmd/metad && go build +You can specify any **alive** address in cluster by `-s ip:port` and it's +always required to call `metad-ctl`. -## run +### Add More Meta Nodes to Existed Cluster -配置启动meta server +You can add more nodes into cluster through two phases: -- 生成默认配置文件：./metad config > metad.conf +1. Add node through `metad-ctl add` before boot up new node actually +2. Boot up the node with correct configuration, thus, correct `raft-id` / `my-addr` +3. [CHECK]Always check the cluster status and make sure everything as your expected. -- ./metad -config metad.conf +Please add ONLY one node at a time. -配置启动data server +> You can also boot up the cluster initially with configured `peers` altogether. +> Set peers expected, boot them up. -- 生成默认配置文件 ./influxd config > influxd.conf +```text +Sample peers format: +peers=[ + {addr="127.0.0.1:2345", raft-id=1}, + {addr="127.0.0.1:2346", raft-id=2}, + {addr="127.0.0.1:2347", raft-id=3}, +] +``` -- ./influxd -config influxd.conf +## Boot Data Cluster -## Getting Started +You can use following commands to generate sample configuration of data node. -创建第一个数据库 -- curl -XPOST "http://localhost:8086/query" --data-urlencode "q=CREATE DATABASE mydb" +```shell +influxd config > data.conf +``` -写入数据 -- curl -XPOST "http://localhost:8086/write?db=mydb" -d 'cpu,host=server01,region=uswest load=42 1434055562000000000' -- curl -XPOST "http://localhost:8086/write?db=mydb" -d 'cpu,host=server02,region=uswest load=78 1434055562000000000' -- curl -XPOST "http://localhost:8086/write?db=mydb" -d 'cpu,host=server03,region=useast load=15.4 1434055562000000000' +You may mainly care about following configurations: -查询 -- curl -G "http://localhost:8086/query?pretty=true" --data-urlencode "db=mydb" --data-urlencode "q=SELECT * FROM cpu WHERE host='server01' AND time < now() - 1d" +- bind-address: TCP address of data node. +- meta.dir: Directory holds meta data. Absolute path is recommended. +- data.dir: Directory holds data. Absolute path is recommended. +- data.wal-dir: Directory holds wal data. Absolute path is recommended. +- coordinator.pool-max-streams-per-node: Max streams allowed to single data node when +forwarding quries internally. You can adjust it according to your load. +- coordinator.meta-services: [Important]Addresses of meta nodes. +- http.bind-address: Query service listening address which is also called `HTTP Address`. +- http.access-log-path: File holds access log. It will be rotated automatically. Leave it +empty to disable. +- logging.format: `console`(`auto`) / `json` / `logfmt` +- logging.level: `debug` / `info` / `warn` / `fatal` +- continuous_queries.run-interval: Interval of running continuous queries. +- hinted-handoff.{enabled, dir}: If you want to use hh service you should set both of them. +- hinted-handoff.retry-concurrency: If you want to constraint the max requests retrying. Or +unlimited retrying may exhaust the connection pool quickly. +- controller.max_shard_copy_tasks: Max concurrency of active copying task on node. -分析 -- curl -G "http://localhost:8086/query?pretty=true" --data-urlencode "db=mydb" --data-urlencode "q=SELECT mean(load) FROM cpu WHERE region='uswest'" +You can start the data node using: + +```shell +influxd run -config data.conf +``` + +or you want all the logs write into a directory instead of standard output: + +```shell +influxd run -config data.conf -logdir /dir/to/write +``` + +You can repeat to add more data nodes and check the node list using: + +```shell +influxd-ctl -s ip:port node list +``` + +Where `ip:port` is the TCP address of any node in data node cluster. + +## Query + +Data cluster is compatible with `influx` command line tool and any other clients. +Pay attention to the following rules: + +- All data nodes can be quried equivalently. +- Append only. +- Carefully set retention policies. ## License -chronus is under the MIT license. See the [LICENSE](LICENSE) file for details. -欢迎微信交流 +Chronus is under the MIT license. See the [LICENSE](LICENSE) file for details. + +## Contact - +![WeChat QR Code](./docs/wechat.jpeg) diff --git a/cmd/influxd/run/config.go b/cmd/influxd/run/config.go index f707a5d..53f32ad 100644 --- a/cmd/influxd/run/config.go +++ b/cmd/influxd/run/config.go @@ -9,6 +9,7 @@ import ( "path/filepath" "regexp" "strings" + "time" "github.com/BurntSushi/toml" "github.com/influxdata/influxdb/logger" @@ -75,7 +76,10 @@ type Config struct { func NewConfig() *Config { c := &Config{} c.Meta = meta.NewConfig() + c.Meta.Dir = "./meta" c.Data = tsdb.NewConfig() + c.Data.Dir = "./data" + c.Data.WALDir = "./wal" c.Coordinator = coordinator.NewConfig() c.Precreator = precreator.NewConfig() @@ -85,6 +89,8 @@ func NewConfig() *Config { c.Logging = logger.NewConfig() c.HintedHandoff = hh.NewConfig() + c.HintedHandoff.Enabled = true + c.HintedHandoff.Dir = "./hh" c.Controller = controller.NewConfig() c.GraphiteInputs = []graphite.Config{graphite.NewConfig()} @@ -93,6 +99,7 @@ func NewConfig() *Config { c.UDPInputs = []udp.Config{udp.NewConfig()} c.ContinuousQuery = continuous_querier.NewConfig() + c.ContinuousQuery.RunInterval = itoml.Duration(time.Minute) c.Retention = retention.NewConfig() c.BindAddress = DefaultBindAddress diff --git a/cmd/metad/main.go b/cmd/metad/main.go index aa51dc9..c700376 100644 --- a/cmd/metad/main.go +++ b/cmd/metad/main.go @@ -20,15 +20,19 @@ func main() { configFile := f.String("config", "", "Specify config file") dumpFile := f.String("dump", "", "Boot and dump the snapshot to a file specified") restoreFile := f.String("restore", "", "Boot and restore data from the snapshot specified") + showSample := f.Bool("sample", false, "Show sample configuration") f.Parse(os.Args[1:]) config := raftmeta.NewConfig() + if *showSample { + toml.NewEncoder(os.Stdout).Encode(&config) + fmt.Println() + return + } + if *configFile != "" { x.Check((&config).FromTomlFile(*configFile)) } else { - fmt.Print("Sample configuration:\n\n") - toml.NewEncoder(os.Stdout).Encode(&config) - fmt.Println() f.Usage() return } diff --git a/coordinator/config.go b/coordinator/config.go index 1d13771..43bd0fb 100644 --- a/coordinator/config.go +++ b/coordinator/config.go @@ -71,7 +71,7 @@ func NewConfig() Config { MaxSelectPointN: DefaultMaxSelectPointN, MaxSelectSeriesN: DefaultMaxSelectSeriesN, MetaServices: []string{DefaultMetaService}, - PingMetaServiceIntervalMs: 50, + PingMetaServiceIntervalMs: 250, } } diff --git a/logging/log.go b/logging/log.go index 57d0a1d..d7a140d 100644 --- a/logging/log.go +++ b/logging/log.go @@ -43,7 +43,7 @@ func newEncoder(format string) (zapcore.Encoder, error) { switch format { case "json": return zapcore.NewJSONEncoder(config), nil - case "console": + case "console", "auto": return zapcore.NewConsoleEncoder(config), nil case "logfmt": return zaplogfmt.NewEncoder(config), nil diff --git a/raftmeta/config.go b/raftmeta/config.go index 913c455..64565f1 100644 --- a/raftmeta/config.go +++ b/raftmeta/config.go @@ -57,15 +57,18 @@ func NewConfig() Config { MyAddr: DefaultAddr, RaftId: 1, Peers: []Peer{}, - TickTimeMs: 20, + TickTimeMs: 60, ElectionTick: DefaultElectionTick, HeartbeatTick: DefaultHeartbeatTick, MaxSizePerMsg: DefaultMaxSizePerMsg, MaxInflightMsgs: DefaultMaxInflightMsgs, WalDir: "./wal", - SnapshotIntervalSec: 60, - ChecksumIntervalSec: 10, + SnapshotIntervalSec: 300, + ChecksumIntervalSec: 120, RetentionAutoCreate: true, + LogFormat: "console", + LogLevel: "info", + LogDir: "./logs", } } diff --git a/raftmeta/node.go b/raftmeta/node.go index 0200ba8..6ed4711 100644 --- a/raftmeta/node.go +++ b/raftmeta/node.go @@ -469,7 +469,7 @@ func (s *RaftNode) Run() { case <-snapshotTicker.C: if leader == s.ID { go func() { - err := s.trigerSnapshot() + err := s.triggerSnapshot() if err != nil { s.Logger.Error("calculateSnapshot fail", zap.Error(err)) } @@ -569,8 +569,8 @@ func (s *RaftNode) triggerChecksum() { } } -func (s *RaftNode) trigerSnapshot() error { - s.Logger.Info("trigerSnapshot") +func (s *RaftNode) triggerSnapshot() error { + s.Logger.Info("triggerSnapshot") var sn internal.CreateSnapshot //_, err = s.Storage.LastIndex() //x.Check(err) diff --git a/raftmeta/transport.go b/raftmeta/transport.go index cbeea05..8d8c342 100644 --- a/raftmeta/transport.go +++ b/raftmeta/transport.go @@ -210,6 +210,10 @@ func (s *RaftNode) HandleUpdateCluster(w http.ResponseWriter, r *http.Request) { resp.RetMsg = err.Error() return } + if op == "add" { + // trigger a new snapshot, make sure the new node is existed in snapshot sent later. + s.triggerSnapshot() + } resp.RetCode = 0 resp.RetMsg = "ok" } From 05e0d174c609879d19a99cfcdeea8944f40c3031 Mon Sep 17 00:00:00 2001 From: Jason Joo Date: Tue, 13 Oct 2020 11:15:53 +0800 Subject: [PATCH 24/46] Polish documentation Signed-off-by: Jason Joo --- README.md | 7 +++++++ docs/architecture.png | Bin 64317 -> 96111 bytes 2 files changed, 7 insertions(+) diff --git a/README.md b/README.md index 8a9108e..c55a438 100644 --- a/README.md +++ b/README.md @@ -53,6 +53,13 @@ You would get four binaries (metad, metad-ctl, influxd, influxd-ctl) if successfully. You can put them into PATH or package them into private repository. +TIPS for proxy: + +```shell +export GO111MODULE=on +export GOPROXY=https://goproxy.io +``` + ## Boot Meta Cluster ### Configuration diff --git a/docs/architecture.png b/docs/architecture.png index 9ff51914177b4534bc78337efb6c643197f54549..982268936744a9b660aa2e9b966e5e40e07ba029 100644 GIT binary patch literal 96111 zcmeFZzwAqFThbVx`m-5n}Q*B~H`ba&^o zMsRQLd*6S+`{Dhvk3-#a&AQe);}_?efX50l*f&UST)A=u8!9WQeB}yS$dxNs?_gX5 ze3={}W7!`(X2J7dk2irO%cB`JaX! zG+FYE!T;k!xcr~5w9tr>{`WIiP$41vH~;e}@MJXBhR)Q6ju)u^Spx3*d+@Zb`#c0v z|6Ly{F{SJ6io1&c$0HN*C|##x(@j$TyA1IOunttdDa`-WdXBd}HK2{r|DQT^OOlo9)V6O8;F5 zr5#F`jBoq~+W*{jwxYF|fAKIne{Qu;k*%|o;-Q#PGV$qtOOY-K; zUl}f^*cm9b=nK1sO(J`8xYIGPS-*SY;Nc-Wa{2rx@IjcdYRG0F<6Tu%f4I!L$gGF@ zZ0TsNK7T%y&`Yl)jwfBG+WBT^=;VmqJHJ zpS3hF_&8%on*XiJY3Yyv8W0eG$biL9iF{3>xs*IUFc5cxMpPJAK_V6?C#R)J>Uy6z z#q!xEdK}C|H!TY?F+G}`oP-DpR<$$J(ajc&50~2}-6kY#Ssks$aX;|#p6g7Qy)#m% z8BPbAuG#7g2Y=rdYno_2^K${~a(3S*qI5OlU>KzWU;q1dx0_V4Ne5>DO|G6}iSuR) zeH&xuhiAIETIEr1-(tvp>rZ{2t)3sHmi;6)pJbp%vniM?T_%Ja@$unxIm2YG%b8q& ztO`8A+kRjJ{nhkWhDzxS((=KgzdbYWNs)>*QVJsBksw){?TCj_w?xY$669vIz&6he zS>{JF=T%Ur6>{9Uyr-mEKGS$fxZpb)zMzhKYz0B@)Sm1Pzv)PMKJ~-?3p#Q0plK3| z)8fyb#)$)D(sA5^8;u0aefbIG2(3vE zC?p|2UoBsE6cP#|$|Tb+0R7VGDk4j4BPSOW4cD%!;rL(+HfgrhpL_ag9%ddVZ$4D= zsW;Q5abN6kKJ}Z0KzdqQ3`98#%c~kc28xV_i-dUm0OSIp|D-kevv>vC>27&_OLl&p zrpbO<8l>`gv+Wz9a`O+JVw0fuSS}e}n~`}66)X!&R7MUC4x{JnxPPAmo~Vo5pqW0? zRSZgfOJ73fCo!YadY(IxVx4X~i!EyEI<(&V-zcoesw2OcQ(40(J$DC9jjL*>f`z7| zv}_9P84e{b^+8`6@P3nq$Q1qcn39c6k=%W`kdrSo!Kq(kHbyf+(8=n~^Yw>^lkeE3 zJNU*UYTfq^uP^;7Onpx#^du!E#gF0&&86l!p#ztlZ=!@!w)kcOGIR(I4XHu;@c@4lk&hU8mn{X3W9zJ?w&8h6Xi_juwBKTtu+ z+Yjit1dIu0DwC|98b#kZ~xhFAD`>UzM~J1 zrAEczf2e=3IYaln#IWfWf!)-PpmfC~(G;g(vo}~C>tC*Ku2!!do>B7fsI6Ack!M}* zvklL{`w+cEkbwhueAgEi&)bsVGR6o3fGb2sMkc;d3FJCcZ-Qx}>r!s{^i+u0ZWo`3 zh>lJ53U60?(&cuJdjmY`<71PX;7up)pjNxA@dl(Cw?)OefKt!_3DzvP3DQDGGwFz< zbsOm#;N;|-u3E~=ZV0cr9FAyA)Z&|2)Q@VegBZm~@gXl|^29y~;u~YgO!9KWm2uP? z$gkN9ysuOBQ>+k9BhyN#uKl|%fRpCZ{(hwdJ^tqW%>b}l`)K4|`v$ISwR^{uB5vEE zjg68%sJFXOr&KcJ;eFZaM@b%wpE4^xvWm-$1^>Id4REl#85Z~sBK;k8RW!e0sG>79+cBq(S# z%moJDuJ!0hU%y4lj*%eb64l09-Cj*e)Zqbko6dK-x-d^YUzgLYoARAo=Dka=Q^N?t zC4Q+h2Tz=`9aTd^BP=P&HiaLg_)M_SDu#pP>RqpvFj`K}L&r1!421-yjg5`fSW2!3 z|E2+DBS38uH#_aX+I+@t^4d*^i#6t!k1=H?IFqmw-KYT>s{Hh+nOn{Nox+Ftq@N&6 zSfNnpY8<5a@**UWi$LpOM=s*V^vq1ld{64hox6AMBC?cd?EEO%O*=_7D(pTkXt+3o zNMylC?WVm{7*+w6oj&C;c?}8!f$!wxGZo0N%e^~NMK7z_L9|7I65nE5e`0d2>S(o^ zmW73-;w3`p(&t%kgU^4JGJXU;uju>X1L?uh_?e9#XjX|M?rXKmxtcug*OtPvL=Wd_ zT#r`DJFHStQ;qv`G?eSTJfqu#%`epfToPokKR3CJhPz(TFjQ(e17dlL*>P=5XQUk&|&ucBg z$4?XcY!H+ZE_|0U*UXU@Q$bmmUcg5j9Y9(&+`w{V6^|tBzE=;>juH7ld>=ornnp{j znEK<#`io@dLVF_)GY4}?`5vai$b`ChhEEf+{8)X7Wd*HUmr_Vb>-3<@JI+_gd6@w~ z9&Arsatb9|-FqT#C3*&{BbA7X=Y}EK?4Fn6nu!i%`ZI+ZB-Y3(Y2X>qrn$#w!`mZRwR+Auj|%qKwXKB>r5o)v&OgU=--+zy$ru3 zz)}y10@*=RaYaSeNMUetbatc9(}Y6IeiL@<$W$H4YFSz3)PVH(js3Rg63wym0TlB3 zeS9ivA0ICC!@V^&J&%o7n7Wo^<{dPe`{*UB@P07_uh;PlfwSYSZxpxjE`^f5A&5j? zT~djMS5W3|0Du|!DKoV^XTjRCWF2dIZtmS!O$N((^GRI4E`@e!(Z8O8MF$bT=>~wU z64kOl2c9V69*4&aDII9fVXa=~Y-w$^AG{fR>z_Ehg4%y0 zRJf==d<@yvXdXX)OnLSCE$(_zaNM*aB4l^tF>|;P0V><6XP- zfL%}*jJlktm;eH*Xb@Vfp$QS*d>bnTfM&-AL!1a2ukDzX1;~Vjv~Z)@_SjvF_Y9YM zXu&72ck!$@4XAv4iqSI^lUTcix8BD}zPUvP5fB(0B(PW7cHdowE2T=u-u>R#*bHjT zT*@E^;iXi_0j15TmQe)BjyFdfx>}g?Yo0+^>yPcBj~+>Umqx8(wmLgKp-VZ~nv(;t z)9N%Lx{Hm~&%PS9ylc5ihwyM$|T*tR}WUC;9mRx%%{ zc%GH1@o8hsV+(e$T-?=O40eBWO#c${wA^Gv~yE1Nu9Qsr1HCAZb;tOtvvki<=++$Ld=EL{lB)GK;+ z>~Jtpe@grC_08w@Cn)-+1Y;PQ#WgUbxP zGy#W*Gy?2P(4G%uhW*rMPI^EPB`4mo)v*gXb5}mDGgoeoCanU1Suz~6&bgH_v-z`H zcDooTbn1Y%m_UVno$YI;;k%gw{bC@E>=97<*dh6u;jvoxnKu+(p`ZkUS_x}?I&W=U zf3h2YU$^Gz=GIoa_t`0;z~Es(U|?7rk7d-tqj1Z(w7}qCUYXg}GuLE??8T!}^1gev zH3n6ci=VTN)+fH0%>4pi31MYj%|h?=gr?Qh2)l0g$lzc-43~9V4)Rx4Q-3@6Z=qU- z04Su8{op;kRO^y#b`d>Vdjw^L{>W}l}ey@l0UGk;{M6ADvAxjCy#2mkd6PDBd*FALx7tT>S7g8W+sI@<2sg?A4lm4@Tph^$ zAT5B}Gg7onl>`D%_TkPFacB?UxCafOCA?kyRIX_o79G?HOD*(C_d$8$dR2Ch$HK^Y zro?Jcxy)vi=?vtoJf|#ySe%_wagz7xAmka*{il?*ZaZP%S@EM%hs)7=UW{C(o!oq} z3rg&xm1#~<+qL7KdkKS|(mda!s`0p{ISh1#kPr5v#h(sE+X;_(a2I9>C7AuYWDI^O zP_^Af$23hD!dQ4e>fu(>nbnxZoZnzw z<$P!CT%x-INY&>*HO1=oI%7D#k(#^De|!@hFDw`pIfk0)6om?FS8J@NUSrf@uLfkx zILv_1wJv3ivG!q0^wT&Ym%Ov$^rrn_)zINR*S;>AR~5{kJC6RX6@IjUqC73yes=}6 z47YnasGkEd%9bAApRhq2=usM&bMvV!-*BC0^}^lC+P3a_fvN@jb)XyxQj$VCz} zG0CjmNLP9l#UcTPX$vM3ng++pu@lJM^O5JB^!qzXAE)6&!~JnGRac=$gFyDFPZB%b z2(j;l@x6-s{(cXD%U07QPZKXsaP$Y3+^=Y3lwW?%7hHP&fTuSDDlfv+tW<87e{1R= zz&S<;!1HLS=`04nrZ@qO+ijRjhgmL}9<$CbO)T!Cxc;r4QOUYdDb05bF|ef}GaU+R zmr*D6r-2Z4Zr8M?HYBL_4wHj9&B=jiY`?{ntEO7*Y^}2HHdfYyH0KH_mr?a`+bnbC*4nTc@ zb(iDQ{~^G2{6Fsk$%cs~`Fm6;Mln(4E>EtW$|-NjBOLvnmq>^pWK%L~-^N1soN)08 ze7dJIL1^YvW@4C%qDWu8(9%q_9xt?#>f!4c9qIJp2u2m7(JH4V82Q32lG;{XEIS}B zbr>IZ4?z!21x+fP$wdGzDso=YB#7>WhHE*SSJ}87CD4s;XT`6Ix3;u!>|}Y!j5xaB zU#ic(k3qw{5y?FT=nyUH!NEaTWTaX~{_?VwLws~X9Fczi(^9pxsK@Nj#B?`hvD~}j zc;vRT$pb8Y6@0{e=rTEuK6PXZb;zJS-E zkRTXqX!Be4=@TQ-KScao!Uebz67r$kMDo24J-VYQoID<(a|dh-*R{irLb{-B=@Na4 zhgi2y0P!Il{gS$}*~W$=XWterjDU0isms6_CG^RN3Ei6mLi*N z-NEcVhod2&lNux*0IVlyy-2jPyM4>fY<&+(JRSPq@GS4X2}THhrnetf)Dfn@J>we4(ciK8i1$Zuk@JJ^A(CexP7N z6%@*TW>Y5|gvq3jXD4d9%er8gcYJ)uS~XvgYJRA#tDC5S!(x}q6R25gVeDLNrVPb0 zk5(WzKcnzi_Z4`S9`MOBuTmPc9^r%2qqUjZy)iC5&sSxor*OJ6q_Wu4(_;W7Ul}Zx z0r7AAl-o4}%8|*J8=mG=3-D7gKOsw1E+zyYs-R~+Df5?vdGSPw~ z*qN|HV`I&OKMK^E7aHnyP{JP1=ivg}{3K7?JkZEePF|k-d)DYLN4~IC%3jGZ@;zbY zUNWd#PH6nd$UdgN!r#{7;{=|YQ=A6y3V)yvxwjjToa9l=$8V)>+V&mZ*Q}S#XDHu# zUJ}S!MBazclo%r6v48;ifY38y4_!llWAghZ5Qbxw@+u~8xGonq`3XPUuNHek4aG{> zTkK;#et?hiPv`Zq2K0B(*&O30iYeB-!u3&Oz_Z_{uDY$$zIV)2l92E#rq^b5gM~%b zp{#yfu)kRSX)2Bw55L6e#^%PM=Cb1up0JCU~8F zd63;RsD~Y{PoDJS9XUCC%{ul6Dur{8<#Oo)D@24^3acJSGMRjh=0FFYoA$$E#qfS? z5GYOnogpZX6GueTg?qrnn%o@j{rl{Oc<|_&%GGuf-{c!+AICtI{3-T&%q+dyPS9nQ zG>UoYrQo2Uqo~uOs_Sg5X}i$&4foL*39S4E00WG_`eV|)1yXR_B%*`{%WP9t53qK2 zzIpTBd&4EtVPnp+fXQ7P_IfaYv}-v$TvC+tsDSzscrFBK){%(CaU-IMW_L8TYTZh; z-B(IYhTKMLiLMUhm#kyXTYDWM`a$nULb-3m?;lF{8(JR@vW&jQp|Cl=l!0Tops55o znM7ecrAz*vzcU}pb{7C3Jw$WS1MQzm78akgVtGeh(}Y6JeiNJ%*x?us!)at;(PO`L zS0h{o;7}%&>Sd5e1fxoq*Zmr-+ct-*STea4RkKM}PxDC9h0F|YInJ8O3n*1j zXEk4pV_UH%7FlOzI0cuC!FgeBUSQ42pfe5suq(tUo&vGe@Ka(0cZ^{6;-WXj`QhXiUpkE zzz}AH)FAcIq2exs3oBL@x^)X2yfA8~1;i_eW9ab$P#}>npd$5n;qNQ}kX^uOr|M@R z=uvOscE>@g?#+I~5sUbwk{O5kbtI0c=GD?jW%2SzW$(>~p3(^cirhKS9$Six+uA9{ z9lAsi;I+E1|1QMC!oupUt zO~a3%doYBo<(-2-{g8Wn3fz?suo8V%8`lZ%{?+on+t)#~V$X{aUuMC`gGEWC)N*PbRQsy9n{uS0oY7 zj?;$$;bLUaqk1sr|5r5hN48tD+Pm1kp_ru1<`FS548U9^IC+GXApZP$q2p42M?IEw z$PHcRLF12D4Tdxe*p_aPt@3d%E-nh;7X?TOlU$`>&#^SCY*iAG?`E}vbZo-H@ZGP& z9RF^d&vj7PUF~{ei>lFONe10LcjBjHC_H~sOoNJ@1~lT`TSFgv7FbhXVG+m8D+;Y} z$8s3HerP&*THwi7e1Ja$O0E&}rBX`{HfFi-mZUELX+SP0VSKGqSJ>N(uiGXFRaD&5 zGxP$0wHX|&t;V>c<70gQh&TZk6>ADt6lL^2Xw#c`9^*q7`YT|%K#E5DqH@@!gA7+^ zWe(r`?9<=7(f3p7YMAuyF~sBaU~cwTa{egjeI}+iI|O|Ju-DlB{K6VAtFVLwYn{36 z+x94W_|P-2-qFb0{kIPA*(ooA^97i{fH#RajLt-cTx?YtWlI=r>yCa3)lE*BmeG%XjMnLOR&=q;+QY-%CNfgjm zkLEr#aVNShGCF5eSk1VsjalC?ISQk2gA@W{il}n3jPBs(??qPE`R8)yPJvcWIRI zwRLHc$YHOJ$;r`PG!XCNd`g1f(S-uZ$0X#6HOY4^+ge&+kQa&x!oH3ILxVRcsz8FV z0BDbuhl|-Ap)apthvn&1zqWtz;~$Gj8}Q*~wYC3mi5bwzO6Mb{a<{+gQ2+CkXE(xiaf{? z(e!^^YBXRtFtD?Y1&KdK=Qf|NmSV}e+EZzbj?~-2#Sy7|Lz*RK2F^*PR)dH#jyE&~ ze*||A7Ft>)Z`_tHOtGZY4Gig6ybFJOL-NrRBKzv*jmh zD%E3MePH8+#m6%X2nZOT$`Z(s8c%H@2PFRVA64Xt)%V;N^r%soW(e7L21^5cV=umW zOLi2Y%r|eK#!(_MS6lm3PH|_L&1e-A7~G^{^{w)M=MNSN1531Z=6Jw-t2+7_E?%E&s~jJ0UR(D@y9gg zXYcupMY_Nczbb(kJ^-Sw)(*-%%HJ=tLyGCqtoM71k{VK^-WeV5Z_F;vBUor?9wHEk zWny-`L1}R@zTONNGl8PAa+tQ)!Aw~I$5Y=Njp84W;70?w=D2f*l-j%1`7k< zQDuZ$GhUVKfNzve`8OCMfS}PxkLpW_HP6R{G=uMQnMB-$)+_ZG-p0hld=TujI!c-B z6zWKfD6<|u?iHAcP-*8V1Ff~nAn;Ac@6-^{FwmN++t-%zm|>)SSxVm`9ZZxz5_At^ z@2pL1Vx9yL(1(fN80BQg{|g@eS~Lkj>kINI4f5c68xX+J^z)bXbUP2W=}+VkiSX(E zREqk~Xd9%r`oa73z#dfW*IdGKKl-(1iCDGGbOrVFvUO{#pd>n=`A5BfNDoZzlH}bq zXn0yyRx>1s+g0*k{La8EzWG)W4fO#o|Bwxhh(%h{Ph`1fXVP(#(qH`fX|tWHi7dG@b-wOiP1M=$|ZuowGN8AQ9C!C&Cof)m{C%H&sgF6a0+RLbZ=NR3Cbn}1l zErE6u0h~Ks8^DOF6<_XaGbCd|MlogneF1PvreHn_Ex2@8cFAHsNR%5bhiD(D95bZ@ zy5Yyi$KR@KMk)%S)JTa|Ia7fw`%Rv*H`1rTq=ETG+R`U^6hTQAn5n5#9!;&YqpIO` zkIF1dd=8}Sx|l`u(L(#>Ivo{hL~cObJ>BDja@7bp2y{fb){Qo)1#It3QTKnpp#a51 z@36z>pg%5abt z#(_jFZdX-jI&!uwl8L%8f1nHSXrT^TGFt%}GdDC92!F4kK?ZpIbv~;Sj}#S&MdC)v zZS}VP4Rc-OkY{A1N#X;-tS7l~KnaiIC|PGhFMQbwnS)?}%|S{QX17n$rSK^MjN^tOc9^vo+8Z z8s5LaLIoD1-w-HTr?c@J&@=xwfmU$rj* zUnv?eV&3VGNCyJsEgPUbULrd`RUVRGy-NL<5Je;|r`&k4UZ#%MdHLzW&rd39RWlm@ z^maTDfux1asYp`kuJsCk48K$n&g*&%|7P`jm3@8l&2 z6a3u?Mc~P7P8V@4h@<~0qzEtz#lJ98>5#0m?)B%DqHmaSfU%|p^?z{DgfwXRHri^l zMzKEq6%}WvM;*a5dX{4~mF)>a14SJ37|8P3K!ysSYl7@Qx!GV1HfE$OrM=1CH{PZS z`XuM6!C~JJ%i!SPY!>P{!u8!mtuwT0i)Q2VZnU=YkiOuC4cN$!76i6anW5t0;T4&7 zk)JF3+zSKwdfh)#4F4?E1p~>cj`t)8WM5v#i2~kjmEp7Yzaz@Rp!75s%XKw|&_6!p z4(zsei;Jsi+uYj?{ zRBeN4D~t*)WSc@lNN4hcIC4FAkd*>IW>?cqozHt?1>hHO%jl>8HEHpXPv(=d^@#J= z$qHi6XJ3*=|# zu)vr~tf7^Zl+0@Qc+03OGstSD!Gn8K6tJech zLu&HVGcsDcKR$fPLF##40b>c0m=5^@Vuu2s+h3Lr7@zj65+ZhRUn$)N=k}-g2sB6+5^LtH* zj!xR%-k$$6(QRwNKLhFfQ9k_-Kyl&&J-A+mfL_6950B#R`3~R~NzwXk>y*EHDnwO{4~86nxnzd&J#~{{0l`K2pn86-GC}ZB$d)%WC`I{+ZrI z3%+j$?Bv0EZ!zJ&VdY~D2>hT4BSzyRU(oC{kxEL{Pl+@)`Jj7!Ikz*|kUQY3f5QwF z8d3o2lwXI^lD@nxQTLi`C69|E*(B3^u1g{O?qB>(7_4I;=YRveWW8$cI)?G~w-g*b$szO5Nf zBf{l!aXG-}MyU9t!(TV{BZ%nRkF0-uE)rj8gy$E7Au*`De2)G2#o&=oIGQY_fO+X# z!t-}9MJi~!hpw!LE#dU<>O5GuxO%k><Iq=nHtAGgPjxNV8ehpBL5?Vg3YigIP%x8r`J3ELjOuMo#5lkHG+-KG2%u> zY0G1^!`d1jA|m`Rre_n0LEnk+IynqD^`R9NMcS?6s4ZJbH?EwQ!2)D`EKS@dk_2h8 z&|&enQSd`TqoYlorw&OlA);g#yRBae5;I$BfsFwb672W&QRbh&XT_uRoeq0tL~{Or z{Za)_gb}|kxP=+YKiPY}iwzs-vXo@wZ(S}DO?(EaWOW8*Nl%G`5Pj2$UUv*O?;Q* zI4Mh?6c@(?sM*eks|3>LA#}qRk4Cx~^k*oghA))j{JsAQEzYs0(=FjrdUa7BKHNu^ z4!IYN=OZ;fH0Z7Lp~&0>Zm0dUHU^jg{c{Rm0q|q+*nNLo?L)58t0Mym-j_Y!DE(UT z1GF$OW@Tvg_hiJAQ^4|=n7$wRQ9l-a4(gt2hV;Y7*V0waA0|u}nwZE628aPzrvV+9 z&ULzQ$F%Bvk*aEdv#FuG)*(42c1T*oBgzthBTu$G5P+q5M< z<^yEc40?$EKfIg?Z5J>we)#l0g z+zr!JG8LMmP)IycZ^l zL50^%CKD$PIDwA?Isn|QX!+%|;+qqP&&N^rmZP90n7y(phk_iW5G2MK{EmK34eA&2FaG+(9EO4;qa8wU{LvS$x%)+9_wjZ=6JNVwGnul_xgv_<$HuR z@4Y8CWMpF6bom%SRm)N6(L2}M+uL@$+hg;Dl+We^^!`!cwnY-;!`CeV168!`eNAcU z*Yow!z{f!Or-!YB7@GWstBL|6G2eU(PVyrm7?J?y5Y1As-^Yh<4qq~`;tlFihwusY=Q#gh~OxYt@zG4MvUvhLlC_R^rF zc@qUA7ukB2_kkpdvX5GldPfZg`5Y%Z9^Y&9ee>%SUD|g1&_Zu!JX%a0(8g?nf7)@W znu)2upCv;9hS69o2iL`2J6}U!Z%l3(REm3f)#W@blz4sfuEb=sy^i2^Q3IH4G6L#f ziXKmFN|T(ugF_qSaUjmyWfr2!gV(AW2c!040}H0KtuGa)WBhyS!d_xhS-+CI0Xh0o z6p;A7$sY32deY!0mmVp-r8?PSaUbIbVr%yyCgb?kr1rLO#j&<{WuciV;t3B@O%7-(T%bN=i@P9}s1jsr7@ z@iesQ!-i81kkfX_gWRXnvylHh>(qzK<^vck?p= z**#@mm@bRnW~Fz#G<{?2)3)-O5oj1+?J{2F;2N=PE(1IcZoj^?7A=qgE}5Xk(?vPH zQ3evv{2Hhdly<&AZEQo)D-&36a{bFNw2=^;7R&sN`8;0@vefA9WXn7%>BMY$v|J$A zsBek_yJ4O{ISNbD^DOn>ZvMq~BJ?{>w`T085l)7!w=Gen**DS^h8+09Jq->GYbvH@c(loY#ZYr)3^r7 z#;PrC589pPdVu^5LtQZAW7ZI5AfwiI0K_?Pdp=@qKVlU!l$&2m-FqquHK}(qBZj4B zbQtd!_dfLUZ=a1od6OwK>DUz3Z_8AFRSwJ9RP1rtl?w9b@j@;(4O8pj@(vhg>Op$U z;QAHu#sQk)U6SP#g4ee8 zA{u%EwblmdIVXZ0HH!ATSq!U=jN2oG6%Y6git18E<7${a+q72}uZPR7)xenvk~(%3 zrNXBGshg7KqrcC}p-Goq`VGkGuqFSKCA~9hxJ4{0**Q(p_ePH9>e0wZW(E9dcAk{C)N}7^-BbQ?2n2tqEPB5zfKj?LeG=(j##QqeqiMS(?F!U|GSsCxreg%M=`pv?7if?{6{5A+kN-(sB~6zFZh`kK zc+1GdKZz2W#cL25XlJP)Fj#57&-d!$jzM#h!sY|Aa?C zOlH2IDb!LORq3q(f4z z&vqu-Tu7}~KF`>9qC@RF51^z{Q^kg8trWJsTeQ#9Gb;u> zFHm43PZ{;9Y`81v$lw&bZ&V=-XpY--vpVh*`u?R^$I9bFFJl{NhYtVPw=<}2 zmSf1epg21dt2N>%GjyUch)9l-eK|y}l&-L6&zwGU?0824Ws%|)Zv4mduPO)tjX3_k zzsgrIyzm-1U*n(6cWY*LR*=?lQgWgK)eXBk|4vR_rSZ}W*XhB9CeL6#%E!830>nMg z848NPvA1e{l;+QxhC@W{Cougj*I|7MN+S#O3}=AN7#* zG3Z6p+CSh=#BtaSxU}mJ8kA5E2^_5JQYC4xOO~Bb7g>iiPh(CQjeS@!X~K{}ke22I z-`J&?l4+*jnvDw3uw6~%4NHxJ>q|+7>?$OAv46?)&QyqD|FSHfaiInH90B}YX$yPF zO9L1*JdDotM0nWHkC8eaHzAT{->9JvtN#`Lgg_cgeC-YWsjZzc8Wu%*j58ec0nzN*SG_3 zFXT70e9Ik7bTIc5N~e?*7WPaRTppw~G&wr$6O`WN!Qx8eZpL>qA1*-r zV*ujOKh_-b=HjF_^#QTa3?nQo?D*{Eyc)sm3aM{GeSp;<%jV!25;iziPJ|gP*g^TA zW^H`GJ2j+ir9TUWlP`xq3sycP@ji3gDxQeYSi*@a5!vp32nqgBWY&0or$-$5%LAwl zo0D2?6zZUwv!1EUf5-9bn9FH$cYN~$Z~`MRc90R(6D2}jy+?jSyB;&2fXs3$7MC3L zI{J25S#)A&#Ec&CKGbF@p7rXH0~93*&m9W@YFS|I*S>@ZUhLb2lQia=^J*HT8t=Q z#KW`5YP3davP+&DO0#$u{b9W53R6K{fw=T77~k)OTr0o>XabA|zHdI&Ce_>e<;SJ- z=a~b@B6)&;UGzb4Hu6^lz7Td#nb$3~;+P@K%9}H&;gk6Aue8@KzlL((SS9JM7D1;A zZVmmO)WJsZq5o{sgx5VUdLU(TKW^a3hl4-uv_b<={pHYF5-CDog*p}1q%d32C3iQoPoj8fE)jxa-{K~pdx)Np;x?ODeaC`YE&;E~axlKp8^x zVlTcwA1<>xZ$nNhc5?L-fePQc!V79jFxbFYXYF2$m9InwpXH$DNkv_J%}1kO*#NAL z<8p&r!96xM@*2;a^4r3oJ{DgJXd_|5V>ke&R-mKw} zBH^*NxnVHO0afmEeX{EsCKDOCHS18nI!KGN4TX@WLSg7jcjtkaSs{% za;-MamS(YwygD9z8c2pWCcOBpny)mK_1k7_{iW4lg~IO2I5SMajIl#A@jhHhM)Yc_ zWmDJYw^rM9o0Hdwq8;v!HN)u{g|M0h0pr=UmQ=vQsw@)aehLI!=q{uBIKOz_{PW`Z`lGrsr!8sMjh5@iedT(Q0jcI&PwclwLz>M^$p!6;Y}a&S zcSjgB_8-)p)plqlP>b}wY2E*QwS6rz&sDXD|HkQ7qP*L&mxJU+zKBg!EL$)Ej3MD9 z^PTa;dS=Mdw|lpUyR|USIK2Gl5Bmdx;e)oFkw37rOq)z;xl!E~9sunK=>WLxF1Pfw zbQf^lIB&I4ARdY$dIuBCC$qNs;GK!sJ;NH}-uSuSe|ho*4Y|mMZC@{1AuY;%h6Qmg zv25aXO*<Bxk%7a?fai7z-vbi^xVyAJ*M9@%j#!&bO01ywSU9P3T>x z4)j|tU!#*e8N^O1be}EuM=OP!9AW7zHj~`fJKcNuNvSyM_*1W5`hzKg>FhHvhWP$m z__)`o3Z3h1S<<>`r5zHU^oket5x0roogQvGoXT%z1tv+3zWO=u&+l-Pn{`)T1VV!P z5f`Y3w?)dzz%8D~H=FKUh#>lfpcuY;kT66hRg_R=-}zohKtKa|e;=m~ki7S?oa&2` zTM1TDPq;&;dw$j%w=}KQP<3dPM&WhU&Wc4+@II%}p+v znQ}dEI8~}1?x=Mdj6iH}>{va#ae>b&2jX0#TBlX@G_j&S1j>r-zDa=){4Kww3zIb# z`=$7V^{7aGlOdGfqlIQA+3EPMz0UYeft?W3cz@LnO(!OaMt3HL$JJG`@1DZlN+q}B z-E#wO6U!(a-q_qIdxUYDm{<;qsU^?+po!-E{rr(4+0%n>LKS0voZwcpiu=Up?7SPc z?C)0@eeV`;PPa61c6?x4h_blxES&%Iz6>RJ&179@hDmoHA2ui@QMIuvgLr_2uc^z1 zXsyAStM)|i`-n&h8P5eL8K2eFa?&0dXq=t6Af}?KJCw=?Rg%KX&kn*MANq7Tr4^aB za#ls%SeGUuN9qhMeH)boUAKQt3n2FWi3pp1Cq&|#Y0DV>^js*)N^ZFQv!A+&8_f?e z#}M!%6Sz663x5%jAM(aKMwY^YJy>xH8**4IgB%4L*bB?dL>lEPM#F5+7Jtq6#6HP{ zpgG8@@9Ok`8xKk4t=(hM6a&dB(Uvo>PjO=Q%ym-Ab>pe^JT~i~f|0iBG0iFv8B)*M?bcn+*{+V&dSx>ZN>!pK*~%h{5V$HCGdVl!0T6qLlLsxwi`=P*;+ z^5*njDEL(i8tM1AzzPPFMCYTt7m+nV3Ybx+2D2PZAI-x7ZHJ(G#ayxQFlA?Tzx1n! zSbFr5GWZ%syQ2Foy5K;8+{ZIr5>nwZQbOfzq4!uhrV`vAn5u-&%jV9sWX3{5={(8n z_nr(EuF!9d`IoJ9te9)s&cCd=TV#GhvOybidiv8&WB-d*1}mcBO%hXY?pO`4MK|{8 zG~6rojx1?9mPtr#n7T2WAb(xMklL$F#J;>nxm)E~$Kh4r%D<4m3Tf6I`a_4BHelA|(SKY4uxJ`yx8JlaG zdcEbw%r7@=$Wi28>-JISviauWcvJlN7MT_n?<0?;TWK{5;nO2q;Ser|H13o} z)qSFGGc-$uJb|Xw9`IOlWU2DWMq+de0cO#=qprZ}qXl}9Mu=ZW z=@V0X(>YtLVL}uFF~>c-zt1|t)aisxyjJAM&2!Uos#Rd4Vb#*YU{t_`+dJ=jS(Aue zbu8~3f6bO#K{bKU{bA=uecIF8od@|G&p(II*To~Z@5Vt zf*SC08vcrc2&?KdIivkE5%^(g<8gJam{qsdpc#g$T5@u7!_R(1S1<6PvxmsKE-Vaj zTwrdY;|+A`$Vf{c<}a5yd}y2%_6^c#?P$7I;B;@Huq4}NnQiGw ze&~ViNin!p+Te@tx)n)0J<6BSQdjE^qbN-^oSv|Jg8Ov!Y92S}TV5ytcsduQ-B`ZV z{8GiV^#ReZ3OS>1tnlwxb8hJ!t9Pd=FaIqH2ee7Zfwa`Z~3)T5l0tlhVhiPApfaR z$1ONny=EWH3CNI9G;YRY#6I3;+v-apsb|lg8G&Ce@F=z|e`Wtzxax=jTw*Vog_j|J zW`Owbgzz$QlE=#u|L0prQy+2-Fu|wh3oP`W$zO`1@rL*p%6m1qLf_Y@bhqeu3AH6L zf*HC(Fd|}LLhR@ZZoKxKC7NyZiKe`8?jF4R$9B5n>w^pj`Pt>`znrF*cZwfw$FY+L zI%YAJ<*mr7Hw;rLJOl3W1m}K_lz>4?5kLH(wHw;bWazI3R1z#E8CY0g+gZN(I(E>a5ka%GOwyS9s2HU9 zSwvL!%AWAor8uXFrzZ_yK12c)uUl*2(Lh(6hy&oasFkRn|Ih7 zeiC$#QiC335c1~A2bT498k*^q<4P0!fxV7Ux%@{CyZ&#1Pogcs=^f}6e&$Pj{XS2=bfs!(DZEbBr zw`nCdtmET%&3sEPE&e$-7hK)1m49+Q_|`Cj$3UNqUP>R_$zGq0EOxYz#{>a5kG`Wh znp%Q6=d+GDa69dvTY1Y=;4sOTa7!4#C<0(IKu6%ThcvzOd%dz5%tsiIGrq71l_9Ns#Y-hNJ8(v%rm| z!hKr(`LRh|3`_a{qv|cgs_eS1VY;Qe8>G8qOLqw(B^}Zo($d}1As{6w4N6H%OE)Nj zbSVw*+LwAi-}j$`eXg1@=9pt*j-`5@xJ)3;`{6&=P$& zTMg7OEG#WE17iUzajEO^i`}WdCU1RxB34n+rNAQ6=5jp@6)zNT6>5%WGBGc|VwXh* ztWk^ib-l!d53>Vr6Q2kX* zu!o}ljOeD8#QP#2w7&ZKi}RW$tK9jgO;J;e!Sv8IENfcPuoEEEO?US?f&M!>=QlNW(-jPLzesQIZJFyl z$P{89^0Dp%zP3DrhH|F4cp*#bCO=~S1KAk4KpfPrjEjrAPsEI~pad-&G(|_4`~eG( zq1uq?RbuV&hD`-DIb#X3w0oZ6&X91l_gx@()@u3grq@vnIS;SAyZNn8s$NgXr0BQw zJ9$E2(yt06fbZ_UW}YEPWTrC#Hdyw&A8PUPvY{unu+w8{QttKyso_zRKLHpy2z`^j zNRYpCdR{b}^o39m&G) zrG56v<+FxR-i&Na zD-g#c0X^QCID(a}Hl`tbX~0fCO?01rno{h^8y}AlC0w~;``RYp-tNPC)!f#0Rdxc2 z@?M7VJDm}ysoMU4ml|&Qd|bvC={Ifa0qpUq4xO??8=0mC2Le`)iULT;UgIl-q`vpZpRZvJ}Y= zM2B4(et7&=x{+(AuX9PF7g%I)@u(DUdREIXC-Lz38QOW{ubuW-HPLT|io4*Gye79c z3d}~h(6wtAr%I9(_)v?vYC`WKtS^%8<|0x4+p%RpK^&Cb6NTzaWy!x%9s&e}RHFOTeHwMTq?!8LioPRa`Xt!iEJx$N#}sBpHfGn`Fj zCG7lamN-RWs%Gt$+<$)Fg{hHSG)ctB$6;{OjM^7PMh8#0@vVaUBi~+W{NMP%yt^a+ z&mR75EPX=O2=HYspsfNh%O_+Z@ww~;qVaseGMj@7d(TC#En_5kMdJuLa{}j!)oMEB ztDQc{YUw*Pj)v+UR4~+3InqAdb_|(v&p1uXB)1tl;VL#3mB(0Q%c8qdXF+VAnWtk= zy`cf2+j0+~g%gu!4gd8E?+rvyMNY0W9}sat>y7=d4johkyo~Uxt6?N#Al4KW8|yw9 zo1w&oFnfsYW3HCh`Z9(>z8f3UD8D_%Q|a(5;md=>?S*Mb-kL?JBB_UM6P-IOqR^Cj z=d<*=4kt5Q#rEH6ckS<=<=QB=n`h*#wJHaFI`w2iv+=JkrQBzdtYqP9w_;Mwl7SnF zLMJKC@R0Wab&AdlKp<4|^YcrTD>C?N=>8)G5czYs&jF&pIcnw7JyW{b)C{Y*cz8pKjEuPs{*HJymv9p#+zTvF{h~iV2WwX-Xoa@ z`DrkJ{)s*@p6SOYNh)r$P~qg7D!^&x0o42>8m>R=ON1WKq`@3}`3!TknvRu!WnnJd zh?Y#+SF~M}#)0d-V2NrfNanN{lkL=3%{)mPQBB#4Of*KW-X?v#T|qtd*JKa8s8Unp zLM;{u6k;Woo+?#E0HhVIr%x?+Cf*7}#dGzr7YxjT8tkpsDx7<))G_w!NH)? ztgb^^^0KmuGBTmcR-gGQaR*07`G3{yHYz0k60O=z3+Q2}eYtXRx_f3`eEfd=j>d*& z-_#*6QF*f|f|c#dzzp0{0_RJHECrU8>UM`(L(kE;Hwp3)zj2Nk5*LjV>D46!v?|4F{Umv6?6$#=+@4E;+U zmode^0wKGzPCLWGypNMW5m=$9#O25D#!ETcnafrbs+cuI_lqRTmz7UTF$HT*%Kz|C z{>rOS#%|1e;A7}mU+7)S@#hX9rqe$qU+PvC`Q@Z{dLrtSn%TWlT9clwemB`&U-5j0 z)oVrowOG~wYvtp!8dLMut3w=DE3=4bkDhjNR*mM8!IzeT1;U;j2#&d3Py`+*0L^+R zsQ=BMoejad2N*sakpadN13+c|Bi=I=G*8(Wq1;tefnpMg2%|ixBftI*3D#wbPeOF+ zJWRNqZtEPZZL*`#!Cpf9-O27i^Kf~H5U9&tgKAQZ9ls~N-{IV6hy5}J+ zo=+Hp8RmX+(#3Dyxq53Wt7khTtKS`iL7U%W4i3)?`b?-+Avk#fKvT3R>O-MVg)-?t zv&aaYKIDUOXF#ynn6D9AZKq7}F;f!37j$T4Uy*t;hw1nzP&G}DQ~Vp&ZCgrnMsL#G z(wwBc@`Jj1iulcYl4wGvTw7d)X^86)le8UmoNqyXd04S(iu^mL71yJY%!Lz#d4G2? zntSp;FJKP@(|XnB?|^-?>~GT&|(gBDp;J3Mb~1})UQiQ6n+gJ_w@|^#+e85CxG9s+vfhWhr|Ac4L#{Uo?Hv4 zh$TrA={P+Hifqm{=b;X15I``!LjBm3*Nl~>-1bg?hW{?~xyffN)~z8^f6I&NT&wSR z<^d^09M5T(rdP=}6>|8OQ@2JfHk%`MUm3m9$ue9M{H&Kky%>aOW*6Nnh<3?M>73aG z8fu`;kv|wa0Lxzy^m-WzNSFMPh7IzECQ{byEaXv)(0-eu=1p#q)Rs)kSJpx^hW2K0$FP-!U8`79Ak*CR!zKs`Cs_i_Z z7Ng^jA{4HB;oWbg?TfNAX0UeP`|4&N6a^1(h&BJv1lOg&v&S@?2E^R*X9w%5(tJl~ zJMnZNqy*^6UqE)?6;W?DuklCc4i$2Lo92-iige|6Ia$m+%^m5vYk5HDHt5kiU@$)~ zolNKY|FRm}!Lo&&#X+;n zVOJUIf-RKZ?&IuT;@9cgiV?`DtDh@%!1_K;dS7pdnx*@&PQm=^qB%Vl@rO?gI8z6ocrCxwAbVMt6qhI~>31q!421k9g63uY z;u##B`x4&vj~N*HjNu3FC-L6uz=59B4}Mp!Un#=stF+!x{RuFJp+%<&rq@ynnEI8} z;cWo7_(_;_x4cRzvo+x^&TE}B@>d^| zj_S8~_p2PF`1n{|B-Omc*=lK#7Cff;9srAG-5wa1`CqdEzOQ5c5eL0 zIb)jpGHf_5l>wUIU93t=7i)t0kq^{WaGEB~^OMzxoMQ!_k9i(6`)%z1iKRXoeP-bJ zvs7}Z+9~XpHvpgnk~oZ7s?>4hLxNJ6&jpmFb0h;hEG#Srb>>PGx9z^R>Wpfw9J>pc zP6PSddaTiPqTJf%A4XdLxB$}q4?dROa4s8V`3_NOu=#mE@)r^{$G=dG-KU^i52yOU zbrj|Q+Up;t%8rzAp()>%Ajcp3y`0Qz3jLH|Y*4jDZE8|ttMY$2`E1`n>jpOzl82Q|e&&XjoAevwy{PY@Gw)y=eyPbtNi&$5N ze)IX!2>UNuXV$k|tBVHGUp>dveb;yUqhG%`wlB~m5$sx8K*HU?$DoC@*8?UY>ow3_ z#Gw#z=PHvFwEFV_j9mz>j9r^aBl+>%=KbpvG&7TgxHw3UHnT%4BrF*pC3uoey6`u8 z15zH!t(}WFR*Fbn#s9Q(HEQjyJhB(?utcu{|MC{yDE`| zfT^6f;Sdkb^`1;;&6{w~!rBFiV4y`zLr*V5zu55d=|~p;m)|KrFf#w{M@AG(jJy?` z!wA>;p&sZw;4`{HA~9#y56qvZr~V$HLC?^8()evwGPlL$inb(9gJ76rH!=L~Ky3-~ z>{MlA;0puS!|UCW##eC$CWG5+gS;B?O1N;@x})7=#?nvbS`sM5^^>{h%Riz_a32*E z0ErwR?o8O;^nV8ue2Zst!u){2a8?^|_i6;lHC&b&G)*6sC<;)EzM%-7c3Y`2pZjK6`+faVtnvo+t^5q=Pfdp;#!aAw`P4A+1fU9!JT(E7VpT10} zEKnOo#?9tIVapRiE4`PXPX2Xrf#A4pqN)@)H^^Zd#a&kx@gCa;~FCBvZ z6nJfrS72im$LGs%$~2m;+w>$pSm8^(-B@YWPHp=2tJk6-EUIn^R4%r!znhaBIN-0N#3SW5H{Vh-^q+@`^ zO;C_AGymVqI#mw>9)Opx6}iGq=2!e6KuMjC*Po;$bYpJGQv->b>BhowizlCuP7iEP zvl)x%qJ42=G>@95&L4_K^}Iz&hz}A(chya17kK8KN0r@eW`v1XkSz$cX4lZ08$970 z;~`s{vw4UEPAISvZB{7kj$i(+mf8!fcIX%t3?iTeR+5?)h^f zT8AZpjoCle1)0fO_bJxrdp4Q%osNU&{72~Auly6d*#cE{F!EJj!O0iJY<@FO>w~OH zSaNwRI89*<(+ws=TFrt=Q`n@x70^?6$cNkmsCp)sq2a(#!}*Ngd&R!IuwPFpvZ4c8O~O6s$%^rZ_MjRzjG zvA=&~KL2=?QTc6ay_n9#2|1M&H$oDN3g2lASQ6-pZ`?*mcwnK|DaZ$<*4u(|3@HvO zuSzcBjQ?K-<6BupUW&Vk01K#47HH*X))b)D!`%W3~f!Pa7QX~&3ru?aWT zM_|-*_KBa;6P&G&6aLPg9w|P(*^bmo|CVyFFrFQ-KYL>Sl7&RY5~Q^3oM-Dmb{YEm&O&@9VN(y(o0iy z!yzW!v1<#u6m#DwWD!z`975F>ln7(PBbxAoAAPYlKbrM0^9Eg76Y=W0x)PN3CW+F|f&^`OV?Y^%sedlCsA1SJ!^F6Cn;V+Jdc zz54qWzl~G{@#3LClw)lIIcr|5;9&U*>f5(U^u*kjs1-8~HhIB5_j9MdW0^I86w9&v#uwI9@4I0$*Dal z`f-q!EV7*$)|K?h^wRX+;$#Ns+?by4OpXkOqaq+!%wjEn^x;)XW3l|a;A;EFP<9~{ z(*eKbZXfr1M1D3_&RU0}sIbua;?U^w8hQQq)Awk(A^fkqm{wj4)LYE7_J$m&eeMgk zb0!X++~IH=zcM?}ahmM3>om1F^Q0>Bcah&?f(*=MHKIyI^xx0sV^}A$!Y=yAh$apK zy{vf9RLjHt$7}&x(uD%Hn*>&Mn8VLPRjKXtRCR!%j9Fx*V%EQlIkkL{)9gFHbV&@O zFgI-eZyOeJ5ki##V+!qoIr@72Pr;t(+J{)-1o*C)noH5mVW;HhL*aSEZ@ZYcEJLdkT4uHycL0c zkrLXScpIX+{bh*z?~O=|26s{`iGPZdLbu4=&b~=CNQMqKVOqf4l347czrE;7L~E~y zu}3Q8w@uXRd$$+{`g!SZ$goi zadaCl;`vip=MC#`ehSleH&vULn-)GBpG@loxlPXbqU6k)XJvw_-<9dXG4}>~R=r%r zh)#>PN&!vj!HzG?|Ia$cLeI}-nSQTNewJREm35%IyE`&65}TXfaRKKpJn-@ROCjQ~ zxRGLBaPG?+ddnXeC+VWM1%)G;*{hFGlJ^=FI-c zewR`7WM@94@N&n-GRb;ovBu=>G1r?f4qbn&Sc#Amz94p z8FLueIG*g6$jHcNKDoef~K}(uBTqN#~h&}q)bw_ zdb`|KpImhok3U4qH^;-Qb-uaFFq(tsfAyV%a$x~JPY3oQ8lqCXODDZYaoZys{5mX% z#u_#~xoh?w())~>0aZb9@LY9ZL1`A=PzffAyX&(uq{siI-pAvBtCu+ELK>>KHQJeI zm#%dmJ>@bH;johFf_GnZh4F4oa8!@9*e5`g(AHE;ed~uw{+xeME7TftO!>rq1)~0S zHV&Xmz2z=!vkK7 zm4!3!Be4Y{Mv7F6(jVZG1{nR~uh9Iydv;pa7+n?AJO?*cVfM8r*(}I-4Uyxa`a1B~ zvp`9pIsNX8z4q*>4sd6|C3vv7S<`ZG(x0}1&{MO#OWeZ*=UvfP0y z0g40W+q2R^0ay2pz>^rjEXn6ty_@jGF{95?J@$%k6SEsT? zOA8pXcDQr__$n*7jyM<*AZ3zBX$XT$1w#}7s{pBxghA1-Ce8MwUlB!24L~fL(3%N& zkXfJ5R?HMrZP4X#_1v>#{QiciL4-MS5G4gK8@ZC#6EEL+!Wtvayc|6D#5gRHgxz?SGPRe8`OpYJxEX}Y75$gnd znZ_S>72s7ENq-@O(|oAts8;W`$9=ccVn}11RxllalbD5|;+C8`l=J!MHf*xd*jB-RW4{3Ii(Ym}9U+vBp`Os2C?@i~#V=Hg_1w0zoPMSQ-dQIU_O z&aW_}d?cmmX9Ud>S9f1FHAnNhME6ePpWZMwFZUL;-37^ghN&TiK~-3kYcDKO@YUr3tCA`5kY(fpZ|HXSW2&0G=?N1Aj4paccr9fz zRM-5-jso~zEDD<(JyY&l!*hOY(YWEZ-iloy=mLm!L7>yD2tX}*{!Ok*pXgCHPcnOUIGLgeX@JZuYwF4qcDYFVi_>Y^$) zapIPb%?nnVyFR6`oRrxTt=H59i<|4XSc*6OFnX$U7HRL{wtgKvLPEzq+>8pzKgx&A z6W8(Q6g?U2MkT(HX;2h89 zc|{+2uu|Wnz0XxuArKoIW-zOO?lr@S4H#iEiz6iezkCD<`XoNn<1@o7`t$-5Fx?){ z+)g{jkx}PLmXl-Tz^gaLA`e$|)6o6(`AlYCuESW*61S{~{z8;|m|?J{7tFQFnq8dZX67V2iftd3Zn}QX zRMhzBAcyDa-ia2&Itv>9I->0h4v5emm>;30R24TO%y#mQe(Q@x5n*-ErZkW*ZXh09 z@gj(~v()_8Sq6VF7%+x_j2y$+-qvOY6m)|nP(PgmL)q2(}{>q7@C`#3oG7I z&W}S3qRF1mI-D6kb4mQ8IWNs~dDw`Xt1F__9_G(+D~ObVQ(s*DfS0gyv%569vqlt% zDA19zzzS~Q+T%6yoqA+ulTwjQ!q!IxQi}|G6aMI3i=JIuYrOsqxyXE^n8oxJo`MIr z_$@xq8|j9Ilon#E0}&B5aNu-4KZZqBTw6%N|<5xa6tkDTCvm)XjKGn$1t_vcxBDdUlRPyDWJ zauh=}VO3O{fQa(FqdS?PFF(uWTxz_eZ__I{ZOF|=Scs7SX-k>zfR!i$P?s!j&9MND z*acVu{|i{DoxrhdjVhcH8XhUQ_7YO8`;a>S?vRyU@3w@jM<5z+4(3(SCZl>d7Am(! z@I(#=Vdp5r^s&Obcm**D!bsP&HMzlv-NE|GQ(xy)C^VB)tyJqrsy)k<2FjC^s}5ur!_zGw;;WDWWqKCeo@ z*@6&$)Dv~t2W>f~dF57!_YTGW#TkXCqdo&ER{1L;_}jzXxg>+k&@NWfUx|5d+a_I4 zY6eWk)A2Jx-&cv~@dx6@AFeP?6`89U2VA|YDyeAsxO0lu)+^-zhTc6ThgZu0K-+&g zA~)2lhG(JPIDewU>q&jf+NQoYB2LY36yEKkV8sA(Lh;#%GDlM1_sh~Mm1l(-je12I zKTN!E(cgZhoZ;rT))gE_5k;moG;#?C>V7@Aso#_QhB0u5Pk^M;d)+MN@m9Mbsf|Am`WkEM)Fvs2Fi167{>H z$?ulnNHHRgdpRs1;?q|o7Nw^i0(9KXfea?CNwu$q_*s9_-$dCG6@-Q~_h70;H^?6P zb^t{4#Dse7*8FXGqQ&!H2ez`qNYsiwWD5+adAUk*>-7N#I5!$PFs$Dj6%f3SedLBr z?1b+@WKHg?q3!qXV#38{w)(wr_!f6%4h+Df?ADJ$Hf1ZtNHC2fpaoMd+iB(#{%YS| zEaZPXaR;K)Vk)MTW+3;H)X&6jW@uytjcgjoVJk`@9t0pk;d<52bCMZ{T4}@skL-2J z2KlJ_QzCq><+BDo<1ybm>a3&6B9g5=9ntkeJ*iF2g-`4_%=*M-o2JSnE*$RY6~D7R z4KE_|QU6^yA1JQpB**CFve}D@K_Q}j`RnZLt9`D(6=Z*c7NKEnl5KPUHR2wYK0PL- zb^yW{)`@BC;lNxaf|OS_M`>1iqV3;+uBEw5`&k7MCd9(z0y<9ym1o; z^*wjPD5>J#c*}S?sUk=mtl|$TqSAh!up-(xdSjuwVPI5go}k-x$n1r4vE(n=oo%$% z+(s^B@&)hM6KS|bPXaT(%N_deLmig11$SLm=)SsGs{b)yChg$Hf9P7aofFLx^Z78A zR!4{u5y8hD49-Q$=**^nm4t;kCPV!E$f9Qor7 z&=`QPMr7>p*qbdl`)%QOSA)pdg&|ETL>AWhAQtp2H)NEH6w4Hh&=jJDjjAm1g?rYC zV+vNRQ;B@kYWiK0r7jYhw9aP)aU*~?9)28R;Bw$veAzzgffo0cxkf@eJ#iN=ncra% z9}LJ#)(s+m^zVvgLV@)WJRmF@1!i$YYAUC(yj$RL_`&5Zgo-NWgJc+=1=tAR32%z>i?*+bB)XVmdq3^ZXi|X61IEf0fE3s zNlAStVV1Y#3XYA%03!0F%L$@?j2jG|1}PLz6@$*y>jq8eeP!Cug@rN__%nUb=qWhZ zDTMkM?CCA-(Z{e3Z%OQk#<9z_IkG|@1rs^Yurfo9)( zw22wm5gX$0{7=r3-1f3MKNhMgDh|Cz4b1;%Q#YaJr{w^M8U7Ij9Q0Q4Ev$P4Nm7 zvb`#eukiY*07G!!Q~B;{n2C0S#ChQn;MPEG zbF6`jBjmgu)M8rodyhi%V{E5PQLTXiZS75Se(lRGv83b_f03ppG31LLKldJdw~v>} zSkL6%;8#ccruCc?7?@b=PT*HBAmc#@unKHUkxZ3ATBZs5*x29ow*K2>E@80AD)D^Q z5uh&H1=PJ{;h~_ILGxH_(V2!8ZUJ2+J#~@?6<4&AUJ7p0qpSPfuP<8sRU7f@e8Z40 zD`(lshR1lymt#3&Ic{uikd8#{abMWY4DoaymkB6#eMV(A9$|d?)LzT*LZR;Ic+?++ zF9;UuxnQ0JWLH10gFAmR`(BqbmQq}#lcTj017#wf=YS4__gQ%N=OuZhBo`D{Cg;v}#Ajv#Rw7PQuf8p7PA4 zSX)Snii$3G22xd5S8s-lUZW;@H@N@gS#I}APpH&; zaL*DCG0>&svV8udDux%X`mO5E=k)`o=&hrf(Jd=( zVEiD=`F!oE&eo<>7zMZMF0OW^{t^$~U=d4rSH)F|v-^=i66ac;?cLXE{}WlDO3Q6G z%eJQ;&HE31FfoJvCb5mZBRS!^IL}Y(#9pT;;uk$sMc#SEye#}FY0L8Bkv&t&ju6H>>|7a;;{^+TLFHg5>f0aZaN!L6vpZI8w_0D|b zmK{>y{G`AN6&+(lT1o1^_w;`U8k`RG0O6&Zy-|3P2?-H3HJnZkg78jm7*Co%OdPNo z?`4Pl;{sSvuD;_ke^>DvkqnQEAUI>EJwwnGrSNDKmtzI}-DZ1#y5QmD&qq`|Ft;nXyTQQ=+I zjgD`DiH@ejO?!^QJD%ESJSofH+Pu3=wt^h1O^&Yi`~`v%5u%dSo7H4pXY#mi1i;C} zIQ&fU?`ks#L#x*rTVPz#I;h#26(=TQBOoCBS0+?~12aHBFHUIC_YRoxI}FM?Y7J*K?qnmn<=9-$$5s>{dC3(-FAGZm8FcNVj;c2hhkM_Bol>r5d})6rq) zdV=zid%3N2b#+@q`&@_>gFGoGG3oy=--Dz>KgF9C$g)?J$4Sq1Utc=ASn`^r`$dYX z2M3PJbQOhS<>Xu;hJ5cL5bKKjLWJFB!B{PuH>bgTaE@VawbaZEAssn(h9tThyX9{mR%#(A;<&Qc%MkjiX2hC z&`S}Yo75-AiFIW!nv?G1TNv;X6Jes{-6V8TGtc|6KRxNST0yw%=ikedjL-Y?-2Rq? z`12GYZLV8r0A+*-y4UGRLhJVD;(OyEytNW=0}spWo-}=u{*|crIyFMERUUYjMtOS1 zQVTNVuV%>H#;$h#N?l?wQ7CyZ!zg0rW#V%}53q@eQ+@Dry@(7RK2|sxelw4pl}yip z{bNlt-hk~c*~%;u{I3A0(WO#!!li{5sR~!!uOyCkV-2@}7wPX!WtTqEn;D4tlMjUj zk1qP9E~izl;0K4c!?3uykIwvKnkl-lqM`^`lrdHZs~ZEssDsH}`!4V}K5C8cy+KPU z3L09nWe&OaDuR@>be-dh2v7<(1zleNBC}r2Z9)8J+QXoUrV8G{de_Be7HXqjcz+Wr`aEYAE}QISqj87#QvP}IQ?2%=YkP<}<~fCVv0-{0hN zvK>?lpMS83O3G^?`?m*6(BcxwkJR}0lXd(8drVv}r|9jMnpEkhK#8tlm)!nN(FIB1 z$@*#7)A>$+x_j~S0p%-gSJj@SEi>$Ok_Z;r6t*``QnpV*CEdI4efxk-98h-gPCJ!7 z!(0KZNYQ0_lZKpbSR*JN(hnk+|R(FY(f9u{y^U*$&WN>Y&2h)4HG4|7^WfJqm8JwkP$=< zGNf~jeHmJs$J{g=1K5gUg%pa9#5Ofo`AH7Lv^HJ{5riE~JyO~ma`Ww-8sPQKiFA45 z9m^Uo>Hb7tZj1;~yJk*dpReM~mN%q-%F)l{hS58XB`&#xV12`A*hX?W2{q$2$RC5m zA6f+wm13@T=Ie@t^3>%1W3Bw}3>mxl6IAXo=ouJj1Ox~Zs3ZyVe|`g)#byb=mzUS5 zJQy|A*BOJq0f)+%j}S$!^{EKl2(+UBet51|;T2_!WO#|Z!Kta+mv$F~GCATL(bxUr zlQs8AG5%TDNz-P@yKZR7U-yr9GDGB;wb12SqhuEShIPwfAM|0%R%#ztOqZo|x`W9S z8C*LKVlBSPmo99Z2>GsQnr(&*m#ObVrz}}NHy2VF8eXw!c0k;$KcQ-KV76GkAT0Be zbhPn-+x@llaDO3b*ygT?FbygnnlMJbcR>G{`$eZ-K${TUA8eu)1{OpS2=4kOIYKT{ zeunLys`AWAYH9+5MMXt)(^9zXYAo7iL7*5P)TCNpYk{p)&o?#*Gkc(QAXG@y1tQHw zl;svbO*;Cdf^#;#MN$R&&57~oz)_^*_XIHMg?%dF`>O|gmu7>aKG$v@Zn{y5QqL=5 zc?fPp%PgG6n@xV8CXnr@u1~KaJ8x?VIU2`wk;^ue{uQS8T?sOuV-G@QBFrZNfY&i^-nJrD5UAx> zN_)mdc5U{>=!qr&Vta|`b+{}{Ls%|bl$whFNy^LFIjFwAek>v;CT53>iYg&A6d@N& z`qma*5h%S(u~Ir(=)+xn3tm!J-Annjz!4mXc^^+uKcJkqzVnmDOgG^%JNJFLoPrj* z(uum6=HMhwwPRnHf((^y_aJV&x2MBBj%1*uaq&urzi%vsC?OGtAvV?acs^17OrtV} z>C>K4Hb2(uPVCg%-fc6=)$o?}CN%n5(<}S^l{PgCHTfLmN2J9Nv^VNkfX^)QI*DeF zt$fz$e2hCv;k*^h0)7+Givx096I?jk6K2fx3+vCsn;weEgy3XL*7#5ndTgH$P<#rt|%L{P}i>_NJ=~;^H1R z>WAQYxEuv^G&ceM^5I42WOm@u$1_am0O7OljU*7VC58f@oou4>@hM?~#XU-KX(@X5 z4$%4s*X#La&Bq8ur^_&O=|CFni?Olvgm$08ykOB=)KoSZVnV5Y}OoTHZ7Ao|*N zVX$?0?hkHhZL!d0)Z11&-dR6U9X!UB4ut`&GrI=fWR>#G$((WIsB2XVph+MPW-tDd zQ|i1>REds_okmeUKkDt3iNg}DH(jUCGj;hL-EOonSsghbm5e9j<#S6k`B zVWuS`KexR>f+NM(7I4?5Q?&*rjYR+@UNo90r|w^tlSnBH!?}eToaEzb?p9ZZMe=U}>u&Ok$&W&mgSG@2Rbg z|55oG#7pIJCY?=cL7syXY9x)Cb+=%eBZ>bbK8hI458mgudOVC6uh0|BDN%|g@6PeS ztVgS9^9JV)smd3h^h=NS99DbUsp7VYJmhe697{8Hn6=-phw%7y=NDtaRz*!0YR(#k z&Nc-U6)w@!PDW&tNeU%P2<_*fW@LbaVMH1Rxmxl!Ugy18b{gVI$D(OAS<|1FdPQ<^ zy~uY4@p2w1Ao66C*jZb&a&oG{Ng95`&N^^h#l!du-Gc>AMHHv3mIEXisgP%|Q7@P$ z*eU3>$ZVB}5~kb5BC{hRkb$0%C9_oBo(-ta3He_)l+@JQX`YjwlaLC0qhmmV-@>er zHKyrWOEFb=vJO-aNGAsg=bReQqhyMCEQb|>6Z`?5{x`8?ZZ|(sa2)Rfr%~s3b(-FD zdVY@oS`$bUR{eN}(I)#HRPLo@UR@qhox=yN%9q2Z2mD~90=oHj}JaG9Qv8@ zy;BZo;b;orD?vgUm#Amnhd+S>k**njXU<_9c-J_FKkL&-a*dgdK%^URv6f+-eN%=q zPXM!|0&D0$Pi`Xku9&)h{pMK90I*G2mCs6G}zWNXq@X; znZ}13BTtwEMul3(T+1!aR_L%c*i81k)>M={2MPtXU}i&Za9P1`C|BifT|fZtK%b8L zuCY_7RWb23KJzt+)r%!cP?DmH!C4v77`Di z%>;zXelwEujGfTJ+Ou#7YmkzSPTbvBSeFaA*44sR&iDuOZ=m|i;^R4UFFk$52 zt@MiyKN5px$C>g78PO^mA}ugyV}vS^AR!|a&QQZ>x3?eR6v(8rKjM|Yn8Aq$g1fH) z(6e)3Ftn*DwT!1id`XDAnUD4Y%&o&hBV;QRYzNBNXt1EkLiYUU?V*PIPWLa_bELsX zZVXViaIef7fdj~D92_I+r@sdN-q)*Jikx%etXaDUQ&Zmp$E2kW3-Tx^$i+ta6E!-U zLO3D+fZ)DcQS?4<3Uz6&s4DFpbPl6OqOz|WZhwTh1c@S!ca zE7uMHgxCSil6f+%Bp~wEf>u41+J=9Pg4lce9*ms&T5fG>ZO!CK#&17A5vw`q#Qe%S z&y(g|X{p`oy|1NMHaI()5Axv^j#>j@^J@TMW?PWc(Vl z+_Kp{f3uwRa9-~8$=fUVhPJkDgWb(A})H!v?t8FGC?Hk&wu1E zg5d27Yx^&5)9?T3{)-G-WE!A(qJ+~rWxDKCh zc!{T$CsvMn3#Q==h#q}3_m9fp-UMi@jE)b!03vQdE&^;#vw%LNa8c}x2@_m=FeU-^ z)rAlzIw~qG!PWRH^z7umqQ8V+bXXvvkpL!x!RuBVc7xtXNPGOWT;oc>CC!D|yt{i` zd~LbI`8oFG4BDWb{^gAdwLF|Q*UjvV0^g(KkeX7~l;Myr&d%G{Km_~jS~Y=*VedMw z_Xtd6*qKbp22mO{B3Mgt!LS4fMK4JT4En&ZfdZ0uwlfvCjw+NRFYPXYDd(#y6SB+} zaG=k7yyvFgrv_$#!N_}2@!8@a1O{ln^iQ#Q9jPcrTW7XzZA`{S-6tN^*E5t4-UM*9 zQqIoKc}Z13QG36Hej~Ei#Q|1vXme6G|99_!-2-X>J>ldyvrD1Zc72vaJFDWLsQXUh z&aTcd(bG5gXriKb=+4{1Nz0I=|A4bLW*(1EDXWck=`Fp|4j%))kQvSXS^@$*ePkF= z>RU!WaQr{4t3z?Xa-83kzXSYJ zolbWK!a%d*^=>dN49B4?a6I_oi(7mPEO?VarFa_E#Js~%Fcf6(zQ9fQ?+M=JrR~o; zp%Zav`#GrJ?YOWqn&?A8=r>HhPl4x9C}i3IDMX=(379fN+`Y79fFfUA{ZZAB>O0VW zkb*?V#}keM`b?esPc1NL&nW*e+NS6Z&VTwGd)22nQZGAPJ4}m>w+=4`^;SuRp?w~DSJv&{8sdQC%#LyC zGyhDjWcWu<7`jyL)~{?AnR>Xx+ve?S1W0D^QX4_P&{SZpzVDLulxu zvrS14!pHL`QntQTmlrB5A^{zVqj)?NiUkG{(}c(Rtg8N0*ID|IEMYfBz%ZH%W-V76 zf4dggryf)CskhFm%Wj9?8{?zWb@Y+A&pEbn!q13^i=y5KNnq@AjW8Efdx6M*o$rO? z3A>;dmddpwoniEDr^okWs_c{P!!<6)-@9VlnPl@4!BU99`oeQ|&HgaL7TzE##USB9 z08WK~-TTgJxhT=&PA>g(V$k=!_tzAIfHxR)4z-tgb-41-aJ(K2EOY~+JTRc6r-zq9 zlWrrjNvxhk4V2!Jf@BQd4(+V?EnaDqY0!MmU3iYQxN^cqAmk-NPGed@v}-?1m#8B~ z3nz@KfOWX?WNm zpa(#SY_xp#v$4;IYXMWH3w7HOJJY_$%6|2(bfzCoKZgsC!5_RXpbOK~Wa<9dIGa0p z@s;y5r}=xaOl1x)Z0UnUWIww|!CGX^>HgdoU85H!KI~A45Zks@Y4W(*YE8#76QM+%WY{V$CNgK zR?(RJ@D3c3$cwV3j?au6jn8sj&jTfHQ3wNsF#yO@IBIb8NcQIN?kva2(PRmMb}Zcd zTeNVC2R~dxh|Z1mQWNjExacBDJ#bDhIx~-H2HOIP_)kCuSXBtF9S{_$nZWx#si%S3 zWEF*9px!3ZXZ4d<0qA0>3cYqUn&j-t54_id54tycD}}fP=ab~qD@}dhy2;H9kR5NC z_DN9O=M$U$e_Xv~SXNEjHY^ANB1(5iHwZ{bcXvt%BHi6B(%mH?-AH#xcb7CL4H75a zyu)>W&!6wF+orM>vu2Lj6IR&|7XF8=<3Xyq1O*M8^{RC%|S;g%1_jKx;!B$@f67Dqr2Tk-j(w5JYUUh==qqRxS z|HNDE`#Y#i1de z5`4Y*A5BrKh;1f`p(PbJ_|ctSU}@{X=Q>*hxkEV<)J;8a&`cBJnLfs0ZEJ*-{t}V$Mh+ z26M6<#?K61wTi_=H{eInycdidAMf4P%YfnD9V?4{Vm#FT4AetHKwkmb4j2)?U6^mJ z5XQ&8`q@}fxVK|iO&Xo<3c61*o|}Y1!Nq!OAjW$%Uv8*{z0~|rK&pRSuE#{zltlE&kVO z#{|KiUO3|@g5Yn~<%QZ^?R__v}oWO=R0na?d-ZQgJ zq)q7oz$&=vry=HcQ}gfRWEtk_8mE@L7m@y4!j~->JV+CUdq6bXDWRbD1$VYTY4*wsvEN)Dc$4k^=nuQ$=w4qV+ zfIbs@ATRb86_ZRK*b;n@%rF}x{Cf#Kv##!-l6b@4sEIfFHwjV{+uJH;D zAkOVuk`iczJ`pu(1m@KYg^DFUPT0Gv+G>K>t6NqXj~YUR2(25lUV%lhciI?GGz zePbrNyFFJ|0cCx+sA_ihfZJlwYa;|#oYBf-pw6BE5M=uCR5>Mj!tOl5gLFGZ7PNF8m);zSC~#cn_X&DDY#$xHU5HT4l|*BU z7zMbM(eqfOd1h5^smVJ?y$8A|u|d`jmM$+JpVL0^))c$TSip<)B%kY(U#8F%kKM%kWMnVPTEr2``wJe6Ho_^^SA1s%`VGste;K z7=^^ZoCq67^gz3s#iGE=5BEX(0WdOwez`Z5nrnvRK@9~`W z-a>gvww_-n-|_3cV)KtOlN1N3dQ9`a%o-Scv(mDr>-4)`sg~<#t{$ncMwV-+ zYfFA^nAzzTK^1k^17Y~dRh%m>M3PAe#+T^MAk{Y@95(=8>BA*HB{iYKn&(@ zs1;e%%Y?c_xo#pLe;bQ$w3)75CPL)cA^%YMwM!`l8M0dSdgw)K}0&cYWOc zBx;d&$o~y0ITj=eGjd8#e7Mwzfk7oZ9!;mwh~GPQCk3;)nFg8gTC6nJ%?C))3U4Ly z0IJ91z0*bi^#5rAKH+m(Gc*B4t!LV({gA`_m2xj{8jlQi)Tz+VmD(n|jldJZ6;N z`E9zmQ`GKtYJGkE3XCq`ZK~PozzBKmcq-g_XD_ly{S32A^}X$6B7JQnog}QN7=1`x zmOzSSo`106_F$Z1HmBRJ9q4N;I-?#0C{dPFE@a?I%oF?r4iAa7j zCXYQ80u0?3PfVs{sUOYIcRAR-WbZ!90~R0Aa#b$VdP$rVE}nrHTq$9JCLE1zP;Wbr|bZTG4O966H&%)j{Hc z;bZ&L!#h5Y%jvXUfV0`&U)xQ1cgOb>s$Z%7?>CH!K@?#3pe8Z1o6p~5r9>-k^lCJn zr|*ChS)t+bwoE(%|G6kQfE==(;%oJc0oHA+F#QEDvRf9=@$2B~HG7`x;=i|xpLPRG z&+H4grFCGA$adVD+sMlBb+}Vy;q+nzV4I(xpQNm;sE`MO?3X$`Af`D#96iJo{`ysM zyDWWueXybS4E(U%%~xQPZ4BZ&xILWH^T84ZOz2nGd{S^9Gvlhlc@|*dK}E;0pbk5u zSbRm29IatJ#-Ct;Ce<7_8t?X~Wh;AhwI2_!Bc%D}>`$wezo2&yd3Lm#AOv=3+lS+S zV~0?E;RP6eg_?Cx*qZ{BTV;wI8>N%am&HZC*m?w@((J*;GZjZ0!j$T@6R&!np;W9I zvTlJD2(Y-&1?&Ir`+?wwkiXuq`w%c3Hz}OLD!~$ekt7|%R>z?Z z!)f8L63sq4uQ*DR{T=$ACJhw=3?w4(0|1YTD`p7V-Elz;+OBrH6!IJ50{6<{Ubrgs z|4FNYh5#J+12jKEZ{?^|%3A?OeGW=x4bky}{=k(!gA}s@LH5JGF2s0T^a`2$VgOwE z*jl(&nwIumUH8=~tHc?#LPoGf=}eu?Dt}rAhP#p*ux`g3-E~m|>?|@!w$}g*zNZAm zmt`^lPU_4WWA&*!JxIGdK<7HYW1esL#*`r-GwazkH>a`0{k#K1Q~2NT4L*Yb40`3o zDd)y>S5W6w1zbLi{zP91w5*n9*0pi1U!Vo0#XD}o(d0{e%*mIULeIL+FYVsTS>#=; z!b89MeSPyXNdVh*f;#^d5v&2OA|B^|9U6LGl6FN!u^MzMd3`M+7hc=>fmpkX~LRTd+$6dDz>WPZWzM-K6S9Jg2FB>BwAlhu2oxj8b=wCl z+Q7sO_yhRo?&+`>K*ALQVh71SepJPV+M>=%yyxP=WiuHS{aa=eCZ=!)%q2jBFam2;>i7X91v-#H=?<=uqRT zNa#C*mZgjjUNSZ;`XL=heWk0RHDnMss)zjLP73xG_)pMV3on&I&?(=XcZ}q{w|#o_ z8qsu9z6NI}L(hk*lTRL(mO2fXA-R}YeL*>j`PuCZ5XgH55#Mfd2TjEf$2IY(7bEmZ zwi~Bnq;$(|_@w*-GV-kE^JMZ&$yJtYq1Y<@v*>`Rcu9GougVy!dk(0!^_ym8hju>CiZfF|yL!Y3vj3WeHHSXj$D z#BaLFdW3u(a}?+(e4;t5Ova%I#1gS zEb1M_h!oURh(c1@pF>01N(O%Y=Wegrit_!8r3w#ULII^AC54eoV zdep9*po6xf!`P3*@B;92sjF5*!%^7S*i=&tKw>mM&|9NoN4-h5 z=PnoTo9z$9VP<;lfBwOUcHGY;Yl(s)%QTAx(?13C%EDRma`Z{^0vrW)R{Z@Dlsuxa zTLPub>T($87*dpDyKgfylK!8I=WEu@E6y`XaQBWZ131j^+6AG!#{>BH!ASGz5af7W zpHC^U*FtLgz}4s{o4&Sz5mpopPq7S_LGK$^H#eya2Ed7vt8cylyTgaW0{QgLwLku1 z@7l~pUf3F&7ke2Xd{F^AiWggi<)c0^#S6~NWFH;kXLub#f&E}mj}xzVlm(5N=ULiT?pH@vf;QH4?9VI<SeQg+tP_t~E`>l=@WC zc66xnDuYsbh)UpXHwPkbix0wtQoXJ>Q|3N@{a9RFk3;fB z&4(~BWnCXH*~L4;fLs07DfTF^6dBF`p4`f1ylc9=wF2?WkXapvc>vL|eC>EKerRrf zcY9DF#CcW}0anED539&ujKmWM=oiQD2NU6x(W94^mnmgdw7PS+#X{c~erB0wREDD@ zBwVvJ2|5_d+&HZ^36_=bWv;9YmXjGum=f~waa~*tft9*BmSE;b7HQQNb#FGeC8_Y;fh@J8R|Pl*Mmr75RlTX7b|Rf?ZJTNsH=Y zY;;%z+(URbKHJ9wSCC(%I9R5!)zN43U@`y|zggg75{m%t+$x~t1;^#scs2wer9bEo zAy8nT;>Y6Pc}B`CfP(*t2)y07O&MwmNkP!*8TZ3)sVC%^F2N zSX1p9n$C=Vi+XUyh>bL~f)m8Ji^JaFMgXl<691w@zUJ+x2xof+EG zLRf0_hZ(s~mXjC_UX02uh#Kvv3bMg*-il>tgR19qh*A_Z6YM~WB$y_s{H*$yuLJ+T)EzWZEjj#aT5?jcHXoY(68%m)U(C8FC) zhC-o;Z{NPc8cv!icBP>8?%^7vBUXYe*4yCJJC#H@d&y_=6M=z%{S(foFo_3V z-rw1tZu5B27d~Sdzo2c$toij$&%jbyiG$@4?E=^-}RB%>3T)dHy5YE zJ_-sf#vxM$BOG=7U>D7Omxp7OFPlQlUvWQ-1IDp>W-M#zfaCeuc_(}za z<2J`6RQl1;K@+6(QnZift~rAt`yQd8IMR(y60 zfuu0*t_8=7&Q#N8D5d)dpzA4kJ0cIfE8pVIXb$2`tvwFyc)KjriJr!bL?gtTeIfYy zGj7G|<_)4SIN*PYAbW(uK>v73EC>ZiEH9tmNV+ke%`K$y%&XlPPi{=88q?tCPySpS zj|=gZel4BxiG0iU)g9Q@Ul?`;y_+Kp%PuQ_e0+R$&hcdmpnYU~d_?C%;#TKOua~if zI6J`&=nd%l{bNzlYV64OdSSE~Fz7W_i|QtXQ%3sk3TS9(p8>;byOb-1;dyshpHCwC ziWtyx-sr?D&E&e_aklZXU3N9XJ<@+_Jzq8?zC$Gm2ur#AC$V&4+x2|BT(1@Mcr(vS`F(!6-HUh- zA_NIT@F**%sww79S+@w2JCo46a}@Y*;3Ogm(Hg+TCbXcphQ9`AakLEptCCN&7EljH z4o9FqQK*UZGsKE4EV|^Di4~kLf*_iLQIB`|wcJJPwD=ewlBB%6Je2{t1hju*^lg&G ztmqzyd@Pglnq)jTfIiDIbP$4U=5B!fu^M0_fX)6Fg{EOgRAD%KD*c1yJyniX30 zH$jjMsD&j%h7nuhK*!D$T9`!dONJ9b?|jjmaMh#Qy8+nLdi&EOMjQX_E_pnYRf^b$ z5J2vgr>2S2#N`Nt132;JLuXbhaNFIEAtNsH3thmhxF0Z<03q~Jqd)j&L#FTV?*-Z4 zS^{n_)>_?0r;SE8{HfODK*oPOGCNzCMj1&wQWq;1nIdfGJl;f0c+!qyNeL@(jsvJu zyTi_aZ9kQfiMXE?4wz{)gVU5a%Udvg9rp0GvW;KPf5PO@j;_IwVswu!6y^C>wE(jE zkVuPkw#<#g!Vm`2ebKwgi~Vr%@CFVzQyxE)+zK ze?Q1?i2)qmSK%6@*+UL1TtErXw0wOL8&4qBE0B`ax5tbM^zbmyO<_$v=WlYu2qWR` zXa7Ry+B!D&&ZcEPMT{qo?k8W#6g4xnACq@v%fE+wL+aTnFThPv9Ie#jNjqAw?@!MO zm0T78nSuXAvpXjMR5xiS)0~){oegS@UMMIiq?eHGc{r*v0*!EuT9SRaMg{RdfFuK8 zkx1Y?=o)CZARdl7k#LMM_=kzvi*NdOZ-)TTugOX{(3>wH$eF!iyX^`0{X2pbM`SFa z9`8lHD~tF3ykF9!^JUT7t~6)l z8p{_nAEI-AxXxHN5D`GRwzmQjujrT8W`6U8x*A~z)O^fOR?!cBG$BXNW@^EdUKaO* z;FST$K_;kZ9%_)Cg9;c1^{ETIJrGu;)8Trux$l?b&Hf^)VKFng z6zWVxi{!+tFjB#n|$s&JhUC1@}t!ppZkxnfq}oH zLE3-5?{2Jr5Z2riGM~0|=Ba^43To=^&*-*yFV*!DD=hT}nlOE|u?St_CJB0o! zD4;>qof|-MQBg*X`59gtMXWzJRlse1Pee4)>}m(C71Yp(P)v-@h-|z%wpf@PaVV(u zj?#osheeg)3AWm8^k&gF4(uWDH}13-$*cp+bAk)lGc{cW*{ zsQAq@VEXfkisc_@y@xbtsplLQRV23n$Hz0QM=BT-vnI-5nuHM|2VXmykMGO5Do2ZR z!T1XEUHY$jwPpeY(Uf}KYPoFLV4u)&N;c^6F9BWeZF+X}Y$cW_cJd@< z15NLKSB+z)mgX?Az%GL&v@ZI!Up<8$~Txo9#b z3N#{HF<0p27Rg-uKXWxFw|y=o)a<9t|DWK-T}XsE5do!YnUlOwoU&r|g@tB25uf|C zp#WWtILezhnBP+tOi`Wy)0b-b#KU0k_o{96+Ir0);C(;&+wpF(Hxx^W1!vuQ zJ1lLfYUPHbSQKga@&b#0Pg18V45u>B-Y9;o%1?xEw>TO1O@lX)xqxO*<*9Ia@PK5WUZ-mR$)8jZNSy$y9adwjUkQ&RB7jWM@I zQW+d2{|0h#9y5PeKRwdwMo;k^FQ~T|6F|Q$ zZ2E`5XGPGBoxoV4zu|dGfnMv)ffATML)s2eicTZo0RF#Dc-A3T>Vkp|DJ^_yLp10m zuUBYf67#{d+$46?ZoQncdsE?@sQ44pDP(ow)7u8Sm-Hn3n0nu!Gy>I`e>I%@W=nr< z$8fmcz64rVl**Fea`HSqSyjVk?}uA%Hc_$sqR*tu;@SL=M3s>opeNn{+#Y)UNCJbV z@&+}i5c)YqsU)>eukPU960dv~d};!v@o6{ozO!R-w$v%w=srj&hxE9j9ZTOhSg)_V zQ?Tx7ZC5A1S$C47FrvFfhAC2>8FtZSAf$SiiQq?%Glpn)28L8Q8Huq|nj0-W*VzNZ znfuu63dG8@KkWhsn-gswJ}nm+H5!iE>|7WK6%;@rZcYrKGqvi;c=o7;S_|sI4}}rs zk_DXn|5;1G+XNj3OuE{59+P5V043DZ3-`41WT0Zl%Ajib&MyE#3jsocY2V|pFTziJ z9)W(G)@zD7Kj-m8`&Rf*Pdk(N`DxCDb#AY>di!#eDz&4T-7q5$AC&YfQG8qdMUZ+rGw|BGG z8tJwJR`-_UqLw^-bu|-zu|@@xGyaw_sFvddtlD=#;{EClBQZC4Z4CAyUCR)45+b%2 zWm2hhTXpQ;otTLmay)cqDc)Eu540}MRXPv65&b#9J1H6Pd&mhF`D&lhipq=bx0l16 zEH7`K_I2|j0{M8;P%}qdK^aw*#aYz#jmPq0An$=Qh?h1P(R!h6-d2fo3F*B|;SQ)R zne`S!OsDi3lHqHCJkU$=#!gvTw0L{mNGOCZ&R!lZmsAtyeK}VRUYy5)ggyoQc|k!f zSRlIMb@k=he!_-*$Z!64vf-i#jFYQ$YQj6;dPJ?MF));#AYySWBT=p&%N)%-AHRvf z9ikU|K*`9ei$Xp&i6VKPK}_+HxDrOwyUuX7EJ42A-3DZwmGv~wT^@Nb{!hWmN@`2d z2`AQzIS!^|t9t{|=5z_iHMJ_r9sU46xl?+c%14wlKSY?^%r+!#b(LPC92Z z3HwK+_m?S)nm#|r4XW$yHJWhJa7PIF-A5s;-_cz3jSkSwA+rgReH@L&e4hA}{*S?M}PnFrc3H;{aPb zU~7o+%B&t zjNn7Wh(fcTcX!S%x6_)D4fE0iE`~q97LQj=UlzGl)WK}PMi{y2v2HU}s9qevYb3cA zaIWm_w2Dg{TeP;Vs{hs+K>M$$g`FX2%lviVrWzAdM`Pd_+IulA!K>UJCZb(yF;h$l zmw`pE9t=Wy=TptG8XFpdt7~#?r$!8KgC$fc((W4?8`ozv^swqn!O`Kkf!{hL641m* z37RF_Noo||AIw!`_V;1S-M#)B{o7Vsc4G5W*^Tq-$EAz3NWInB%usT{I%`MKV7S}{ z0_u-NO;}f=s|wV=j0G{Y6;{npd2qbpN^rbGsH0SORtI#3W^g)VYDw~@P0)7>w+@$) zol@IYwi(ZFx<4|LU4#gQSp7Tnx3yK;E(f$00BIBK!7oYF)m zUhddBDk|{;JTT}BV1taR=RDm|`M8)DQ$=w`D_A2kz#S(DY8-_e3owhwT|oa_E9Z4j2?kUZeXT1R2JhEdcp{Ri|6lO(qFpU2T||J^$?a@@~r^U$VO1i58f--8T^qjsDjrD zQJp|qdBhkK37-Rq1f#(xD&uk5#{}}mNHAoC#&|#8m?&0S&|w#PU69vO|Js9Y^l^i; z`w4`=pu^%J8JXu)OqA!8YTN=FI0oRmgIZf#d%AM~Y#2)JW(Y`x;W_m&PuZx=FSW=% z@;a!m=>mrn`TRyK+PX&Nie1tMT3}bOc|N_K$5N$WZDLJ#Cgtpt-Fn~ZeCm3qJpVg*~8LDjN!{a9x>YD zZ)WV0Mc7ytwcqQU7qWxBWJ)V3x|zZ#7oQxC*y5xpVpE1afQs)^rUL)MW-AD&T_ISx z8)@9x9^7G?HHE3@#SR~|7on!63}tuc6kYf(dnr^9HPJRloro;mMZPnL*-POl!}FFjIpUhnv_eNj za&szpk*>~XyVjxFiPl;$F)*|Jp*BRqYkBE+jiZ^DxtKzFXZrOuL&3tt3>KwshNtu1 zjf17z{b!$wB8ZZ@!Y3K#{+BE)rnq!O=A96w!~&&RWHoBynD-*>n}$GxJgO*`nDF`q zdQ?9?XQD1s;VLTaoDA=(+W%<*fHe$bu@|zjA=EcH3C(Tl^_|WMzPy{d!e=+97<4M) zwFj4GTPLTUo-iDh5Q)tsINV!_3m}q#Axht0Ux!H-6rpaIc==ga7#7?X0=(`nkp${( zR_l^oMKdu2f`X8#C=B!X)Z~xclm^=~Gf5nu5F?V&5Sfp;rBSTZHtsJDUR&z436fOb z5u|s^G?r=qD`P`^a6^sN?MA}vkwl=*7tj#l5hR?!{Ks4WL|LGca&$0IL^e!tcToPFI)UEPCr-!ns^!LsD145LTGE)2hqhLXh>p#+nbvd^3o12 zE)rdVD0(+R(Qs6Bi+sQzwd*0u7rLv39=N+11+V}p7IYU_>#l)gPB2q?LSR#M0F#J_ zQMB;)?+>KPN@LFq%Cm6$W=)1{S`mD5?7jov4nk0zrRl zewxiA(!1nYju#i4+58gR^t%uIgHDoeUu)_-hl!3iWDsoa)CLm^L)9$wug`n>Z>sdO z8v6`v8#W3# z0f_0N#cXMcWMZ8+-ce>^fbSdmA`as1fCI?8RJ74m*+8eTj9&MSmn~$b`Zy zfiU8|kX*^-<|kX03vA7@d~-)#N97qvO3wM!#1L0&e03hx$=a}9LzRBZV;cKDdoMLc zUa>gG4U(sT7nl5Fy@G&Z<6AbxaUPEmmaQnQ8#KvHib@remwjF@xmlta@8D)1KOn@+ zbt<;iAQWaN6o=KC*N9~nmk(Uu$SsfA0JWPq*kg^pgr37}0I5M4i9Z~L`B2s!(Fe>Z z$JUQVD8cC#vA+6??|?v%S`G9sa3|P)v|i!@P~+^sH`7$aO5d%JF*|Iwh7KH-Pfkd0 zb(#Z{=MW`Pfb@N63d3;$?{8gS)Wp8Bw2kO(MiCfsJURu(v&C0tM$=y+shX9!XYgLv zD^q$AidZjoWH-C?7y*&J*G=lCB6y)jLf|eTX;{OYpkVRA%zDK8S^0kg#BQ(K&&P#O zI%_Q}nSGIq!p3TTS|>U*+wKR?zY-;*#j|>X?ziGPH_GBjSMxSt=}InsH}!}f#s7Hq z=P6LWvQxH{esk-LdSk7e@ixnhhtjIV&?7-`GOPgX8ShP-De98G@1Fub^ZAkYAf;yvta$T6-nLQi$h}2>+CkfpKVyTza#}ySquUTwM?Lr zA$a4DxL^|~{)o>8Mw<2DM?DQJGix$)%qW_^=)nY~MZxz}e|njD34=Bu%4(!rmDODx zodGOGpII8V1zHL%LHN_g6^zg^^0)>PcON9+ThuHa%%UHqgi=k^{OMz~20kl;+ z2G!RYtvKWw`~%q~AsK2THVEIKj;_Ui6KwY8)`ru2_Bu zW#1nf*A-doiQ_@Mb9PZ$A*ERGrwFqdHiS^k&=@whrJO$oN)z8#|mEbD=+7 zaQWLOoY@!Z^5Z8y*BHr}P`sZ&h&eBNx4-vV2=7@w;pqe7ZU1bCCW9V_ZgUicnumukb7x?{6b(Dj=i4`3{X`;Y z@XrUWHr@psn&?vx5`wTpg^V?+Z)94VLqWZxqwiT{O#I9B`XEVa66bslp%P|24CsZO zdMzDcn}#iRk1du5lqHcs>qu9WG$)t*0S0)|(VwmWdc2pWPOulq%*h60KL0SMr)=|v zJFn|E;oo7AqR!QnW{uj7kh zc&;b7BGL5Im$1!k0sR-k3Pe~2IHi}w zgHAv7W4KD4#&=H)5Wg^`CyQ~oeityGwfR=I_nFrkot4NQ_FXpFZzGvO4(TuiDwP-U zIY)x;yuIU)s*tQvGFOM)WE)w20jKRNAQM%%!UmQY(t$aAbg(VHX;&DjYIfWs<6&Tb zW+y@@9+sBI3lvFP);E8|H)UXce20#{=Ckv58_3D_EN;n+GYzALeOrUs_Su3I@7p#asUzX9VJX67i+|KvlSvQfeQ&c&=Bq+W6-H5m#!x^X@ob+ zBNu3PK{3J3k5^DRCg1n#HYSEeaE9L~xS@T>E%s_ci?lWx-dGB3`C*e1(k#Il94?b^ z*4YhDtNp?X(f_Y9<2J0TY4e)oSZD^8V^GF8=grNCpzB{C60%EYYYKv{0e9&O649A06IkXSvI8kp%G3)Aa5;%@7FS8znYn&q7NtJqH zC`%js{R>00Ln-zv9M}J{P1uB>tPPT5sKY!!o&ZS8KhtI*}cXgOirCKGfJlBkHC z(3^h7l^b2vX=+fb#W*(VY4GVK3HNVv{t$VT`RnU=jJzqhg0 zf|Tq@m&x9q9^zNwJcg55TAydC9eIVRGF@lqbNl02GS{#@8yhO%7QxeF0c z^ax7?0sX45Dd8&L^~vwy?q`3Ol5Ji_-3xkYO`&(44vFWWaK%u(vN%!WNA|` z(+U6)3i(X|J>&+6r?r7_lc5rsUYB^_?o*CSVSyfn>EHR%C=D>vRPS}_t%qiS)Ux%F(&X7KCW1_|08wG>2P62r(A818c3&9@+Gm}uphAhSqq^ZAy&it z9WJEgVChafcX3DqJ=&pJi1s0o6U+E{0$YGxBji~#9g*H6r6QHz#_W254u>H1y~1Bc zUzMZ^@6$hrl)gq3_r{0gv4%#GV$}YFO)^n=^Xw60UO!fNU*Xe?P-{;`))k?}I7ezr z=vi8LR`u?}$&J->R1t`Sz&?&Yu6hO|RDL-2LSj5iYmfda@{+ALf!K)9{9#B;c7v7i z2VkN}-Us#~&+`e1TOPA_UVs~uJL#@BOzHriln`H}W1ZWXF($JvZ?m_}d=@sn75NiT*nDnrt-O94CQ+P>Uh+nmniwSg_LBv? zVj$Mss#5YA2LlS17`USJ0MB^O^7MHr(3DcSLl&z{3ydMb@||VOQu>3&EI=#bv}wr{m$U1 zTj=&5p9lPVy-!CmCgoeUb{C4gZ|}4lN|Y+l_*0VYm+BgO$BQ!wG=!lanO+$ozsQru zRFUfmLT>?0VxfRll{yH_T#XmZIKmAavr(0MfcBASGbDz;q7n+@X8_!NKI-rDTYYGM zyyk`nD~OL$GXBAO?e_UIE>JU4dP{51I`*}v?nc_M#k{c6%$YP{<)-y_$DI#dc6}q- zvmjkpbS8VJtozr7@+WWcJk^_&sJ8+RCPl|jO8@01| zco}RLw{P6s-L?J`N}EQ5odEx*DpqbcM2UmSuQ;N!t1AG3&cBXwos3|9*qNU1#eQ8; z0D=2l z4}+FEwfFavc``-rYNuyu_H1#NHxc|l81>p(>|stMWjn;5#hv6BWwxmd#6}nm`WG>2 zX#}7LDMK#G(?cB*5G^>*kWJFUw;8l8uYF6J(Unl4kg5!D6vSd#rZ!M<(V*6BeJ@pv zP^VdZl#$!a>!rw8ZVqHa<_R9RU*P$=e%s!YDUCl;o8H{8z$gEf8(@JZrpz z-jRf6w%r{yn}Rk%j@Hw|3i9W`4#2gG=qQ6it9xJ(?@!~bkQ2(;FxZ>=2CbSNQE2(_ zH&kf`O^qZb;ou`Od`X)HeUUcBm_t}I>5+3XZW>)wA*>Yb)P)h=C-QMnJ8?mNN^n+d z35%{qsI-Nqxsshx$ofH&){x`VsDCw&tD5za~Hw;($= z9H0I5&Dr7!MhafC@g@Y|qu-xP(;E!IqwP)?#f6g>Pt}%%N=)wr-x47W#k{QTdwR)( zAmJn*E1um{%kv`UE-A#aa0l&zU%B%Q!ELHG?`u9HvLED)a1{GaqI^;P;9?dA8F@wT zH*DM|8R`>gz_#)#HV*#fzK7^T)V!`jmMec&VYQWtR=YPJNbl$wvw6+nUo&QNUMd?g z^xU@?9R#+nQ`=gn3UD5p!9VN$!1<50vOil!UmP<(tl-O^$L)SzXsZKC<{!)~wn?Ox ziNT@^1I=eD0p_z7-~V7d%nWv z;Pb007ZhT;MJ?C@xR7`USXMXQ7H=|Fi`C0QjfQgHQ9kFN{{J3<5R_b~wUdK5=y50% zU`~R4`}=j~F>}vOHXsE1xcWkUla_)4uBNx2w`Ar|f662_N7>>MI1bh~^<#|9GLdL> zg2c2ax9_JtNm^VnNdxiuy;$gGTt({ir(R2ZjMf2jK1aQ;Qi9P}&j!^2AuJ;95*7$ES3?8daq#hn)Q$_QYBVR;vW6%+ z?!qdOw=xP`*>;)9ep2jkuk^O40r%i5Km zD>!5dNt3nfGdl<)JEe(mBzoMzVI}-n>kS_9-RG3;DSS0GooGZv@>tG>_8}5mg9%+g z&)wv&6H>ejahO7@ZyXy1biU?16&{UJX;4D{_jPez15KDXDhvN zK0Yd{FBqG5TQB!#kQ3VJZ~Hx)6sj?at>oc>0B6ZpeH=)p< zU0u!W$w^YF2l-#XB&s72bAO-sz@844TJl@sVFX0OOa_myuqn?i3=ZO+0nq2)RK{P6 zV^=5y9$4jret+0AwEzy&O0P0tO)w@N;8+5{t{KEd*=#PXxf+vqL8=F=%{T(c7D?>$~LL{r28o2;eusA&NI&g-jKV^?cR@ zpz9g@FUdSH-~jkoXz2_9;m26+Y-fiC8{6FMfqk?s;|K>7Y(H$DLGv~>kO|44Rf9+= zbBZ?1EsiABA!U|gwvsrE?;q`x7uN>h2*2+$%C~8z{BBqjuEzQZyv;6hqO4nYPGB0~ zUsN)pAdE4{>n3-%%O50nK$JwAV>TG1PKU<<2Wa8tUauwFHv~=U|3iWsevK|Ne)u{( zr`i5Du>AZ?3N37RQ{G)%KMn**+PwsU$so)+_u2cieEIR^+BUXq61)15hanxRz%#M8U$JR$2#3a^xv&L8l;@dcq>ZUSb;RzLQp(2OL(I89HSHQyjzG2L_Q~vQ=Hk) zg8)X5p;1T@kZPjRC0G}Zq}XL=UR78P(gd2XWWfnA5&XXjn6d; z>Y?|4Rbt35Vt-|@KEG&P)@hZ%QiDuV)0-GGpC}_Su56Y!J>-p5vK6Z`&TzSh?d9Ep zTW4Yo({W+$4$n{EZm(p+RJ9c823@{wPas$t6hCEx-5fMC5~3$O0>)+*zyp9K_T6J) zrye96<$x=ixa*n>sF3}D117a$vDgRP=fEt#i)U++=Yx~u^gik!NRN zUlhq40>AgYtK;r_uh`*BI+!l;0>3}X>;c$^nmP>p$CHw|#ye{?@|Ceu`&bA~s&V4i z8|c3`P7fqJa=6A1w=R5iVe3rn9}Zf!>XLN#vonWMenH#zFzLBRLjNs0^NklBf5=HU zRO2G+O9YVsmMZYM7em|neKsXFfRP3*Gi2MqWI;LAY!n(sHhG4#Wl4mIr z`#Lv9#)tOH)^*Bvd@<9gD-M^3&z*u9X#3Jk66uz%ij|4~y>bXlW77O!GG`;7oiAKII|<;u>uDFp1ZrG<=?=T9jUNa>gKO4K9~+by#;p6J90Sh-M!} ztW!x@S$fW7VS&@IA62t=Ds~FkP}a7~5HXpyLO9x9D)ujn$n5PoAn`c7ap=9kc{MDTeK!_Mn)H=Zq%zi3fy-Z(UChzy^K`JuZ5TOx?sv==Xm zR3A{@Jek`i?sR-!rx}VgY8(FyV|MhzuK{pQ!GqO1SN$5gEBsc05GBKpAGX-PjYi)J zCM10qa_-Klmx1B4c!4&tj~T^uf6W4r9!!T1_S^4HO4YTu)xLQhA#cf{HJ35F4~fcp zdF+Y^f|oKP(7IC(qI!ehJv|Sqzn@Pdh9SL@oNCwHli^qJ@P5SqkEySY%WC<)mXcB= zrMp8?I;FcokW%UHZb3Q(q`MoWL%LDAQ$o7C4doAH- zr9i0zXj{Ajf_VVdtn35^AHP9>gTG%lG66FL5+K=rg`W}|?nQy`4DlMZcwS^)Iw85nx06<>&d{-`+V(39w8l=K}OKJtqsO>d}>lWoQkt7&+x^9 zSuf(krjv zH@42vQ#_^H?zUq!YbuD}o2*_FrOi_YoZ$0gI4-Stx$+#?R*cS|V+Nh%@+tPtvvn5n zQf~C)^E%sSY4Ms$+>gnDIS8d_)Jas>!1t6%6_jRZ?5=(X@|+CDM5d+skNz5lt$->78| zPMQqm-uvBgf!&7;>sM{v%Le3bv${?MT=RJ(;m{4dxD|rj{V}SNLo-i#HaqV!_!G%$oK_^`->bvrWG*I=F>VCcmu_t7 zd>g*CW75ydk^L?NNCn<>=d3)Q@ER^xYI-fdgnqO795hz1DjlTwSUXDLu)Z#;B_};~ z{iIqbJkK^>wc-t3ajg98e&>m5xUq>+MRa=TrQFZ)gzsh6$>9Z^=H0J!ZX!ibIqt}c ze@o;)yNQgi^&4}NGs77MhBkoz>W6Z?yjeFfY^|%Si^Fb)S)}GZdpFEg;=vB)C0T*! zpXAemJ#^4}wLt@X1$capoQJbC`CMSfm7?8%}!s>pGWe>Um44*_?MrGRmkPY9eZ>zQ3DgQWtm7yeZNtKyWJ+jgGW^SMu$ z_ri0H;?U7n#i-#@SW=v6l1!(u3pu;qtxqhz)6k<+P&TzyY+DDvvD>Kb&02bJ-}gbA zE+;wKWyuii2V%3-C@2|%rK52Sn$bIk^ZCid9v<*H(uRuP-GE8}>QXWPG0EPZeScCB z7^%3X{=kw?c+_p>(}kC+R5McI422%Vkp@#x4txPbCnd@8o;W!0vM&hOYN3;vW5N}1 z%3tZf-&lI0hH+kivgQmM!X-7W>rR{J<&6`h0G-Dk>-9o1hsRNC2SeOz;Y51%B4@F% zIC0I#-hR`${(e}Y!*QO-Z?s58k2|13I-S>BFWgb)UJL+xbv;1(q?GjqJFJ{{JoqD{ z8cprZ-Bg9XL>K|L^7HWy7Ki{f$SlcqQS2Zah*Bhebq559m;qI-t0#|d-`&k^wY2UR zv}bfSWwyc&3K3s{^^t4?H!p9fS@sKoD%Hzpxa^-Z%5sgIAZb z)bMY9`vyIXOh2cErSX*o?m6F%3i1&H1)@ZwG{wny4c*-asOXk~SO@=q+a!+|a~hY`!G{I78uko)HSWT}Zp=ZX|1iwIat)U&w?K8C|) zBVXpIz!d%%ROBmq3w>1|M{y>*K#3@5P%RdH$GZ+4M+6hG5Z#ilG)#^z;6C|Pw8#}+ z1(uP*-7KgJm!KRsUytc`7A}#Vx>;r?u0s{hJZIK%E27jSgw~V5bmvw zZe(zM0KM-lq<;1ErIeHuX16nwCV)Fv9vS9?{Luj7W^wM5)0vgAvrVwdb1PQ>+s@phIMdP*f%Ze8%o!cgVT>j3K z`*i1;zUgxazp{Kw6uGg;vB?0+_E$Qd(hzwia@!zT=jUora!b^f+bo!o(rYkVG>(-G z@VUAYJlSSn5W{ndNQ-tx)NBlLKX5erk00+f>_L?a3vpT(a*pLEnsd(zW7+@4WMqhu z7i9RYiS_$lC8@V%sGGgrY2AiVlV59q?K{Al8XsVB9wVx| zEzVGuH9t|z$XZjW!Mx%jQ^U-~ihSA(HNk+C(ny{vz%m&;`m4P%?r^|a4viS&4AQam_Zp_+E_9IA%X%!^x-Yah~XteFme z1!Jl?;HJJdIS-~wXagsn8Tb=Hx0Rq$a0&+_>>G4zm#l)cP)DPRG|EY zs33gL!B^c5DI7B2Ah(7)_j|SDt?*T$RtADI1DELe-LCAXl)fA5PhMb4VKToi!o#J& z!t678F?`s*UNbi3k9UsuJ0`dgzqiHeJxKh3?aizmi^E+bwu!z!`N{Qt^#?9jmsT6S zmY)Rm>Q;2-D1cj~RySJctqN6r!0>GsNwJuQ`RMtw7XkAJPN7|{QkOGBm5iX%-Oe1O z23jU9?Pd_a_DzY^Va>2&lFakjDQn-RX-z!0b?)dI!PtEKq`Xew&jUZT(UgF^{K=%g zzFtFXt0*EqZSf_O*Ey{8E5HY-(pqY1B2=A3j`q`s_AxOLbUAwOA5IXAqo82a0XrZJ zj(XMm*XbBVLzx1S04os{{xrcWVysFgONQ<_>Rmiq;E@bVe#6SJo~N_KAdxF0rqqfG;<9}L#nAxoAQw=k=7 zk>M1#JR;pb-l^HpPMR?+ENvNJlIt_c(-UpO%DhX*aE9qzimv5VF3;YN&&}HdRHM0f zQlBW)T6H`VykIAmf{FtASkWU38ck(I3aQ6Bji0a|x1Njcgin-|P$<%p5~bd4=ydf8 zpjHe$Tq|JGXv}DoB(=17yeX^@1ll*+tzLwHHjdr=rTXd)GC8a0O=1W7BF)YlX@zfq2zrZ zjH$KKz}w*kxgje>X5vTsx+~-sq0D{Cb}s~rF^bIHoDg$OEQuDh^(Yla_g3*HCPFm- zmAJunu*8JBQ9HbHO_V(B2&Fu~RRM7{`s!#Ye<*5$bsL_EK$``>_JLV{zi>T-&d!&$`Ib zxumuzpkk&9Go!eq^b{~CNP|GiDml%UhNrJ$&5R!IaG_>jXj;3ls${2(F5KcFB6tuz zC{ddCZhB4Z{aSeSecXfmhnUDG{21G!Iq)wiNL7rdF77sVcId!){gE!%>M=6I267=I zi1hbt6yUzRC3yLCnA;x6JNnGkRDJ6feq+Y1+vM_wcy7Kr3ROf+l7bBM-tvETfO%!m zAixqvzn79LSg%e0^pHXA#9%Qsj~02-JkJRL-y9WZRCSu1N6 zfF*aY>52(d4wgS(@g{lO;xJUpiLnaaR>ECsAycql@>}9JWS67LcAIw)-)dU9oDgK5 z3d?I-)9?7WYRHa7NDIMmEepvfo{KND(&eAP>(wG;TT-nBTJ(5rpV71OAOikbp_wS+ zF%6#!&q06%y1p|pA4<63mI?ZP36f;MYE-;4VNB1DEh?VM6o%#>&X<*5ckxw}JoVn5P~*B%)7@<~O5P{O+rk*HglW!6T#F6( z-o%f6L=R+GSEIBI43BOBxfY3Z8}PTw&*h&NNQsUiN>z7zoTNjFNS0+E2!Dn(49FNajA|O=y!z2{HStGuW>0COiDBJoGW=K5u zYD-&f$6Qnm|IALFx>(1JBz0#aS_VB?m@MbE7&ERcJKlq`D_f>Ga8+q$iEO4F<4&pS zK}PJSzh+;Ej{yx)cnTVV0vll2{=WYbt~spV8IBj#Ld z3?uSn{i#vym!ygqxp``}x~N=`FOi0AV$3Xqj*y=!%_TV6p}K4vS-3Z{!#U8U5s3zeuq+q4n*mLF7Qz|AE4?i#{U+Os+j{gZFZ3q{XO0rak!d$5d2tAm3-JgVp z@d}gCRTL8RhM@xWh`h^wW-Agc--ih4yx2BO*8LH=^px&pbZ!3VaK9u&mZkXau=FG; z=yIPa%QI~n{NA_=tJIyW{4B@@C1rX{^SuMDy&l~SuS)zu!G_c}MpT~r6{yvFZtFP| z%dd*HM!_?f;A2?wnkzreD77{Kk`<*2@A4zla)(w+qsM41CQ_39Y0(QLWp|VaO%$bX z?n$y+w3C@6sY|1*0epbYDYZsXte#IQkbY8{a#h;ntZnhSW5_?Ftd81DbwpRKXi^rI zMBt7Wzjet8fbX;7 zxw_0eXu%KB8EGQY8eqc8Uoi(GtYjUNi*iQ8?pYC@%gI7CS|8_y0&?DJ+tA?0CvzyG z^Pfuk3&?W5#hj5nkTB*(lNWQ3$nXchMd9`hqE?Qvq71Kv8LcI$O#E5vQ?yWNUXJnQ zD`w;XDFIsOtx^=ubX`rG@I0NVdoDY!8vQ7T%q>CNzQ(fa4jX>u*AT1Jai^g`raA|s zZE-ejhszl%f9o_xK6Z=Ir_F(~9DZ+ioj+)xO#}IatJ_;GZ)LnbpR}cCpL2j}34Juy z`2acY@ob~?GN?2j~<=4>?qOVbLOQQ_Naw<%_ zh3_i78m<`G$?^iaKob`s@dkjjl-+d8(<0xTB{vzIFgp4H<(S<(?xC8UyRLNfC;qv> zSRz2!uM$R{r5q>(MfNU7?jXPAwwuWuS|g`IUD1-I6nef`Siw6BpDG{@u%Lt;X_d>; z`wu8imMG}lQ8c_j@=&*I8F{fFh<&V?kgV}Hty39?^)c0M(*6%b~ zQs6OM_@UgewHsA=w*12p6Ox4hQ>gw!Z9`_!ATfW!NM}%HhG>4{Xuf=7;IkVQYMf)3 zfH@=PQ*r;Uq=?SH9G7~GW3UrEx6WX5=--wF*{-2SK8xv-?~ck$Kg>gE-_CvvvW~rZ zkGZg0;q&35JKDE1#{M0dfVlFo#P9nTcBD@68yycPl`bbY*olYsZDVJ`C~d z>Pl$(w@r5l*3sFSfi@`j{zx|^JW;!EbvYK!F7=*dX+7VSnWyh_NIxnjXN@4^g?-@0 z_ub_rXLntlOAsJlU1{cpj+}mkUQzC8lBhQh{2q7sTF7*1RI*%6YOM}_M73N}+viuhA~BUbzL&pMW@ zc-Vpu9LKH>oY0_B9q0~0R7{sj1uCrfsX(6(G&27fO9E8m7SCk5j1U5<^=LM`u{YIc zChZNZ6SWLUAunfPzIW_Zn$?JW^CIjTyQVwz^{h|9|pQ6M2dQ~ zLX*iCwC|d)-d21RE77JsG>w>&2_XmWB#`97Jss^4=rvT#Bel|Vy8TpWHbZbO4J^;i zsfDyqB*Jh=$1*Uiv0p_ebyfj|{sqG4u5s0%AO_k};_pD%O81$C*J%Q+mDSmM% zl0W5G9Lar@6zfs5`!dYS*-@2W@e$a@yd}3*68NMl{}4YN+1**i+l5Me{_yP3`7S-B zbK4UiNzyJz!VXd5Yg;jDRC-lT2Q$hXE8pM>((ZRjjr`|7R%q1p$c14bYRavZ9ZUjut5AhH%-215aoi zFV=0%QWWtaS?*5@H&rpL8a&3(D2KNych1K~eu^b%fiVU)IDT6KE-to+tY$q5 zQSQ&59NO=Cd7%28mo;46lJi?*A6*1T4g(^~J$hp${PlXiD)5Fn>=D>yUtD?n5IBX{ zR@L~DT{3l-E-GX7B!@3av_|R8;(eOlkWrIR^@Mnk0!^c^S6a0}6hNIRq)ev)x7Bf& zMq>EyGH*bP7P(DZdfpDDaDOoFW#T|uiSxv{aj4XuY^2c`;$xbxgtu6G0uWpu(GArs zJi5!CXS=eOCuM-jjf00I9~oM3iEv#5 z&GMv%sRIYwuj|}qH22f6zW6`3jy`}3xuQD(^PJ9RHtOXRj9Snx3Masgga9d`=F7{? z{|=GItD=fM;PIlXp1zx1u_>sjoj1Qy@TNHQC@mdFR_ zxk_=Z=@}nePfr^`I^i?66jnndKEA6%DRW)y08d)yy%{KM?3Zw*t^(Xnf6pj069fkQ z$tGf5Z8-Srmu}{$3r;RVN()Srq^j9#!`{ri*$KYn5yIP}kD`3|yRg5wVSl`c7`CZ7Wl zy@Ac}e865W)d0R(z<#x?+%3X)1 zpnu~L6nJ?MWbv~)5fhEt!^6W#{LcJJaxPAe#G zai$-3xapT~J632;`3^-4gvQtxqS@qB)Ktr7pEz8OMbQP`IqnD6F2b9>*18_MxsMl{ zX*4Rt0TmqC5fPw%+u6nDHv}ySd_xDp5&Pg|&9Ngax826B-Yuu)chT&oK~SYP0k2!qJ7$KtC0Ksx3VZHb;Vf`@SH@faGdPaUnEB zbnr$F6Gv&5)akAkX*Yfl!*63C_H9$bJZ}RXzQ4G{_y6c=dq^)C18`-$15*hv#n_x0 zW1`VnTJ*H|>hWod3ql;F1r5JDqXy!hOfcqL3%zBl2F^UHGDPZJjvL9hXv z`o(^GUle(RU_@+r7`Ocf2@Q>;K|6J+=Cy&{)!#=YK?Y~gpgpNYMZi|>%?#2rEId3q zU7P|1VN&N81KKZ3+6sQWI9}gev^h0Pz?r!ZAEKEw3p;sZV$o%MS^j&#dH-DVoQM_o zxmNY4M%<6w)5D>`E7x|_d!I>xdDvdz+^@(&+2lG)H|$vvlKGFZG`qn}EwQ>Kv&smn zb}%V+ENG>ulG4&0KsHwq+%_*^YFGRCR1 z{3yyA^Rh&|-%fW>!;CU!vZSfz%p+!$n3}HR@UDV3W zzY7g~b=@k=K1yd6IvlL(6<#Yoic4&oPfHGM^@!-je#8nMk6UQRsK<4fQ(HcO{knFc zkaV*yDG5{p?JD!^Rt^4T5ecE7n%f{t3?>G64Zb-Ytd8#oprZlkh2oM-8s|yxk3=n7 z-s2KU$u-37a$&#~*Qok@y9lWNR`TMbRSJ4AyUBsy7nT?>w@7azSF;g>iGVZTJU5@7cu6yOh4pF+7t7@x|&6p=?|??4}FmTooD1520Cy_k)U z)?yfd6P2_%eD#413F}yq8QSuNBw=ZJgj9gs$FRa|Uwx!2dErpKAy)Mj)>I++lBqJz~GgobJ84Hn+Q!le>GmfpIt7Gl#TT#$tAt^ZEfW!ma{y*^sOxh1ls?dvo8XF&KWMq3S59#;(7LGe}9_S ztS_=LeM2vOsfT0n23ddap`dgmwQ=9HUu&F|48o+Ay{!F6qRSVD+0!8=6BKX9;fnC3 z&-L;`ZiVmd1KH7DO@e~nPv{-3?=WCIG4bI?nHTx!K)%+?gX&Zn6D@WLowgc>%K!E$hEKDRcvAY}y!Bn@^*F^57-8-M9r@JN@g4W-gMuC}BTMn*@*r zzh5H8KS%rOVT}{gG}-wHtx6o850v4 z5U3P^%#nbpoElqa`=3=jykRc=#|23E`)w?T*?@)opk-1p&Y5-GJN3%sm-wv~|AzN5 z;t-)=4z*5Zu^^Iaw43iYDOko*++Ydpi!IwTLKedFB#$;1OtPb;UxMETytBNjp<|P? zgyX+j#v^^s-1%VQImR=o8T;;>wbj4H#QSrTVf>VA|rXw(vS}0X4 zNQL9!p-n#DV$%W7E2;sTKKR2pb|!EDOWugqJfS>+5!2B6`7B25K1IH5au!(mEw6Uu zBfp!3wfpw>bIHJge7z#Co=;R6D`+>-unz(8GyfHl(2mR_8zU-IG&Cq;V&dlMYx9e_ zF0^URGXIc`u8Psf^M?2vn`aA4dlp6=JIa4qrL)5Ur}lljIbVmP z%;v|A}q^l#461rOg`| z66Z+sDR!f&5k+CWdWQ*dOOhu0&HQgs3n0)a(r`bZTzzxgRcE0~pU7hNfC~URF8CcC z9bbC;HC2gBJ$I8(Ou-$Mz-x*c9h0-qWB}JJD73AvA2mLz33U=jyxP7kh}Pox50NrB7*M#xBpps zErRakE*+|`m+_Z9GUPa+B1B#PD6_& zu#FnapE5WgqFmuMVJMm?DR1zoi5_Xdcm!{i7!>lw%`s~ImAa-XDR*o^UhM00SJ6Gq zupm#toFAd`{(EDB>tERiDJ^^a?|N^2VJU7n-2uGOw73vvF}X}fON$sB9BhX0S^rPo z1;SK@8A!A>-@Co5HX$Vv0Q--ur>BQ3+9vWtjs&-@0--@dsr)hfaSv5Zl0`uov)W^Q zCzy@`qRUfEuxziBG4e%Abj^C82}joP1FCcf=KkYhEzyGr?d_lHV-ibnQ< zwL_(}8R)Hd;WayxIU1UubK(aV)dLzt2`iHfZ1Y7R_e>49-Cq8-m@-%gas_i~vf?tq;@bBIiK$7s6X)k^?8m%CHvAqKP5}#V`FRrfB z71ulDA`^h^6oeGnJU^As3xmS=IjY=3=lIisavmd?LZqFSrKF||inlp-Jf{IExvX;u zD^_ukH2F&?!(^mMZh>#~8YScA*u%N_$@cwtHlyX^JOB%3(n0}4jzVWO*)DdmQ#(ps z68WvvwcsVYc+P3m z<#f>y-jR58#)AHlIHVjX3^0jhXG_ZexJuZdtJw)fiU52HeD;@mC4|t_kMq=1+YP@1 zS!L-0b>vAMLLA>U)@r1$(kpmn3yoa#`Om;h2Du02JC`G72KmdON3vEn$ukNwXE`Rd zlEnApz}>N9L$smnt!?dBRAbZs7P$y%X=z*Ybxe4i8?8%f>~9F3{kI*_f41Z0fx7|N z4%e&yN;Q*_CU(Fq%lN`ee4Ijba&?7$<=lIS!0(AUK}>sd+{rK~JG3yg3b*L1JEn&u zeYO zLXE4wG6zcwi~hK^OKrJ3OldfgG(TOxnYedeJxyS^cI-c-yTJhmL#)$zzN4M=ZQQ}{ z&`S9~V`lbK<(u@Sfj*IuFA8pg7)VH-0Hv?mv}JDPe@WU)FsB*yxotOKy#*Y%Cy9ai zJLET?LgC5S7)zqKk#_Bgy|$ZlCCT1wUsvTtqolx_3;q#)_Fm* zC=elLvYibmky45Ix6V2DrD$$A4UqejrM_2f6gISPG}e0x8#kODM$qT0aN{IG$;4!o z%>wP8$XZ4-p|O*K^4}TqwE$Hhjn-AA^C_@7`R4uTMIh)!;N|62Sj%NN%(hg@ z0S)Fr3g?9?=b)>iT?$tHbSIe7cq6uYUx*r_ds0$S;KV{%9@-8aA{EWiyvqusz;T`a zT#SDx$AUzmcZcK)f;&QcjDOBNXL;~B?&;Nvd7XYFm*Gnndfak1$xU_Fb){k-7-ZorS<0OS6{XqMQIX9L-PiyS~Fw)+zqp}|}= z(hXU5Oarj}Gix_-F=;?t@PJn16ue;FDC`He>&ioG?fZUePt+!mmd3oQO}uB}ai{dz_hLAKfJi^b4~W z@3`0?N5O|3LsH98G~Y1Vrz`IjbijRtdWi;UlF|R^ivKsERH+Nz8M&)Va-Bw-h>6}z zjjQzmr`|@*Y^;B1LJG@mW$*7AoDf6YU2Gp_F%vMcwvi?u2E=C9o7$xzYJ$?&pwEjB9OmkCs6UH?`Dx5bEnui1f+mHYmtX)$gD4huB! z3&+}406xk9U9D8ZOyT?&>#`x2%4=3vt8Z0RBA|!8Zabu_1~HyNkLs(xkz^n5@cnZz zAl;UMVUlXuwr_!xKnPqB3^l#V|Jm>labTtgPd5-U?qICyC^{)&F-P@Y!&9}d8OMk* zM*jQ*-~~Za)}QD{U`S+j*oJ`@ku^ddOG4FERQ(Uk(xBWkIs+l~{`v3rAw3i^`<>;s z-k@R=9s}gh5r}1K`Ha%dD}HMRvjHD^|E~QPL@1sQ`(2ryTL<&cyH8mdUtuJ$8KZpq z^hq=Xc*nGV`_NgP!npYlU25Bdbar`_#|g)3+2>k%jPPNBKuVvAYXtt!O)jj`S*as0u0;Jr(UHZM)*$4ABW-V$(mhW_ye`TAj zIoegE=5VY0`EuY8ySl~orw@zK2ry~rlnSsT0s{l#$X7Ghss7928v;K({$2An*w+vv zK`gm_(9EiUL^fWhXQ~2+s~sOdh2%-pDz-dF=(=Gq9&Jn}eph`SYf0cV@b~p-VYTg` zhM35JylMcIToN;0I0nS;Od0(UWBTf~z1!d4Cn1NNbAR)a-86ppRp8IDT1%^vw{^LR zzZ+OyM+6P&H@3F6A|%j78ZjVmAW`fl3PgMbn?X;ttchULZ`qA#vn7s}QAA!O0O6zk zS1QQ@&4(z{1%n3j!6fzsLGP9$x%5vvqHvl|_4A8Jg_=3Y$jQa{tl6!@1wi_In zf;?E?@!5X6{FN#8BED6DXO-{OQz4@(L3yRb7et8e5(kdCjfks0ps@Pp22G8!FGfU- z{n}asUM!_lfKa{-0X*p-f`$j}zh{L^i{(`2LOjMsuyKhAODg-7NK)Vxe2ZUT;RFT# z`Rry;p?GvG=QdmFiLpopzGk+3(w;TPNA{^JneE}TkNa+o&a#qa{-nSh!pl9e zin92t#lvxrIyK?J`7f4CWa{(tsT&=(KLL52S@G5XrE!Un2|0|Jz53MuU{LL&shYhn z)j)%*3wRp|SyU`<%uew}5^#fuh@%!-9lgnYsJG#Pb=;YF^ZtFty5A~-0X4*+Pw9iW z`06MKCI-YY@c)G!2S;9x!8xk#g5<&l zzOUD!87cHfy?K*aOHqOF=|+W_g@epkNf1x1<5P%K4iW`~f%LaZZgT)Xn>Zvt#rwRj zOi5n>v`I93^*49Ogs_Z(y*ODx(s}axBUo=oxW3Upk-5Ylq_aYa(CX+6fu=8^8(*2P zg}D*kU(8z@9siX7S9;H|LROgP7R(>F19fIGhqGIvFRlPepzxX9@uz$7L0lF(k_zc3jNA^YcI$f;3-9iEnuNY6U1=%7nw zNbF$Un_eCtd?C;RV}RH1P^afLh4tdFwdK3>eI9X^%wv@J3)a6Bdz8DL0pZ2EC9KQ* zOa5%a)911zFwmFTWGtR0m$KJqIDABozpSFR9Rsiznmv&*ZtBpMPV{PE7_X7Znh$Z; zzcLMd71Dg0@jgwK={NFgNfKaN3oLi>2EtVSU9#_SA<0jGdz|?GY!x-5ZWB$}da1dl za9&|(!CM$LVox3i3IdraB||G7mQBa@=EX`vH&2fKtt+6>fQP&EMGiXP;5D8lO)x)0MQn9L~GFmO${_Gs{ z;81}pxa)t{MFOM;wxMt8q@e05{G!##$tfcqi({V3<{Ii~E3px03pOn+UA~L=FxR4L zfI)gL;w73FVuWjoVv_^hnysDJ`b&KvY^oU)QXc_E5m(;T)9^{heQp+EY+f=}`27X- zQTR?^+l^-QMGQK|8>SSBr(6w^&9PmVG0~{lr=oXt`nuFJ^SoQ$FHhId3L4uP0oEK> zn(%{IN3_`jG+cK)qaS~6DrCBBpU`O3n;5xVZN4OXPje)zS4k^!Ok?SiSjyk{3-g%f zq1#?myY-XAWwv68fhH!SI$5j<7XCeKf4`JXUo3tDp~{8PsONAx|7Vdh?Y{klx5QR= zq1Y}D4Dxra_AL{HIZ_sxX&4xA*?h5s z+pfP0eF}NU1$F`v&{`i?{Q~{t_0T(dv82d)5O&Ho}`i zK!iBz&v7-<#oZgyhLuH&tgO5U!-|ZKiK#HRNA+JonnNPOU*La&B79M_$N~Zqs~OU_%~>mY3v6 zDo5AbL%*Vk0=xLZ3xuOM_W5M{AxJU}Lg*R<`?w1mWQA5EQtF0Hw#0C`!o1S=M(B_y zRuhAEDG_1GYc{-U*;rQAdn>&TFG%$>!EhFjo=GXeAcfip~&B3y0JYdT8vp zKU&kHN)**6^^y8MRqt{?jx6FE8)?16fxm{{nfvOx4m>cEZa;C25t|%s%HY4*sXxA- zTmF@*wR&mgHKOaOHuhD4hkW(M&|~IWzk2r?F^gER4cm{0jN_)FfU9Wn8{ENCa2se* z_|MG#!4~nOb}oIv;)g`8l05FaduHJs2SGo7J{wADPlY^c0?hGtbSwxGEnr=KCRv!w zvi!Z@XSl%aQ(^!$f*cCOx?@Fue)u6q!pw{cF$q2Zhzd4z0Oc$C=T+1LT9p2=ZRNeI zTY|hmz>G31sYdltSVqgR1#eELspR4>>O-RZBg92?-o6EA;thgG_^(4BqV%xo-x)uU zi`p_mvI?Q_73)oG2MA{I---;k9i26@cnZK;ijeCadQKn{-aJDSpn85MY2lk4vzk=5 z9RNOwmEMLJ(QYu5(3%ZU4%@}h&0!&dEx4w~0$i8uQFpiFATQmiDmtEXe9|tuxf}iZ z@V;pKUj0^S67bZf4=q}cbs9`PCCw!v5_M%d9o3mF!Zs77)FEHHey*RkpzeO@n(kjJ z#fr5dcdJHiA)V!0TN%hsg4AYtct+h4V>u1qt;)7t`-Y5T0Q zte7t=)QDm6*_>5Zo)>fg^ow(B);;^&&@_Macji51kR6zxdev!748OWx97wuUoY5 zijP>T%`ueDCDbIJzq`sw%CM&P4oIG6L}G~*RRem*WP8=!$+Bjjm@biMc9Ae;P46i# zZT9pGVL_26W`72RTZ$Q2QT{*P&H=&O#ZLRet4d8O4Ul4!3Y>SRo`Zz$BkZu5BeR`R zhHSRs0Y@=^WI@z<7nHJyrOu>Z=A|I#U28KhRC#4R9+wwF@s}rAePs0+6lUE>zD4yu zYPN__6-ZQS)kuhnMehqj@DOvXgkU+p1ci`%u!VXyS|sMD-bUG;ONXd8nN_Lwk`!86 zxMfrJM%TI{ZoTEQ<)?Srrhrnt^TvR+?$?xoltjaw@74P@pJWprqOVJLC?ZQ1#ole* ztTg<*_ztxjWlO$(+7Uapds4g}N!}Cka)+o`Dzkt_<z(AL6D~#&kY6erK+ajg9+Jkb^vkQkz*PNWwpKh%tlaEWEjtpda(jyx68&EBgE{ zne~CmPxga_-0ST!?!}s~%qBr+tycAWGqqwp&kpQ0E_L5(zj8~?isc()akR*<;TR<& z^eS~etUe0rHG;bL*8li1YioLDM#FVAn0nwd`D<8Ip(jwVs0gV~`m;wg=YC;GOj}Sp zbw3dk6N^qsVW3VWpzrtY{}Pe9O7_y(F-D?&iED}9FoMAQ#5DHfR4T8`hJ3seA#eYc zDU&L%>)EelnY9K9Y66@10H>3q#ZH64d}8TC`m(6C73U-`;=A2++4cjpON0IpPW$!o z7fUR>d`NFw40nig7mhPZbdTdat`6{{zq)!~H?d23E!1`sD2!`8o)v6ZdN=i* zEuE(z%e}dRsX?K3h&#Q}#+4*GZ=zAMB8XiH>Yjj}o%6CcOK}kRfkuiZ>JRHMtt2kq zZ9iLUsioOepGCs=94sniWreAQPG}tBK5R=N8*dBUdK~EH-q|@Z58cP0`G(FtloucHRH`FD&N<* z2EPPd;~bErceXfZ+!!Qw+bN!Z7h#meG5M8WC%0Xy@{WAF(^)V6qBqq5>7(`ShnPYg zhXcy{^_a_ez|+StZl-dhP?(Q-vB3D5K910)E}qDKd>TzIdr*liS|ZAwYM0ecE?R=! z^P(u3&u#y+vEk@mubgE`;ps09$H0=>jhGi}EA9cw-h(b5GhLl(wo82)wd5bQw;2VB zNov*df8IEn8>dVzZLKXOztd@FZ!1|EpObkvqPt57H_c}!4R&bWa52^l$zv%ljtb`H zt$@?p_)VYTMg<}7j4VNoQ4XMI-Ec8yzQ~D4CJs5ILTxM16p;iiLhUPqDcmY2Cnp0@ zl=rk+w--^xj=#w`h68UV)uSxk$=C+Lr}dg=SDkH9$Kh6lEmu<(8RM+A^{#8eb`-k3 zU97D*hHtEE#6QY!z_xNp$0l)skBZI4VgEcnEhgTG-werqSBcBqxclN&vRt$zvZ*=i z&&F^${n7blubJv!cW#s9zK!^z&Y!nX+P)J_Evn=vT{*|@eL<|FNL-6L=o_&vlOht^ za4p#_V3#7AJew`H0&nYL6h>gM=lpUz>a`rY>8R^sdp>8H>FsiA>(2ox zXZR3Nhv~F~jIkSeeQH7DhQ2ja-S#A2wYk| zYf)9x`N|H7iHTIMIdZ$d!f@Go#6~fSbf68_$;2M9W_8S_5#SH&sQStIso1>MgzO!} zkrj7q&KupAPdl(SdZ_q$*zU^uecOw`QgWJS?ic=kuxM$RBIihz-zLLvy^U;`(@s`n z;B|R9B~N`{vrH(1oR-A!Q#aB0S5fc6S-qA`6_WdI`8Ma~pjLG>lc3kqK=wGTaRQ-i zk~Wzu=7TW{CixVFtLnIMd1F8MKBslNtmes`16!_&*DbHTO2)y!{L%h26qzuw6`%I<{Q-l&E8JzkXP8c|0_$pepPGph72Ur4sMw;%l9Pa%*%R&Itt z=%mkRSM2Vi@K z*L#7^p6L7V0!psLZ1wiOTD;ypb%V#UT&z`G5jY#S8kdLctmb9`4TL|;UTM{EAd&+)Ab<=T! z(L*;KBdgpomYOI&r1O}UX%A*ed`10LZ)4V?E{esfu=!#Q9|kn!LO(8XnQ1TAzx(zh z=EdD4=8uQ{L&h(H`3i`4Q!>K>WaX`V^B&@njcS&S`c^}~y;b$gkZ97n|dlqvhu2j!sJ2nP3j4e+G^D0|kavBM z90@{2OE9`)mLy34O@J8vNd76qQH(MjXgD~y=;&zlt%ocxDOMAjUcCL2JUv*{pfI7B zr)0Jgp|1;QE=5lpo_wzKe0T!rY9Q|Wena}gheYqZ^>kdH!|fFBIg|`Z{ej=ZJDX$E zuaJA84q0@vSdxI<@gR?;4&yuB!!mJ_$>MkNJ7Qc~_%gval_&vXBQ zd(RK&dEfJF)|zY1F~|6f&nT*1@tOghlDF7_bE8B)u*y zOm&&xwpDTLPQ~YEM)8<@=8Af3Hq&36t&}Jx%Dmk{u>VvV9nefoiNwxl%RF$biM@Xh zKf%2|Ef#k)`5t@;{Xk}g%B)(-5htO?hvIZ8FjD)L*nVXclF5EuR62``W=m7dr5s;J??eH_Qsj|Jx_|(U- zhY6gbrD=R1{V)Py7H9fsc zX=!PNhVPG9&^}5`Bt`Ze*W&{0=r~cnKNfnXE9mJxH!?Dk^YEx_GL>3;LD`nr_JhMp zoe4`}VYnfA2z!liD$v6H^_kc`27R77JF6R~5--DKg{nJ>pAPE@$c6VT$Q5c1ky>{vgqq8upyNUpZui6VJ! z>c~EXdC7)e%#6LJ>=|d%N5^G=rWYCdc9x22vdhk&{XR+^H67Np`H7F|ya@H%Gy^lh zl4Nb)zI;eL*`Pnyz$F}%eC1K|l080-Gwp)w-c2Jp4f_{re_}fGZAI|;i)XHDwK}O^ zyJp~=xm}<*CO#l4Aj+e)Y{<+uO-cPxLcP7iN*zMB_qAGj)3Cse`56$CIOT z8+AK>=~Cp|2S<;ZBnvC`m)g!_V=m(f>Mmy&I<RX3f%j&s|MqRo49PG8%TX?hX{csA+%(mP*9hh*8<*ur8zS!+38xtH5IHfF?j z;R1K(k1uRNxM97N(d+c1Xe?7O$+V-9v_UM*s4p$C4zuq=-a8+kq8;qJGF8JuLPkRx z@DJ~uvo}GA$YPuO zs)i0&P^sa>JP~6LQ9f7S$iri5C)FIMXx)?pMNU>fXw4N>R3;l@IX*c)*?lg01y_#l zW^3ck`P;ukvm={A-$u@nhok$~Uo*XDI*&_YhxK6%Dx6b@u&sHSnU&4~14@fp;0pdA z%+W4Yo@h5x4vUMEqvjokn@v>uP&ylaswvHU$t($jit)X7f;zOk#zo zxPwtV zx#G-F=BDaTi{-+v<&=~tT#l$tw^v2k(i?lXaNCKaaFOK4maD!o}-l}OQ}At-JBe1YP? zBgFFIuUnsnm$dsZToVA5)7>1bvI}|X4SeBo!Z&EYoT~_?d2{{}QC4QA+>?NSX4MAU zk^rl|@j?QUODOz@I-}oju7l1L$S$_-)thK-3XP7A4hfd@{X=HZ8(Py7*T;w$fj5$1 zT!eeFT2u=+Gg}@yUXwOZ^KWnwHUJG=L%4NEKIme6%*kjkN5>*CRC?~e=YDFCP3RuQ z$ql>-&@oEiORzZT$7qA>uIeW#P=v?n781`!4apMgP(=9_eK);a5u$fbP&QVkUamFZ z4W@BiQDF`LhW6+{{uAb<9-}2Mo7EhCYFiOMw1UDD3JQv7Q#MgiU1V zK>2>|+80lw(gN{K@0|yz)0ag)DCvdE9`;R_?=+#&UYP8x&Dmy7l72OEE1qe6eLXB9 z;)6Y(IERLX)=BMUh` zE1kD3@5|D5>U|X9l3Zp7C%E!lTan=hYm1GD$LRC8K&VU4{4iIPlj9N&HrrQeqZbi# zs&zs;B?g9;O(|+0$ZH-7rcUw-yn^UZm!p$T#;vl+P&cQak*!QVK9ac0uKo4%oKbbD z6y`%pV{a&l37&F^FeH__{TPuiG9Su;+^~X~26FZ$=3RNJWQCh($?opAj(aW`j0EQf z75^F33$nIJ^N&wYFOEI6>KiDkvKpd&`F#2)4lW~0!EA(ESRBv$!Su)lbR_fYi&M_2 zJ?{d(|2T2hu{V^msfQn0EG{MQK>{D2CD0s#wiv4bxt%b6s2K;{eEH6leM(o?je(CT zcXkM-cuaIVw?}jtNB*3@t~V!s)a0qp03NRMVk1iYlpO7RW#KdtR)|oEE{^fT&PzGE4SJfPq1#?I{lCB=su77bC_aZhlrNd~vVts$SgCNxk z9_$v)#kR^K7Ln}Ir*F2t*${#u4Z%FJ(CYA(z48qHq1Ld7Vvd}=}uJ)I@8E%|ynXBHPDgkSe zuP41QcTlbQM8Fb-@wX%W_0VTHHBy*!U@K7bS4#hs;WE2q?=DTW?RbaBr)M&?QSJ_cGjzwQ)A z9i?cy|D^|GvGMr%{;-BsLZvq>^3|K=Z#wAR0fBbSI{m&kzafd6 z`@;BSBOSMq-&DZCQd_7%PPX_8MI}T_WEQLjVos0JqsFx~jC~uH?%p+;uC$U@QjQ4N zVEOv!5Q?AB<5?nmpXO;3+$tv?4vxUMF01AA8A6=g7ip!d)2b6Ng7y<6r@YvMn1^tp z5H-*!(MQW#-#bg0z9UI6rSf8vl~ti}C(ml`3!5^cjNe>ZsAcd$6((l`y5*RWG;_U^CC-`sqzc)VO)C_L(-T9A4pc zg5R?xi3cjn%M(y^^}K2=%go<0?^?5a{Pq^UtFpm(VcJA%SV>Q_RR0G*-mx%DYwl}p z)*E|3+lC?#S1YQ1>t!oY9U zo%6E+wLZ^D)?9B^Uv#~oSYoBesiB|9So%~h!X!}SG9T;L+c+2+a};tayP9kc@z$#O zUbB3IR>V)Li$2uVMcaMe{uchO`(#5oyEMQP z{7T)OxV#SqE9}Ryas;yjr7Ft};5F`T$RGglhJ+at#V)|hMLrhJ&Lte*+t^TiC%>p^ zg-U6yU&Sf`%DMC-(dvoCT^?-=i(AcEToZ9_KrTda34wd(oQPlvxTV;LmXjGPQV1QEh#lCfQ8`ykIi zMuZ|IHzQ%`;MYnuUss+6KTQ72bO5a^sW#b>KZ!nbS2_~@(<1pG(!cW(I6_roxTPsC z-ZI#j$b7(2=coleU|S-eJ-d;W={!5Vz&(NE+|V8NEV%3zIylX9(W|;FqQL8!T%5^C z{}lI?3#@70dU9vVT(P1QLiODuMbS!l$j;REf|uh_wY90DSM#;)yCpIbrY6hDw6L)7 znndkolvaf_k)`dIf~)Bw&pCd6|2AGP8blC`CMmdy4wq4!Y{u{6bvLjm(eco$l-lMr zH<}rB3_>5r?da(ETwzJbH5!i9$vW|)v)ljK zi2>%M9nX?5+493SkUqj3VpW)~y?5O2SU~=DIxbQBez&}&&Gn8CCI0i|x=E+<>B#u= z6<+;pqMQo6kBWtzpGe~4^b+x!(}yiYF|YsFdD5&?l&5N8aY5itxtDPjAI{L*oE5T_ z>=GzSkylaq-NPJnd{0!xC_q%-OXs&d6RcLJWf}f1g(KRbzLH)`Eo&StMRpuV0N<*w za`>V7&-efv=8p0hxojq>QJ_M}TxW7ijkeYnOzKr3@rv=}k?*_EiyqM@a&(VKIsfue*_DjJbBXp53Wv}7x`=J;D!_8{DaiY+}ZOhht!mRDXcKG<`*Y{KngrCmTUW>rf)4w9A7aLByCd~m8GDEM5 zGSN}=63h-&^g3N8=i0|)GU% z856nl#Jq-#vT=F!g0H4J(I9(GMb@_x*6Q}Xl zQwf*17?TK6f;BjzPBuA=1IGM3S2BI~F6P}-1JhP=t;jaL0jQbUKPHO!JS6n%nypB(Y^JZp=vA_#c<4Q!gje82QXhatCA z;Se)?SIs0Hsgilh^?`@>VLKnM^V}Tb>sW~&=C+fIjml^llwh`#k_^oqIoX&i8FpCI zqWR6UXB1(w4BY9|rucR`&8KJ36np6N3!+<_eTa05k z&Iue*wjW;eRlh&Qm_DDbBH(7)yVfL`;0*g6P3}R_=x6iMo^=fM>5+Ed{8@T5diQoTsCyKc?JVO0C`*Z7OiX+Z8FZ@|4ocUpD!8H{(T>zP_5da+tOp3*Fyyof8(=_IK1S5OhuN#fT+T0_3+saxfpsDFJ*z z+d#kb9w_%aadS!=Qz^lzsi{&x;2N#v-bo!Z4Na)=o84OP9T=c&`Amkn^ucNR60P#= zXAYaM+Uf6umS?-{3mr72v5wi>-*Y_#3-OAd5C_y#i*W8iP1& zI^ipfGU9|WEqr&1Y)ABXu~JuWX*W&};-NNFJQhGNdqHJ%Wp#sAPW+-_VtzVc(@1BI19zFUu z>VB&o>DIz-+@@Kx8`5JLRS}!b06cD2I7X!en{&z*o1c4?QQzxjkm&6&+w5uFG;ti~ zQD=)^lzgyp9CbUdImONVZ>7sqFId->&ue*jh^!2H{2Aoh*2l}dTi3Zuiu(A}z+SB1 z+m__sg!y6M7d|fwDu;fH4AbGtB)~gd(&o>N;)v^297(;QmNRG(I1@nAevO%(2{Y`? zDgJAwT=IJG=?)U~=r2_GIh9Dar3;sE+g95oPsGIPAK>a=!5||$IZn4s&YV2S0`$g- zrd!AE`$XXtJ-#j@Gd|Ey7ahxypb{J_RV^$xkGc3Q8<@SGFEOIa)qXC7-vdE zARxs#qzR5QR+4Bck-CcFy+xPcH~&2ZU7HAaVykSfq(y5FM);tA;w`T|n^U>~G@ky- z$OG@ie9@ijtc!QE8j`Zfj5|N-+q%!VJKLW2a_RAUK;@}U|2o~Iq20PaW~5U{(Z7-H zXijOW6V`eD!sU6QZhJwD9fX8JSbd8|Ms+3BYs&RLNIRYjUGGo& zmdGiV?|_mciU>ZV;C25@LjJLC5oTKe~3nt%fw#kzzr)woRWCKyK74)QQGBu=At$b_I2=$#U;No zy()LabET3}@WXeNrN8plA&E)T-H~0mVljwrTgk0VE~a{Qd)~54ZvO6tbHRVsGAP!+ z2WF*T3a=iq5PWSSryL>GYB^8g*OQp1k)KQg(a!?YaWTHew2^pqYj-7?aEYGdrm-Xn z%F5}k#V&L=O}?S9qe8M;an>+_ja5+bfHLt01GYF>?rYrBB~D1h^ML*uQ;Lv?@IfvB z&XBah&+8p2cK-emmPa%b%x}xa&55<$xqDP`ndTycQ&Liq&68ErpTr%sl1Hg)WWv8S z6(!2 zzN4*0gt#cYZhtu;T!h0BAu1u5T6hMn&Gq&3Ra)`)7V$47r^9 z;kX{1qyx&T3bnCgm@AqokSrrB+mtLFAV<@us64;_0KloCGf`~`9^*;^_RL z2QLYSmxz?*%*Ea?G<<=KhJhheslX}C5k5ED`;DFdA%CW*ixh@?uEQuAHWvt1abGD^ zV{3Nn;O?(x$>{BkqAqUl|Kg*_1Hh z(UcroU1cgl`|6tr#T-;w@Z>Yj-iA#gc>{mWkiPH(P!rcN3i#aQgSp46no#| zgMI^!+atwTdx;BZ6kBS%4q`Ws2xYa87qR$-3!i2s_F>%UIh-mH z@a5INtMiHHirwA&F+Df@nDKi{CIcBgmVy!5KO)btd*nm+9Z0CqZ{Kc{+X6#i{H@M^ z;+DVxS98GRFH`g5uw#p${0KM_+bG#&m$WY+9>3o{sDthX8Dj@ezeC*}DJw(@< z!V5i0M)eZr?&A>|LAh#JboBZ1gv7lo;6sJ?>C-2pk!sHgMgnJQH~U91A|H=OX@^NL@CJ`=p>>6do|6)V!g9MweQ>1goGatr!5zYBMa=DN$Hu+efH%|qN+QtLk8d}<8UDDNlmK$zk6gA0 zn`41NL6N{;*yB7TrwWMP`<7oZu!2O6H;L)l1k6L&BIra@Tg2y4;PYk;*gY&7?Bb$BdtInk+P>*Hgi@vinH%e7 z;2*$)><=`m(b@ROt|(+)02-mAIFmGtc&cRKA1Y6MYv75>vTNKA+O8@v52@0*%)- zlJa*^HVEX2j9^qAz$^L#2x6JCq^?ftAVzbI3OhKc)A;EV;b3yQCp5lQ$g5iIFL!J* z-M@ebX*&WB1a)i6Lowf#l$H5qkL}}-E*W|zCP`j5qhcsydp(1Us4JchYTegAhkzHQ zccZxMe#AM}rzhnU#Qk%0EBHyYbadj!#>TF2-1xSqKMH*4R66fixa%}qBnZ3k`!nVK zgQ5lz<6zA$Q&~c2hGK(TIqWt`%cBglZr6|a?5P2?6Yk>DlC%rW^o~kS`byw(rk0YD zl2iGp=)sZtd`Ci2HuCrl1Spte7*$y*;s)bVF|^FNzWbJY^cqTQAoGXK%^%bYpZCAO zhYF#-3{S>9_=y5xrS3Dse}gdq318g)J<qvnQA&_!(eQ>@0X9v+;P{Qb%+;N$d)1 z^K>gxlakP48&gb6D@~j=MJ$65(2<8&mLrfV9>y-uIX*l7Kw%Hls#T2MCsOic3d$-o zz#$;rzh^HInnO#ek^oDHvwbAyj+Mx9Wikk#>(i^I0+l8H=gRC?K8vo*>x@~7QI`yZ zu{flD&wPf{)A@pOK3*=C%f&q)6nzpv+-AGdY+GU9r!x}0!XV9RsY4s~;Gx=Rw}w8S zQyH(}4J|GAh8Myxn-3~#i7J=Ne~o&V{?Q~?-OxE&qoAO$y2&S+d^fYF((P`z(ituT zpZI%oWnYa9FXFztl;mx*ymJ`T5seyAh1lT(BJ;p%iJY?oAltAw-qc*VEW=+F6Zja` zw1N2nY4y!B;ydfDSAB(6zudv-CMzch#YD{=BQ)-O*rB2C$)fMQUA(>lgFJsH4JF!t zJdI?Glx;0GNuEY|GK}J53k$j1#iM$fEojv4))kWcHN(4^2{+1@t9|}Fbc3>pM}aVng-kW8jL{Cqs-4xuuudF9Okf^+c( zVR9SEV%M)|_`aG2N-C5!=m59-_~?T~tsAGsm^R??VxPstuz{y?d^uO#sR#Euj2Mv= zqodtWF&HX$!ltTr+uq>2>ImVUxbKG0eb$kp1;+o4KR6*40&{5<+}zyKzATsUv-@{J zBLmqF7%u!9vbFuzHxhm?((TI_x|SVIv9d7iGc=6abzSfTFm&c*{!GV6wJ9}DYz?`L z$V2QT%lt+$t><#PLyO>KbX{nt%P+H^sW-FLW0=E+QB;Fx_^rf8-PrZ=4tu$ld_ z!st3rIAzbfZR-Ys_Bx254$0gkPaaMXaJ}>iAhYatVSF$Cf`{ zN+qe?C?LX|5?vd8u*sq=nM)f!Urn$<{P$)ow_o5W$m^=&a(P1!>ZI{Cx3l?Zw&s1*e&DqsUukEh|*LE zw8^6lAWh_fWFI6EySy)(hfoXM26(L$&4tyK^YSLK;M!3UTd$kt@1V6( zw;U=jUY_YyQ5Kp`J+nOaywrkhIzmG)G|~HyAjxvsR1R!#ol<`pOx|4g3JmD0i+Rx6DRsmI`vWs zWIbw)58XGQwYrEYEY_2nA;rPaO$U z?h>;v=XLz|FhPD^%J!r-Vq$Lwa}%H9LOc2%|EYm96q8mQlG9Ul&&6%A@mw0vD_jiM z&Oz|Y1FR~y_mPztp{36}GhJuT9$-M+Ek>|tob1gawCqVEB$j0IH6}(z(bQf@njRzv z=QyK;ooBiHQ}`@oBEQ585Y;EokzHR^ak# zrtWxlYj@?Omu_%q6*c5BOdq71k^FiF*iD7wT)6$~+D;X54i(XrXE73hvzS>}RCNZL ziL_XpMPIJM^r<6WP7v*M_{5gsYWDa=;-@Dk*_A@wvXS)Uu;#0o*IjRvP^{9@C&Q!zbE(R_98seZbsHnY2Eqo!UtbxC9fVEH3@}7Qm@s zXlOY4;7)x|M0S34`2_6(K`Y@DamwuM?6$L&gH-ct#g@8#DW-o4FL>wId=0-#-AB96 zf}N_Y70{QZL#ya-TvXeTOi%Y{Xuzi-CQAtf3LI0C*x4@XRKEK=)vvk*rLLO-lid_# zy^HUNK2NXjx{|DJYO?Pz9>PgP0uSqlYnO+Zl1g6zK#ZYkUPm21ax4UV+86>Pz@$~!_rJ6lV}P`l|g^Gu~e4-EutGzYZB`z%dlX!%Xf_Qrph62D8= zXH0eI_%seAb(@jNyD*y5o)S%d;S~C%overD%B%sHnrATZH}3azXw2n1ALUYH0*N;T zhwWT(L}Qmhu^$~?1MS-QflA+F?ZhH+DK_8VqdJ9|CdC1)Y*(PS zL+7FNfH~~vZ1)+1cMk~)jT*_8hM~{pComYrZMN=&rQ8{J|lgB_2Yqm zhr5M%Si#vX@uDnA^2{7(tUQ@$MFfLhKXS?PJx>E)+9Eg_cqRs5AWYgW9u zo~e8xMY^6V-P|ySG?Bp7rMx^(tCK@OpjYX7cp(VIYnOAMKmT=Fd=c7V4Mj=^IT~#n zvRauj%-5YhNNkF2vue1=-m-?qM;&l(ZNw6`5xCWB--r;1eYl1nM&V6&q!B65kxW4v zaT;wz`OH+az{qm#=V;V{0UuR{>pZ6ewA*MB%2$!m{C?vrsEbDkT6% zX)1|g3^dq0V%DTmD^Ga%L@8WvwqViKQ@7|yTtWYigX!SrX=nT*`S@5Uc&Tm%CQAFH z3GfsExS3q?s8yNlZeb6zgf7Q%R%Jb*)WQPzka_OD0ND4#SksNG;TCbNnkIUhxKjiAg)L%|679q|;p%y=++3d&-!UKVukrRSgEP-wx;*@y1jk0@ zZi58Lk%NsnKsIYq=y18W^i#$j^iAIqt{{1Gdd!qWh3=x~~Nhl5o2m9bN)W zP*kZi`W?P{rp&Qw!64-_QH(xw;JcaDzc>`F3$UVFI=Gc_6bJiKJ{9a$yLNvOn&am$ z6o97?fAVv&0v|toXgn@3Xt#D^At;1s>WtM$_0X~~kgn5^Fw;<<^uC|KjDt(`QZ@k- zhTgn$$PgG`>7PPlY6sxu5Xun2OU@2Qq-tYO?oX{DK`s5)YktvT$aFCxV=Q_=gg{F5 zJRS4lBxl$qejQR0*YOAqqwbeCO~z*ODwLN?gtd(Do;-PC1ZF2$fK=TYT*OF7WR&7Z z%j|RS;QM3(N0MTpwjhIB$;oB>iK0D$_iZ4m?x?=YkTj`I{5o_(iHhXVcHg;bKYA7Lnew#FGj=RH1@9NdJUgS0dCAr5y2_bPFXMv6H_6CvrUo=Vz zVoIkemyU(Z%_6!+=%Qi_P%>ROH#D{4$oV`^16_`;9O;JKhaaF*jKr&B%c_c40&>;{ zXk+v_17-}cXT&=waNPTLhwo&Y)XU-~x!|mu*18qW5pXNjffc{mndc^caP~evhvAe2 z4M_2Ea54yh$3iL~pmyqJA4RX&FTx?W$pq}T&z>90U!Sbui#HHaU3vx{-WRDC{vtzP zC0=WwU8v0OucKu<9ml{%p9ZJ~NWGsLziN+z+%4Wh3M|z1gk)>Afr)5TVAkY$O*p!d zTU#ouT%tmTw)KGsP)2O_kR`|YG`x412wSG5!b)f1}{iJt4`br z5|n{vXXC!+!tOyeTZ;ixOJ>E;C{oUsn3jh2b=-qa+?PH49pCTwgT0R6by%wOqdj;5 z+Q1uu{wMM2uUX;@IXQOU)WFLd=?MZbg;)km>Yvvd1D?N!&T-ak6~;WL+^L!^Y^oHx zsnM+HLJiqBAL~SbI=Op?o}&l_t{u52l*wwFn6S!rN+>GX9tiSZ{s156H@+3*Pc1)w z1$dQ?%dErmv~xNSC94p{iOAHr)QN5AE_D6zR#w$ z=xFFSH)>Y3)$XGD4^!2SfF<0j`U6?41Gvj_ap3K(W`pIP6A0=_Zja@)u`2KU^fE1_ zg4+h%4FTo%;g@F?URKOk3v-p85Z`JWB#{#?^11mwQ@_S*qBVOs+WBX3Z!A;BVD!h% zgW>>u@Z$YUVGpgS3WAEwxk0E!0oowARt_jUtFz)=hjVnwuFC|`n3s#LkA;t?$Jr3B zkSuE)#cs~mh}(HA7We9rR~>Ji#{{R|9GZ4X5|Y_4y%8?$J{El7Fo9tB0M+7W3fAS_ zg`|;qN=nKjd1bw0q1BEqa@|rLKp4`x-O+~e-=hkyRnNC#E%OMn86MDTjlQaS9$cUJ`YAienp)1~33hfvN$ zfVlLvT(vEk%mY)`)?8&z4m^kPUS{_t2G%DNIU$X{Z0O>${$U*5vqL}AtpU-Q1a5VJ zj8#G8Lesn-_)mcIb5w*AyOlY+nF=HlaETb|QrVHRcb~ay<)9faI7NqsGC)`47u2bd zqL}9*f_4GD5V>Ihw^9#LVb1uK%N>f>p-QQ|$EXoJT=ROapeZ#MifQHQ(h;u#=0M^e z$ce!mXL_P9e7g@Hb#Q0xpt7E+m*X@J)94ADs$y%P^|AeYLNfrM6l0-J*Mg9|%dlHU z<@!XN3B!@&XP9fYf7&t?o`+fBQQ(Z;I8z}X&YJ#F1thXLDEC^|X1lHdaF^Vi0!6+0 z-a{0Yz5E_)cC9Q*yqDAc8Mj}`b`tj#O%;nBrqZ#)3g+!I-UvyLPAUO`v|z8FgbJ~X zDD_v?xrRLU-=ugZUi@GN-nZ2wC8vI9)jhaW+v;9_)=dHV1|vLIvGNS_=1oc3EhaVFo_oO6CaE?#m>(|Z7so)dLy38uMaF{pKBm~S|5c-jsf?5LJn@>1O z?9bXARgI84#Oj9*Q5MQeUA-at2jBp^d0P?|@XnhK5AXGHs&J)FiAi@IWpb@hs}YZ- zz^TgO>Ckt8oq3tlvHL2HKy!r z*6i>!hL83uiT_zkxu{hn$Jjq5fd()As73 zZE)Nvb2(ERPguuJwJ|*^Z;~hS@6FdAiF7-grRZeBQLHRQB+#LeIHoIucUeW(!M`X$*OD;Zk(sD-1E zi4R`nk|wK6eixVlvB5hH^Z#KDdJ=WeGKS+OTu>?e%x6xfqxWr?ycX3Y7|<^6&CzVP<;HV~4fp1&&S~g6 zMVN(}WJf@`C)hsQFa-$B`ZYMnE`Es>(~o^qMDqYAXGrR;etyj#cg+gtTx7U|9nYCZ zc}$C6xX%AYTD1&-w2)#GS#@AxLW~ch1W?|i6PER_9da0+M;`4LJyc6tjh6EGYu|7iCw^l=ZsjVRq z=L@Tks9ZUBuAT-@q>Geyo2D_@PlWZ@UYQ*dRJwK;_J^3icXts6dkO|c4+jPdNmWRvLtp4a`FNn)6Pn`5T zSQ0GmiYmYta*an2s;qs5a<9YWV|&?Wi=_*>mcPQPbbk1sd(I{M)28Alzgqni@ZAx; zdiR>Um^$iI?e0OU={UHeIqHjY5=~)W-bS@zgJQlDGnp}YD{3mDPPLP&GZfRgp6c&1EsnF*i`C)&0~654Vb{7z zo|YmPA@)iPP>|RwfJep`n>d5uQs7Ym1Q6rH zO2N&;gYk}i);^}h91A^prlGN(maUxSiESW|C%$|E2N3bbJAjC<<#5nS0dB8%D|Yw{ zG7fe9M|S=g1c)M-f<-z8w@&ITHI%RFxpYqjTElpXzQVN63BtWyN*&aL%e{(A2!ffpuwBg>z5FG1ST5Zu zxJV|nTEk_suCA_ur&UCa%`Nq8&vz@W-=+X9xD9F#O}D4`vYpD6KR@&}MgBr=OxLSB zvUi`Vge%0RKIwW@3~+ITrn@u!vV|zCb?mRs6ifFmrEmjv-Q>2MBL{>svf5EySnaT` zwh0IZKf%;oJS2n-FX#E6sp=oL#sr&8&(RM_r#{NF9B`~}htA4RAJE+laO`v&0?!4@ zwLHOx2|37$*@G>s2yC&fKSzqL%>c(u*5!+tWq(oD;N4bHu?V~2A}4`vh!dHiEW2Mu z2nNai4ay)kqa19;%eb?NKu9MNnl;mlqS7(JlGCri5yq?D`EyTgO*@WGu`~9+&@=MR z%sB9{%hwmoaY=qJfhlGaiA#M5po&Z4eQSF1q>K79ss9kU zk52{Y*D}j0KT4>KV@#4w$zgsp-0#$993{xi?d^N{-l`QGRp8ZAqV7e<&hR#j(1VL` zfPW5QNkBRYaTs+$KkwiUd;Zp4pV8m3V9hIy&@9bgS9vGM^9QUIGZPaNNt+^P6>~k^ zy_~x!X0pFefW8ZeN0mk-`vvfdCQeoN3|O6KqW4y#(Kcn{l;&XYj2KEs8p!QNZ%KUX4mp~A)X?1d|)pAP(R zTwZ8oUIrjGft)7vZ%iD7;Wj+tlAef!>>+FeVI?rW++JI*0qqFvUv*pSSi&j*V-Ki)xpk{vUP}kIb!{=x;gJuG)No z#Cp!ix!^e8GZ3C^)PlFH{-cM=v zg@bf6yBtFZgsJ}MBPzX+oSch11$5!%UmsJyJB!Qo3Fg!pgDWiq?X;eUfP-@5FM|lj zK`({CMqGIT#KvI(R5keE97cto6|jj1aQ7wg6lv@VXtXgf@VqId7TpR!ly&1vzuW8n zv9}6WHBk8iy%CyFMt2Gi$v)rN=&kwe7c2LpJw^7Etm%0S<@QqI4lfCKFAiXaAM7UB z?0o=$J|sRNp;Qi`DsR{T6U#}(|9HQ{-nSpR^^`BofSX42^dJWKjt1}je;zaas5Nq( z_4G|37s!z0JSS(A_}14GgQ*hbc2uMh1{QUWtlGh%IY6+__@6q>Icq9U*0^4rV63+%9c=u=-(^LP*^cEAy zw76w6)O**OV3CkG;*H4&et*g{)^C|#>?Y;yg2YjeI^WZE>b~g!=5CYvVc$REYyf#} zOmPO&a4oma6}YchoEGzNHTWhl?3w~x(Z&hJV?0RRhLp=abIXoY|4Im=Po(qx&BH^s z^gr4h*W;qyi05y48=E}RsV}eYq#1<;Yf1>M{KJx`kHDD2PfoqMR2`S6{$|%9Su*?K zV&s9>41gz0mdylzSSf#d^LNzi79n=%Ytjv4mW^Ss@sVV;3}vXBJ`6JW-@W>>*L|fN z)S5S;$mZ;8Bt4X89fFyV^!1e2qN^eC6vOl4VY7_lpqzB!s!Qcl$t1xNu>S{w0h$4N z8Tm~jjB&x$Q3M@>OV_hJ2tlfL?~HB?+U2h3_HySUK!)g`7U6l37!fyzA))ckf1hD@ z3xva$|u14Eot+u-$elh--+9uQMXysp`4mpA! zP>qwR?BQ_=zW+*PEu7#uM97INRcFqc(=zve#Y14)Mq&Tioi*9@CjCDSNbKexaRlT5 zHX=l|gA_5)c39-7p*)Vl_5JAE2jkDwl(DlUD0cN6k|8IfVxX)VbzO!Yo&NU>A8JAf zCi?AZVh~3*gjsrhCp|N>;ZRq;zCLo#>(6>uieDw>vCOqibqdOH{`!dO2^~)AKUdQS zy#{#e(MUWrkO9*;;=NNfMVtzH<`fo@1TKR^0*`OH(G7d=Y%4e=14NPz2=2|+Xw6U| z-YeGI&DcWI{|(vDkMiJDS?H0=K{E(!%x{=k1^4@UqWrA*5Crr!^#B)= zjd;!<{!dvRf{H!RHEeE3m}Z2~186ct)%O(|E7p$Ik*iSTPJ^UV>z6}bR1b5|eM}K+ z*T|n$f$6pv4>&Q=y#H*CGLpyjEP@{P`&UFQ;)g+!U07n`+33=qs~x!4Axpx;San5) zFQIZ+E-Er|g8k@TrrN&RRztA6ld&HWG}U1U2@huiBGN3a%p&kChLO+eOIzoWShRMb zk^fX4Q#MrTHt6qcXdpZWkk(`Ws4Y80UkKzEK-`P1h;4K z;;-K|J43S)0N)cYU{W9ki=?wOASGwz)6j3cmHFvExg`P>fk2ecZuA|zWwTyz#m)4fJ~85(RZmWgJiWR7 zPqko#{hcr>+=o2{P~bvnF}H}G=iS> zH?I6V1(Naipaqcp#P=7V@Bp8lo>5DtHY0~NL1hU{OdgO@IuC(m`YZKPfXR6qW=;tqZP^?HR z#|#V__)aBtZNTa)5HS5X)%Q**;~%Yl1!xwKa>u%3|x&7b%`zRW0^@QQc23Zg!w>i1lm5Gk|NrCuyFCa0>;IqJ|9fJ&C4N8Q WaZ0+OZQvpJM_yW4Do?`T{{I4iM!S&! literal 64317 zcmeFYWmFv7_AlCvYk=Tx!CeBuNzeqh;O@a)g9Y~lcS~@0r*XFs+?@n>hg-e(Ip@F6 zePg`$>3zDl$Dq5Ys+kSt;5C>x{atynvIz6jD~?JRClNl8 zm07JP7>*iKNnBhM>y;{eUJe2-R&M|jy#a^ZYTJIb0vFS=Hm($VhXv*=^ zanq`s`4Ktj8=?RuB_1A#f>k(WciUj}0t3lpgBFA=4Q8BxeXy2S4oC30DX-a|eZ8W`gA@YtqpO)0WCiuqH6<63u;z-5NNETNGc%H{vj&dxeLY|gRLA@n z6R{Y~{+`fqd!pmF{j(g`g@PPU+C5!G$M7tLwuT0~u8s9qqf=ko!RP44xp-e8bDP~k z!mm8UUx!4M0~oMK5_(}}*HEneSX3N`7o`cu_d%~h3g{}94Qf~n1=zxgQ$}jv$up(@ z#u=YfHUINzp8#CxdbMADOrW1Y*;m%#8MX`}*rk#?(if^H0g{-Q(4Mh`;qZ&VbFhcSJ3pDkF3$r!|#65w1L798z|Id&46L6VqL>29G29RS_*7 z?E`G@yZP)TCI3Ps9E>{Dga}4SwVYJNj6&gGLciX8dU1&rB{4N2TZ^!Oa~OK{)@dSa zPj}DTnW6c`a`=(Ns+dCd?xa3DP9&Tpx^aL**Nt&wGPmrDd;*m(mYq7?kp}tf5gN>i z{%T!BhNM;CIg5iA*1}hC^@NL2q+Rkx`1Wto>E+SQgPr=T90nUnmK>G{mrR$~ZBeel z3SBarE~i&FqTYhSsQQt15dAKU&4SZmIB)<=XW*JV616oQ4rVcOMqo=ILl@`Q52{qD zSZ!gGJ#9TNjj&INPLWSZ!^FSQ5XK3vKlg580pIbnmZc2S3h2nzQTSL zBcqp)9%m%2`9VSk_r317)P(%^X);pp@!#tw`V8V2X>k~*(2VHqv_=L-=E;W3{uuEa zsV@Ci8e^SkU2~wqmrEkU`mS8MT*_8YPisX^SfyR*=_|}z{~wq|_Odw&hVPZ$PbxTc z7^Q7Qehd8;ryQ)D_Y>PRygJA+*D>MjGj1u042|S=;7~SW>8EU>BCUK+b@Y#1su9Wp zIXvH^m3TkKs8=YHPf=KDSbqP+{Yln>lUw(74fi6qzSW=={3p_xKT~(PHrb>lx@C_V zQi}bWbTS1hyJh9YtWpiK*X?8y<9c=tdMzu&pI<-PKgQi*pjjZ;BHW{Uq6rfd5uTx? zp_xZ*MZJq0it5vFPDEtNa_J^8wzg?qY#hGH3(I+5J?Ig)D*Ia2Se97UoQK{9X;rb! z@Tcrdq|HZ59@{Z)3=R=fVOvV$&icY?cUuN?LNiTcKARs#xHc|*f9mS$UynEUdJgvW zm=(x1unf&~^31wI>RlE91cnbMV%5aF8Q~_-D6!BkbNKvBNOp35)u*| z7SwnD?QZIMX_IS%vhu6aa&mE@1#({%Q>_rL5X07C7i-sc{4J|YKWDYEBk@`Oc@b;` zRstXTH;S%8QXxU3u4ys0nJAn{L&!wSxwSN(heykdw2d3|&~`A)5cQEd18;)314}EH z=hY5^6bmNKCOo3icL{dYl1(*~=Byhv-3@P?12x4Bd(dVhy#fV@EwMC7`0!1!)rc-> z7dV=!JQ@0g@_Ec&&#=|=y&%VbIg7DLM@GLNx4X|9wRsbpCz9l9>}{)bb9SA?s_qQ`3odxP1$&5kpe4@ zNZLvC#^!vc9&uN5JEHDf*ZfxeHxb*ng11+OrpB1YvK*p=W1) zixaL?Pf@H zMZ9EUbi8KL>m-?Ya(aD!ELWqO@7sgi_Ux2rpLtz)2Tai_H2e}<R-DuXQ z7L(>7Ruo<4M)B78&z6^tecM4tfk#bDOH6S>okB4cclvFs`||?!quUEn`&Y9SSDKf) z?S`&-`}u>T(b}cjHVx=@4Ohe$4NDx2Q=A5GCOl{NZ11yA4R$Ajb7Z>ly48>7S6X~l ze?J|5jiMII6VAiv3+U@34-`uFAho>I4J=XIQFgIBUqs@*CUPuo{}!8JV)W+t>qlgFu4r{J^1&sk0%4 zyN$K26TiC<)n9k;1IN(QY*Z9~-QsK|M5Q68NFiqDXiD*#m7A5FN*INLfragTG~5X+SyV-=QT94b8!};qJlo?KcBzpG)(LoqFfY#=!N2bRe_Z*WC;nrk=08UA{@2KVT=~~X zK{n_D{;{CHx%JmsfL+2Uf^7d`y)cTBlPe6c8zh$E3aY>_^ceVngMcskzki{}UIlh~ zj#(g(DCoWTTUB@P!7`$O%JdWBiCvoWF*^K*d15%#boyL~3W0lHu@4&m)*$A8#hpvypG#~uPOc(zCCR} zuwVOr(vLh4d@7~={b9pqig{)yI`5JnzMgvyp_Dy7RMv5_{agY0v1m9-c0y5vNWqwA zxZt(lY@i~5gwISk{}MyYIY8)OT6~fPQ1;##^hcA<5pG$R^9?iw%9$ z)C?=rZZMiH*Oiu$5$o`I#3Up08Su$mDJ&{d+8rDkdS`FXj(|=S*wZ7f(9*I(&Tasj zDbwCn0~yq5QI6NxtqO1L?ota2lZgeRy>)i3Wm=voS5;FhI+GSi8x1z>4yL4``Jp*j z|0_4A$5czZ-rB#kzp?QR8o4k@PEHQ`;ljrC0%u2=gN*~&g96YFf63mp*AxJZox7zHlY&qAic z7N?hY`pV;cXTGobhOH9*`wxRkS?o`_*_MvueVN)3>z1B+mq!wfp_uJt|y~+ zrO_d;O}#?T<%&0L)Mboiia^l4Quk z-rsIHMSCBJjIFa9$NQ7hrU?fZcYd~1Gmp&B(C`y^Q^mQ^EpsaTG~aH^ar^#<{T^(E z+6tYfpLK2DzxO=t<;5(Ro=~E1Rgg?iPnRE#>N;S^+BR+DwJ_)Dn6z(49bie3tPgdI zJU?uHsDF)DF5WX#GcH{6Q$bp~GKPdtxonvDXalt4O4v^dY-0t`>+wg#*-|x zi=N}8e~aFJ&wDub1qB7=pqemxNdW=%5}(7lVBhZJxU;i%1YF@EB3g3$@9ivPedGiJL+Sqi0 z@(bE6Q=lV9EcAw`*w^^SvTF0G{ zluI{t1g z?er@R-afw4so3USs+K*Y6l&FdA@X4FQfmEa%aVbGrL1N!39q8#shR1y`Zk8yNh~6n zt+Q@*qc?(hL85IpGxGuWE$iXe)x^f)kuwpft#qb`fJ8$QhNzUX+u)X9{diZsr;K?_ zxX5?qs;w7UJCv?!1)m?z%K<{jcF$(@eVbN%kD7MV?Yz7*IZeYC^j(<~ZEly7b`U^E z{oM6O4}JG{`xSQ94G0ZKJS1 zju_#)0T(ewUp{-*rYWl2T~Mf3|DhY11XtoEWpyQSG>kO!V$7eMUXT@I78#!>OM+bY1$)r zI>yuP{v^A#2`l01@XiMebT%&-0pwYtlo$eYNWVq~+S=OUPQ~7x__mwjrk8g=nC7)B z_x<42flj9P+56dh$9Rf)w8E;8dwOw4Rif+OMUX*nIPO(RMZ1>EK$!@C3|Gb3ZMxH# zgNR(sjA_3i-$juZzvIBD@8iDymAd}tY1SRmH?K(bwHgsYsQ9dcTRKN}1E9_n1-@LB z$L%qJW7WlNi??U^u>bE={<1636Va}YD{1ve=eAUSlylJ=PE*{K%l86NNZ~uSA0M8V zYS#Qo&uaqvkw`sl#VH7HwN`6>J&2*W7ix_8IrR-BY9{NG#wsQCEhB0_NiO<3^%vS} zQAiTr5g2lJxxfEr+Luz`oxq{V#76so2MWLILE)M2&-ZjqcGV5)K5Z^#ahB~+(34H= zPoar-iWrOvb$cIC1s^Y_8aJZ_RQahuegs8*#;HaZ1fgbJm|)O~@AK!F`!KtG4$ljV zI$byrrkF~x%j6Ncku!N6tL+|TFxF+VPMm^`QTMbCihy+GQ9Ve#nrgmFPx&+MVOD($JqG6+{~=_v-TaTU%|srz7!RK<97G_ z;2gm34$y$ZbDGwYz{Svu#W9I1-`Gp)-N4v5`zE%-6#Z-OH?u-b{0-+RP-C3|F`E>AffJ#2?cj|a_qKgXCLZ6o-$5wjbpol&KHC^pKwmVe zwr|{i-Fd!t{<@|2Q-*+BF=0f8#~BYOHg9tPBsO-z)IoDyE1S&zHbLUT)c$aT2};KO z^>IUlFo;nB0I-#hg?Z7+`~4(Nkxagq>VKiQvMjsyzih*QabO1Qx99vRsb+eJ*wzE~ z4)7?XRFqP~Sfnn~!j2+*Uu+(G=f6ujYt`f9;QU%G)f=XKlfip+hS2KM;>y<`TA|mb zb3gF84IT9KbZ<9%+8vB;Rhi{ag8UuxU0<{I6(~04&HX`*T4wsWz`#S> zu(-5mY_r7(ln%Fn*S=@`8=uVMF}h zG)dPCpBlZ%ws4zg-hYOVxJ+OkS6yd)K>-1W)1dLB3-$1f*qW~@NoQzy7&vrGiS{-z zF`27qJ^Rp-ZdkA1;ZuIyA47gjgz{?8G=f`8; z#clR%$Ciib+^0aZ_DTy&(8=Yje#_b6NEQigZPgE!^F^ZBz9EPW)I-%mJyb+ppP&kb zDEtC7C>BR1k+~|uP;`IEkKZnLyTwcU5(=G&xxb93{>7T3dtwNoP``KAc1E!mEv z&4-pOvGU>UoZQ?l)?h|8%Q}Et{(ZN*S#}txs8;MEXM(T@XqAAE8y~Zt_OLCJkbwow zH@uS{*x~*vm!DcwtLCGpp6P#AQpk}!Z}AK{`j z^Sey0Bw5wxZcy61CU>1PB)eu}W)39SQL({~vws^szdAu>+e#Viq`G+l?5tMQB^fz6 z#%Tmd{2V7Vp8=h8$5X*uJ8Hhh+<1Scta9EOlegv*nm6CEpKO0qsP7VG`{BpkulGZ< zMso`3E;E{#kEp+FJD%>C#9Q1i+tUTIX(Ad8)ixj4G%7*4gxKOzrq25_=#J6;00OBS zYlXd>vWp5r3a)fNHA?uNz9|Z)a2a=fqs3jdr8 zs7jjxDy>y6w37iy5%B-tal0N0JLu$PWQ)o_QUs#IK-g8Q<254i@4KJ_T7d+^8V!LP zbgqBL{Cn%)(3|%Vqbj4tBSDix7IAq! z(spJcJ90Jd%6qaRpw(evCI2k8yCWIEVI+o{q>jECoFpqKV^ z?rTO2(P_H~UJV05O2*4c0WSE@J)4#78512+M6w&7S#gkXz~BMA;ppn(&CEaf_TiWrvlOAMz__t0Gfu7H35 zEF5Orx$PSFfnHmD@E7+vn+?iLn|h+TPsP36Fllu}AFFKZRKsW0j9S{pZgZ){I9h$p zN%$I>qRD=$!6Ui2uA!pT`hzMWctm#?0QBL9X#m2+oQB9MLY`r~uJG4eR(Xw{?1-y8 z%xyEUQ+q~zq{W#>48O96giA>WZX$Gy>9u=M!Uht_z<(+pR#H)WAtOguKbm#}E zrCl{EBk`n6qx)L8mwG0yo;B=J`I5%^i@KOrdDd%M>C6)@wNQ_da%sT9ex7%O{OC%U zK{{!+h$4&m0fN7v0Av}XZh%uUJ5yB8Fs1o-*8TE_E&PO%vd&IghZ^#b%02PGaJ%~q zIk!Kg3rB3G=Q*FS&{Pw4A`G&nxoN741!?*ujJoD1|4`00DMGY?OAXY?sj;i-#mgVtw7aPqeB$~quhuBe89j=frDSr&Th>m;@V~D_0ngS z@Gz~raTSsNyhx6f6g+W~d^MJ-Fs84o84#XJa#`YcAWlL^79KH968(NNgqkz=a~k{Y zcR$&lU6@u@cio!@v0gZaY-~WP_=}|Rx*K6uBcj_B+AiZtiWCYhcCZuX-n;Bf=Y<~# z>Lg~dq$1-MZ4B;+y6Fg`7>kD(vf|jIoHPSl^}u`rV5T>!eqcxrt#=4aD&_zSeJ^$e zThF*meepwTcPG)t%Ve?Z9(rUhbv~1C-eNHh^ z$sYO!OV2h}#*Zo`6}+CwkjHU!(|(x6x05{z@2>l1z(cq8Rw{<>yWbAr&n?P_k`1X* z2j)Gpyxg$NKG*1Q1WNW%@)wn)mJScFIucC&Y|SWSb>l`&b^oKdtLEigG*`A|&o7o4f zle+`PLzjCs+)EjP1V~||;xKbwK776TZb{nnQUGUicP?hq40dNerd-nCYD$A93;obM z6+co^n0%)aBhuas!{lm#>75OIrHva|`R5yBKwX6SDas)gi} zHZ|^ijZxCj$&tMN%KQ8_cZw}3AO64y!H^JcU^3vP5%uX1>~6N3*@r1a#~45SHJ8aV zM#m1;4P)w)Kd`%J%@%xgG2i{np#H4>PqITB>eg0*!C40RhM&bfTmMyz++j{riuYnm z^p@*tUIO|_H@9E{5H*`rq(XrS@Ep#lj9noZCsvNcq~%$S_?k+P7M^2J7yb9z@w+6F zIe$Qz2$l+{S9}g&c;L@?-I=C#T_!(3%*e*}b5j(TqG0%;$+)<16OLLocDqS_!ILg; zxHPe^6Gb8cU*?Zi$?HDVOlMPgky;z&R!Z5Dgq+e{T=!ch=f?m7e}hyVW&+46xMvh+ zA_Z{&8EW(qd2-;0-f%8EZ`Au9*$$4ulns0kkKC!b+>kzKi#LL&yEhF1 zB5m5KJQWd%Lv;sl^zd1nHkEKDa-lF{hUmOhz<(USypOu#A)Eh!)|m)m#^6(KIA8jm z!i&WOaR9Mg2;Mx3Y-^W`^EMIc4Z^gBW7b_J9Rel7$na?S2q0?s&*zip9JTs%zM0XMUJQ`^ zE*W<6!?FD?saV-y7%(dYL;+z6ouBG})Xw6sZ62#rQkBEzVhikdK)|Y9KOdG0=@&wr z`qBD?)H6s#`HVlh>(;KC`^`$+&!3OrSTF$+Y(q18u9yu5-`XrDGgMGkRO*b}ERs_i zenmg%ZFrzDC}PCtf~ycy_I}l#{{Y3NSlb-^9sidkAzIfjck1SC?GAwMvK6LZzp0JP~x!K=|@CB>ih&rLf z<8Nm(`{IuYdte^5?8MWaH+{0AZ2m4uMM+yb__?Y3t-r_&31t2AODv+7-Y0SxR-Ub_ zz~!~cVZ`Dz-)ke^$6YeP=ihkO4KPN_i~Cd*oTIWpoHGi5=?QDbBWds|>cFJ$mIa*y zsV6lz)yy@WMF*mx0imQ;e8a$D=FBs+5zI{>yFhhwZ-t0 zJDcvw*DOO*<=!1&f2%C*@GfQ4G4g5ng6O?D3ySRi{f*wVkX-d$f(KTZoFEye2}%wU zZ25y+m_fu9tuGxjyS&*mK6bCKn=Lv>&PH&uMXdm@S1TFQubiC#vsvK<3m`V?s&moT z$Qn#>sG;3l>W?C4o*D<{OBuRqm4cakm+~nE)CLj zH-C!WF|O~-l+(qxiG|M$fWbM*12k4-gVPd)o5p_WJHEKK0qVEr7ZUcBMMcYtCif8A zs>Y+z%;fVa)<~k%@V6Vpk3YkS9-4SCxOQw*crWvS?bPn4-@;KfJQ{B~In=+`@sN>0 z5;LtsmoUW>bUKXn!vI-w5z>g6uzVKhFz+?8&wnptNIs zdeL_xj7@Ev^MF}+%f-3Br~Sr#($>YMi3bi6Ua(IEs_tV3Hdq+Iy=$=Iv~T?r zQAiwCoGKR%q{m3e*u@kx1qIO?^e3cn87dk-xTumWFoDl#X=R)>Mx-Np!dZf143{hTmedBA$t|Im|7vlPhxTYKs#KZYfRP=Z@sAkD6 zBHyt(YhYDkOn)~?;6J6JgF*=)DrvcspruB(HWkBQ9)E# zKeKlZyeJj`_Vbfuik_bd5vAa9d84$xz{l%`rZ})_X_onZ4L~WXo4{;E07m~IGb%}B_0Pg=Q*IC_+sC{= zp)T){8xN@x0DZw}&j)hY6hiv=bYZT~4Oy#*)314!o_A9H9(#+PTqYk|>x7&+fnTIrrTS`rJz|TJQI`v#ObwS&q8vDh!0(3(#Qw z@Xq+?H7jN)HW?uc$l_C1Ua$|~j@KT8cF92Q&8Kep#r^w6qGrsQ?Pf^Kxpv*3H`1Pi zVPwp2Vb`0A_Ng$>1_S{~W&x^36HvbgWRm|k(_AujE&(Hn3JSlr6I0U^KFZ`I!tQ%< zD!oUB1n_rY%i#+eC{+L}d8Iren_}%pT1b&nP5!7(j06+LQ}qs3X$CJhV0w0~&n^Jd zg9_k{TFmGWB?vx@cJ+(3)}mv|Ih=s+Xyd$D?QhhbrYvlLIk#bTYCzZ0G0NR(wu7u{ z1|A2t0kdCrV1@}dDy1QTo4 z(a3&RLAq=t3>a{}6ci;vDV*ORXP5J_d6{r2Bhr6316SWQC~XhhrJTE#1ukoS;UyO$ z7rS@)l*e_>J&FZDbrPD@>g>wMX-F#bRukpj~y$b&N-!pGb3<#7Hs^!@MPlf75n z2^~TxI-02=-JyfR529YbKyn=^O^aG26KBCl^2>u(K zfmEd5yyQn;s{r{tAr08^p;phnlXty<)D^6iPDZ~-^$})?yH@s_4N5X7TgsS~U;=x* zf#(JqY(&KpdQ1PP1G$=w5#rUZc37Iw_haJjvQv4Jd+OcKG$qV@e&1jPCMfw3{>F>4 zPa+KCYjvOs)QUkZ7l5*lr>6!e79Rj}m0ui@GF<~{mDoRlpq^t{zrgK)xsbB2)Ew`% zX{esJe!a2y>{2(KM`=+2pe_bbD;rqHn~kF5Hfh!Q#nBX&oro(qP5U-;u7CjmSqugJJ?^^h-`5dP z_Mllg){=yvpT&OZpx<2rX1V!&!E?2lND+~ZE9g$2r7kLx?S*_o|82P54 zwz}JHeFfcUW|e5k@l;w?c+>tlJ+$af0${YhhMJVS9{y96X)z9x39u%A9f6TFN_dmF zkiP;Tnffl17uZ~I*D9TUlpNlw79Lajjpjn7uhrhHhms}x^b=a&ZBj4Re+UKwpC3T5 zu@cXqEXo96vsz`9k%AmJMn;y#ho||uMj?F4^~uov3GJmy1mYoxdQjb>4FH3X6Ni*9 zekTSrkhOuZ)+yURWd5$r0k&<`5CEeYG%C4%c+hCEY9`rS3<-!9*EU*&#ckgbzN6m2 zOk03`+ldC#TIAg_8ocqSt)#0I7#~RAdYy&(H4$f1?&T6M1a8ZP<8}$A$5 z0fBSUt!EIvy^NHf>0(U^ey9NU_keHG$3LyYa=-L<0`d&$aZ<{nHNe8TwOvLUmEeHz zdowj;r&u3m#1M$Gk$9?*WsEQFYP5ch`pT)L-0eJ1htOtg7xeVKSm%jCwqDE#u_Av0 z;z#_;zWpv#=A#eb3}`mj1HQ)n4im@>D8hridSyFlIg~VHKe$cvijx00w_z~Re*U&( zliO|gnm;_nXM5B)=>>`WbiH~5Ne!3{suDhBW`WnDg@Ce6Z3V+&z|<#?$Hb?DImwcMBz8)mpFNQLy|)#dV0@^*(?-HHv$ApKvAkDnsA5nP7(u zoG9ve@7T){iI9B7%JHP98eaQ5ZRY*()d(;HQGgj(R;59G2hJ9KKw;mE>o^&_E-@&o zVtlD$0Y_y#g^8C?*pw2u9r;7e^p>7&%)hSrT>@Q?br+v8g7FX^bu2~u8y3aSGmU`UX^Gvv>8@VWK)x`5Cpw!pcCy60Jf8`@o^E#? zRzF`;Qc=OUt$5Jo%V+%gOhqIDu)dEeb{;4qcmuqLn8(Y8!uZn}RB4W`*8>r7?@NjXq2x+`Idn64Ss+8i2NTOpm6d!94Cr+U@1<(b8_ zf_!l>k8PgrfRO32L&)J6PknO_l(D|1riL3w^Yyj#aJTbTp#IfE0Y<6S0(@5qatcsT zqpURd{fdOZ^Pe=gY}$w+BPM%@MDbYnPApQ#Yy5Aco}ZX;V>3Lgiif7rJL>X_<$XL| zAkzX3{;rx#NSC1xVkhtEJK6+6&&cWDJVS?_k(2&1E- zp@C1*aSYpkLtTE-%x+XxGlIjO4@j)>jDUcvOvfxbS{rny&SXOhxm z<$q!%#>SZJPQ1+Xvt!&zW*V^XuBE?()m(XQ>D8E_L;DR8$-OZv+V2>n1&-e60F?Y$ zya!q_2#I-Zdl(AR1Av0H(PXYfFFtBMhLkN!K(K; zy6457zwqa~LVYBlOOP+dgM9%(4_~sxAu$Q2@1+6Ap!Cknl3{`wJTK|_|5Yr?ttt}g zVM8!Ps-LBJH5VSLD4@I{Hp##MLr7{An~3uzGWv9Hg)hwJHcbr`(QTpGg~^2YX)Hs) z`1j8r#mntW&DQgrIv(2z?#FG{?DlJ-E|6NwDOK}TA+M`!fZu39i#7VnDA|pnAP+VuZt94!LSGZMd@QULLT-!qqLOSrN#$nd)yr5UA5+p8%|>DuQcpF=_mJH1KQ#~ z-gAO1;zvrNf__ZTrKm(_+z*kA8wtq>h7U1^eCK++qa=tO6lM;AoS>s9W-miS1}|K8 zSe%I%h?5hq;nV7UDzjpR{N&AQl$bdXZSvl8s~aM!hJPTCN)j z8NQ}*WKEGIeBZ!+GJ_3FO!YWx<+JL%H!?~0DSE9Q^qwv={me^8+Z+5qT^L%zyDOAR z41l_kr)vRdfI)W7W$Nvg?)A<*X0d5q5v%})By$-5;(e};@k?KmVI6zX{Q$qFD5c+a zKEu0sI;qeiNb9zgZL;L+;0uiK2TIJMUI0e0I3`-VWyt38ANo{shnt;a_7cmGw$I)*8v5FA;PqKA9#GJ0>UKDT>XZLOIsxx2FUNQO~jH*quCuR*{%uJfkJ z!k_s)FaK;W%o>00Jci4e^n_@a(%_?SnXc;YR;H|(Z~Ou;>c1NR@fxqArsrbSs20RAq`qmlJf{bIaM`SNwE8;K-s+I*sKqNDoCe$U zW+V*^zcrCHwQJp#Wp%FlyPs?lJex5sCi}-!xxzFnX}`BZ_l)cDDnlxK@OZAH*N+?< zF9jwQ+W?f^#AqDMkWK1OYE=E|Hdc=251T88@tk>w+2*00TXu6_aBU#_lBW+j+HWj< zU-kBU(H5lA1+7pza~p_%?kDk}v|kl!=$E?l-*m$zn65U%r2D~G( z0!>z}L`nfm^}&gK1g~Utz-B#EYrpQ;`%3nsPi_1FbGyHW(-J6DMz>akRXuiS9uO=( zCxhQNZnpxQ*ut!neA1~rz?rB4{HM??>#!sk9kp~>FR-8&Gs5ef5z7dr%k=qAurK-Q z`v*=)J^@krjNkBGz0i;3t4`0^xhIuAxBirVSCmF?PBG}xyrJ3pq*U1mAU#{;_h9yM ztohF6khca_+w93X{Ds4w z3J{Yw{bpDOK=I7W$uZ`=8V4S|(@3c3eXk3y{n7>eP~o+x6m`t7vI?R!TE{q;E;}Th zA$8_gq8g#b*3XIG2aTZ}FBb2C*dp;$gvSp*fA|kfpH6<7YoBbMJ$^8BKZcXVJ*9FvXU?K>Z7L6d^(&yiKB?8uxh$L*G>vO>i2n*p(O$UCTZdN4;D)t;<^RFde`SXhajXP=?+CyVp`U=)jVSKlsgrl;M?0_%dr0_#H4JY?nuLfFcV z%9rClwa9?uDSj6vlnYxfoY!JOW-_Bdsrj-3ehW#tB7E&+vE!=9IEFAeun60eI@tK$ z%VrQ3VDuxj>ZPAiEsn#U?x)NB=V{|w$bCQUu&{bA_9Z!OFzR7cw7=TN}{?-oN*S`+X0n?K^yYb}MEXoacG z@*}}Ji>heYED^aXl)cy|`m#8F93jtub(qAE*(3oYQZX0v95$c5{eW?{=yO}v@o<#| z?G*;2a4YzMMbFb4@&BU5G!2FQ^BCTsgghh98pjb{A}bBhXh zS)mGE<<0I#Pd(ZdNB^?R z0zKcmb8TUM`?^lMWdBhik}T8AJ^{k?wA*328RZdO^~fb4zLELw37|zd z6QLlW)m;^*2AJC@R2jYV4yde`Mz|qejZ{d9md(lN)W$l#nVQ*`_ zsPSqfGZZgQ#=(*p%vb;1vU&)SGU73BLi=+>uqeV8ZG7J#hkn8glo7CiG6F$PqNXIE z-b`RV)xF4yhMeMs;H^f*$+wLd!PoL9jOa%PZIBhg{<$Y&w}is^G&aDnu##l>8}Lnt z{QaWW3ACi1rNl`I7(WG=d840|=4N(1%^AJL8i2A!Gb!u`pD4y)g+70KgR3V?tL9qi~BJWjqk6`YM980xEP?!xdi0*7yi1{U&nCu$1L3{ zKpkr_las@cz~*}55SuiU?L2Fph!AJ<2u6~41v?hm=+2j0?f?`sg1F)&y#s7wWsV_nno?l(c?a& zeQB%K>K&!d`vgH=tban#42Pgd00jTQ26(#XK0#POAE^D&HGAE)F3@ASex{1={uc2|zdA4R&+>QOrRLV&gVY0R85MW_Y>OdLseIrS)=vskqB|^3?#}(sp4EL*r9v zCKUzJq%z0g*`?tHOY;Y4bA?L}aDy(Fp-y*HAOkm2x^ z6rOf`PKNhn-f#E-0ZBjh1q6ULF(+cCGKYATz#`SH80ss!op1SfGk2~uADR2z?EQ^( zxZupa|Jo5m!_3L#VK!EGNaatH@>;JQ4N{lnCdpXLtdRN+I9e+1=`pE2aVsaBGmQ=C zskyB;DH{?8O6mpN?&~V^6d)C3WW>zxv1)!lEBd+{_4l^o2`lNo5KV}(Q5b)#Y2BXIu#L511lt1l2^ALv4w{l2o^W|Ro@ z#Ef6c_HUwiQs*}m^!?GK#yR@Yo=?><=UzPVfoiil>fWCq!e8Sc&*EO=9t@J{nH+w8LJX3rkPO<3{KQs^{50C>7mwSV#s+(%`;Yrf z(OsBQaeAtY_;co-Xsm!42^hIKKG6G7V2B7Yx+HbLg>Omc&RS?mTl<~AYG6_a{F0{C zppLKdQK)HSK{Be^^JuW&#!HUK~O2ot`y4t)Lk zdi5CGBrWw2OgBt6;%T&du@0@)hO$)4>p|6y+PW$ocZ>Z5w!ZVu67m*jdw)|s@p1St z$7m6V`?E=z^za{+4J7#LeD=y5((W(B>c`<2;WlW|q9+%51nG~hlB7aC(W2YWB8W$5 zT`Z2}bbSMs{O9&PezFyw@IL$0R3>ObjjLEBhMP z(TLuUq1qP_Ho35*N!@?G#J0igd@4sc%WQH3q3k<%TK@1|2fXi+{_H)%ppy-PRB6?(w5fw?dqs3>cG24%{CeH#s!P#? zVCaG4rpXBUG{-^qWxdKnDDWyBPXS8hlym$$qw^m^KSvU*(SJ`T>)kqy5eg?I;k@YL z-HXNAcgd@sYX-qxO{mR-oAskl7E_wfAneYi8z}(!kXha;v)|I}x06&i40@Rxat_ zJ~|7I+YLf$1;C5n!Gn*o-;L8(jn_NuwYPWCbLfD?(}EK*up)AhUDI;6zvE$H&tQ@z zG{4G@ghtlG5BF&5|I?-tc*tO;9r*O5;S)-WNb>y_qJsC<7@`N?(&>$>JudgvYeU&^ zb20K}-`wKwzBzwAvpc*SE0;Ww4~*rCAYR4a(cgpCF6{0?f7!Z{G+Hu0F2Lwt`o~#Q zm0LGR@L&f09gS4_IvfsumWjJ1E*tK=LS{cBk!(@P|3`Pc_cIlabO!oP%58oNxG9bt|{xOjx&d%XO>496`Dxs2d7GD_L3Qow6uN*KuS`dz)^ z?)c;J;LarLVdit`J2OC~$0mE=rjI+6m+;GA{t_wkR>};JLyJlwyp!v{no(1}VPi3n ze6Xg_=b;sHRez)^+QBXxvlUVjuHLy8909@&INy8lrk6`;*6@cqhr_sje6}9G!P7q- z;nS-l`%}^0wt^86E>q&2CQXelSjoCn@~Kk7Lxz*(m=XdxdlmFGz@E+h%gTbwVpBJV zG7sJ_0B?cX&I0e6TXALK2*io@WFlO(?FoTX-?iyoFqqNMR(MfI=BCf{B_9V1KRq|& z7;1s_ay+l}7y=&xFhy20{vHCF5Z!Cs=>1y-uOo*00M|a;qhAE>i%I~#%(b@Sk23<}9^v6i(%#zd3%V;q zpD+sJ>!qycl$B z=-wf?=?_08UwyI76;8p4=5>+xBpxlpVy6rgQ8!w+lAWyGg+%#-^b*L3H{A~dpBe~ydmcQ(Y?DhQ;$6l7&y&Z8nKy=07Z4N(g zdjOUmJiADy83)2er98Y#y2sG|=lpxru|hc^ z3~qqA5^ew(@W@46gJTFKh)?u2cAmRtV6BpU;VYEiV4$gWN&i^Rk}AH4NdeaO#> zYfNj^Fo!`4?rz|z9r*OO?ijesLlFEmqJuHqReSamkQ@k9{9nKC4TH!CK36w32{*qL zS~&|FSv|}Z`}|T9{_8~WIy;ic-?9>N=MK@zbm`m zu5Y=uIJ)m$jkIq9%lO~tB4WVJ3U=rK*B;H>+$M)o>+st)M`+4HrcsmHkc zb5JE2PBsw-LdVwMJyUf3tAB$EEEvEN3=ZE+xZE5i8T(AH>Fj!XOu^`b8F~y&BsrMh z`liw3B|2nbQSDP@6qp1Zu3DCJ{A*zP0f+&C%sDckV^2ij?YB|oy?H|W`*KboQLRx( z7;5@@3aiO>>_|){GvXh_CLI9oM?(24Da>m9azgHPUfg8eNT++ytdCxj<%g}EG_FC- zL*DRHunCbYjWiKaQqtw*1f#&c1V;FsX5I-Gj|4OFKq9gp@ibv9 zoyJM{og3nh+xGLSIw;pw;Ava>Z?r!*a55Fo-} ze4_kn8<=00+8^mE(Jp53VFkbTqWPz6cL|tt0T`U=xWL*m0@F=ddr!RxC`IU=f&qyu zh9m|V?IsjZ1fUjVL+xG1x1ZGs67@+&H0S^fICTS81TA(!#6(qcvUr~tMQ~(eqXU-H zeHHr$_|+S_!%?NBmBE}TDj~)PST&#ggw>wY&9}PokmYb8*CbF@jvG37`W|+}n<|56 zCb{L>xCe5!CiugIya*V^;(Y40y}oco8x4WuSZ(Sjpp<*5xjgp9+I+IF9q_x5@~}8N zlv@_YGCbEMP!AHr;|TWpj=w+LqJ;#Z&_|*Vo6{Kq$mjW+*{QLO*ZtQ{BsT8eTzJ=8 zp&s*yMh+n)LPUGqjE&^o8gA}MPf?1tGLZe$V5z1C$I{#IvT{^sG zg!mX4z^5BRxD-q1oqK#nBpc$?hDl;`b8}xmpW9p?=pJ{ze)x7`;3TrFnzj{J`VB3` z1W9!H2iF%LWh{DAq6d5{WmUPj1}0(JqXM1C>fiqHN~|J%9~Cj zXuq<5k`3!|Zu98J;z$oV>=qaF-YAGEoljI9fUR++@1!FB+*o?sykAbXHBS3TbgTf2 zB`A}>^Jl1<-+<>5EydJR1n4tf4jcgBRDloO`fOpq0!(!s(jP%5;fh@?VFz))0d3m< z5XJHK;qCV9U&QyV*5!S*Di3$w{xR&hk7)YiZ_I<2zXbau==(laUsnIW+(p>_ zDEh%?It0*H#)>l%!@NFbc0PH+tgde->21T|DAG!>wk4Hwn-QbeDf0P~x-jmz2bco; zuN&D1+$p%$FpRziYFoko;_=_aOV?>ON@+j+A>bgTq1u~CBvtYv1{g$RpZ@lc)_&tJ z*fLx>|JYp~Sq`*mzA?R;aL#KE(LvNuwWb5kGH_Be!mC)NCwGxF%jxgE8^9*v;14NW z84QVxrUn%DMQWsWl*>XCe~ZXNA?AA~Fft3irKo-@3sC*7@$ve^EM80PWEI9BvNshf zPBrvSxaR|6XN6cm$l!U|+w~V_o6*0cyI=QFE1Y66FjjRE0=od1#OX$0J zBE+I%bg3N}g<-mxDfi(c^9_NjFhD7Au6qa-JQlmtJ8z!)T&~2age(3I`r5>^kjx(c zMB5}Oxc0_aU_EWo+jZwsVwmaVZ^tnao;?~k4DkT$>I5JfdoTjv-C+x(9bRmwb!7e< zk|y9mrwMNQxdCukb2*jx38<_PR(1GOuYSNrmnlpRKbV1YWrY|G%m}(Q4I;m^6JrvP z#vWK3#l&Os-J|>_vfKwdWrZig78RT$AYx}Ci-@1-wVnLI1{1B+Y!%j-AI7)n8NdAu z+Budo`ZxZ{Zkqjp2m4F-Y_+L=Is*tVNP+V=n$ey6DxBSwX#WKJ3F#V011hi1I^aQ{ zabyN@9walzzlWI`LmFz+IRF27^0HH)_2S)Aa=&&|O01gj1K9c2-6`8VVD&~?7N{b{Cj<&-U<%?r5px+*!=e9Y= z0T>8D>HcuhVN11{;72Q=Q;$-CvErZ;@#m>)?2qX`Ny0hH;RX9A9Q{85?y z%>M@VsX^R-uYLFo0;R;OQ*RbPy*=*++B3D9AHxOO@ycbOq0K81?hT|s;s%y{9XN31!- z1PgbH=Hd0+fCWdnW|F;Po2u6*OY|6D+V!%AGVg<*8NqXg{}y9U88~)#U#8vdvsXu= zSL9*2fI)`7fiHHw(#$XEBx?uMZ2eL)R`>x(h0k}(p?^bOvwN`!S2sc_@D`M% z>F{2}tdxW*{&_ATrOo#2!j8MSMnt&kH;(x##N+4pV{I(j$?MEvD-VRq&hNGvYaBGt zM8KU7a3zJ$m7-8M{!NEt0_+d&e%#;zcAK&v>|9kywrT`h{CgZO}q~vsyjVL3r2GCOP-hy)nwgG;)L#_}w@X2q2wAUjAl2 z1va_&1fGxGFs!VEnMyG!DucP!@jS3|?!#ojjj zwCmx&#w!8L^O2W3p-Vs8ly?fKj?APw;JztRTh(a9RG;T`Jh^%ki_pHLC@69(6Isow)iQo(Z6n*CHkgKaui5f zG#LMAo#9<8xtcQZbz>L){mtWNQomJ&Zo%2th8x$3Y;eQJ?|jzII)k{AkBlf9gq$Cl zV~JSEehG52jWtJr7zJJ}*BqD>QKg6#ym`|IAtvwD6z(?m@Gro@^up8RQ9 zt2Iy?iGFc9pe&i=_=%7$|43udfM` zfe6YZP*RSa-whuO@eWUTv$iGrlyo&AJsTNE_-N-e zvD*TKVcZgUYpN2-6#jSkj^SZPVj2VhcjI;zl7D{w=f8G9^ex%#()*uZ1OGf)4>ZqM zkNwF%HK>37E3gH8eCPEc-hV&s&I;h_^yJ`||2`@7S}6^;758uI=7D*tKgF={VQ_GyAqZ zk7r?IQsRosiEr~7?IZih@{CC)xyNwuZtUv9u@?J1tHP3~%7sV5=VKSO@wW>##vN`P zBm29mZvC9GBCS=^3nmvvwZA@~@7oGZ_cHdLOOLx3EnRv7mzVT!ku?QNxo)YLUL+A{nC_@LuV_?r$Qb0qTV7{xpb`*3wkFAp)e0@7 zVqlQJ%W{;t;_5%XoDCZ}Imqs~ZV3}LlJLA{x=d-c~PZ%KD+ zhq}6sz5A`FTyDBNntN;7&u3n5x{@3iUookq#|~*f*e>IFnLX*5-r2G(+#q~hFtxkh zNxMz{zUW&H^@G}*u#cbMf^L`=5Mo@j?oob? zp&}}*`_=t2(&HES<;}S5SGQ{9^tx4~c{Q$tTjl7R2b)^Iu#-i`qKWw1^cL#QN|EUf z&yAlo3+eRzay|7`-UeRRU8n0}Os18muLf7n4fhI6$TK?Xlv$OQ5=?GScQ?Vq6ip{h zrTgJ^nBMjazO?(bwsW(?!^Ax{a+FQ_d#P$!H;74;UylXkZNWIa`KiZ>8m=(A87)uR{kQ?tOS6=c_J&L_r-4eJn zyJBLty9oC8UKn?}^mZEex?AyF>jci}Wcu`cwpJi=g(3C2Whk3Es*>xP{x;a^PHkz= zDBn3IwaPZ9I^OIh^-P;#tsC>_DL zM`abqM1JOd;67PGpJ(jvTHF_niPL<44UBB_H847Gl((lU!>9v*`xO8v)m4rsck@x8 zt18Jyu88#=uPA$(xcl`9oy-$*Bm@8BC0@2b=4yU5=yV;Q#lo;f@_?J#R?SNgj z`i02wOkZ7z_eWS6wxy%45hdOtiW$X+U=ZwUe>a3v=-><~(X8%}N3QLhPPn6`AWT(E zk({U?ia%j7N(m+qp9?xxWd8LGsW)?EFT@gw?)p!;@?t8acue436V5+e zwngjofhw(cIqfIt<)z4nnJJwqGJl5P2>9?(}yd%FN;QMvoZj zgR|6o1dYOFb2f-{wozFA`^UaV7UY*#&XavCKbV;q`#)5@Km5Ld>foB%fS4viNAw-C zlivGqRslj|<_u9rKqaHdkb_jxRQnyE44J$S(8^c;wYwVlrqU^@>`jAvA2h_~Om0QUGS71+=v~QGC)+NWP~s)dkU0=b$mp|WAStxf+?2$1_`R!_pDaG%R7K5I zqFH~hk;xa`2#NVR%-kDP{H8!bI>k9%u$-Ci<7#_yeh)3JaVf4%WD?XPWsHK1Mgl^k zjsxD^PAXsq2Uy>#l&`f zamT>tQVyuubj0sgUldepdXjn=AUk$2p_(K;sh4oyhM+X1fLEVB-ms@JLx$9Zxz_JD zT}kD=HlFvEVdyg=EA{7l7s&I466)w@%gyB$c{G4SkSAlA;wA_!%|v1(Fql^B6c3jX zCk@)sCdZmUh>MAm)Q@`P*bqIQh=7t70T$<$iE4N+!HSZUthdblgUIyKay=EPkXP)b zGV1IjtygU+HfTP7D%hR6ew0)B3e?jFWlZsZjNvJ@f}+chS8qSDNMJiUJ1!4y{m6W= zESB`7;;D*>4CI^|28H=?7Webf#=G*6j}!D!35uGgVxS%@5j83{YNTN3UxYSUUg?qu znuK4_Ur5NirFa^!j+I6qux?Kt-Dh6tcqYv25Bh%fnCSoEhp6+{pK0`B)98H1U|ZI~yd| z-k4q+cTAhEndppn_+@3L|D>H&h0rX&00;L0HLW`_Kp5qtQp|m{jLS?;M%j15nKIga zdJn6dL7`~~X%zB22~ayJv&o`@v^pwp$^G2u(Mg87nJ>H$FMKCbikd6+B2CE$J920{$$q%zCFGy->a0mMKb<#n z`B+c0%mNp!Ov>u}fF~#)zOhf5-b~+mN{!5xy~PpX;(8o&`}1}(#GGSZJFcVs$r~+K z&XYy9CW@e!6roSzpPI7ySOzrGYr90X5zF0pCmL%9(BdpMYu(hT|kSnHW+r*%KX0OwNmJLRh`I6 z77twpsVVEzy$r##%hos5)Raj6ohJ}?f*5bIbGmBJ5tkwS=Fzqx!hNnoP)T|?~F0_Ertj`K4uYyr0o z)DHd3P>c=*N!y=Kp$cv=#$&NDv1mn&B2Q;AbQlRb}r2*E$ZR+Mjd^sp;0n9HdV2^8sjCcS$aA0*Dnb9(l_-+HqPT$Iwt)D z3IWg|l>n50=2f{}R#ggKYI5$6Q>GqnI_tOmM%dAFNTII;#iLqPFGwVo4^GgRrx*H< zE@L>Get+3OCrpsmPll2G)Z;N;d=)#`k^g$p!(}PnvuPtgMb{(MFCkzL1!%@Cs|129 zjk<=*qrqu~n0=;CKUl>JlHX2f`P(%Ua~k&5D|iIo+FBP)c*(6EJF-T%Oh;o=;1q@1ZuF4*%?liSnE6odk zWGxw6L2fxGlCB`Tt-d9si;4Y`EPBZ^ZSte4xzT^0foyp*B%&aHv`mygVfq6@6%FDl zFl4i~v{5Qz9R2onhQ&WzC^EEoHdMyGlSu6A#16jF6Juw3&-RbzDxQwv32A;4oFS=9 z5peKmW_qZJ^6@>1ZvV2FUscGT{{Zv@yvGj$NDBA1DmXqrf?I^_OXRFy_w^ z-pn)+J{yoVhX>6KI-mpdM+Im26xFgj4yLIs&nL6SJ3Hvf;@Mx#SK9`{h+rZ7DYVPG zF>*(+QO_%sN<`&+6gryt4}L`B&w>Pa3lKbZ!Hryw!3n7$Hi8y7Ez*fV1>OPQu=t2( znGVwYu+ZDh;8!Q#{8_XuOCga4vc3L@3cibB8J71OAF2+coXU%MhKGz~H0oLqS}KC8 zk>mj?)%eP=Eee(HiH&33KoHOvwM^t7`!ry$S*L=H4k-2s`{h-BWO0c;w3=Ga3B1rE z%_2e_4jyd^ih0Sr-zTcruUbiM@LYUG@{}$vmQIW}$sk)$svK!_XAxTHg%c}p{n$zr5{f`zzz%iI)@FCp-Yg#>H^l@Bx3j!3 z&ZXb;M~nEAwL$r=6^7>}U%#J+J6c%NoMu6lrGa_bO36rnSCp6>X2OdBaaiIR?TPW2T*=&24VPcx0< zly}BY$1`o+y$6II2CwERFDNjuo%sUak3D8}2R9cN8kJxRsk`Q}5?u?tv5QrM&^VI; z5AXqrC0h}qgQafFoxE{%ou6!M==tdTgB|m+G#htYV)eUwNy>g(`tbQVwN|u%d)~%C^KX1)~c|B(?JWT5I56DT~-7T~HMF@wkMN#6wFJ zmTh$Nbf3-nbLg3JN(pI|nJ_G71H+}0%9e$o(*ncI6A(-@MlN4)BdIIIu`-c-db9Ip z$2DiLwXr(t`gEKI+VRs@Tag3X`MV73*|#Kxp`vtrT6H$pyeEDl??(JewB=rnj}d6e zkg`&J9myvfXT|1V1N-fZL1X2n2y6?|TmTd<()8JueT^7S7wsgLSJT|N^*jUnP;H;t z^4=r$H=5c%A7Ws0KmsqjeY_3Uq{dZocPh6UF;M$Q{klrD!frL9G0A7mFn~|^jb;6M z8v&v?`3vt4S90?6Ab21u{>oH3Yez<7rVSCd7Y}_}D`@g_9`+MD(YI_{D=DDL_qa^7%dYp#%E0F= z->9VShnsIEJnGg3j4!z1sAGHN4C36Yb%-LLxm8bGs)6h;Z8UVVBfKg;AP__kGCVc^ zY64M9qE#m^BQrB4TOyPj$=tRZ)Qt%3EvW?52=)=CvB;d}9ykEO{WxL!~)U!9Z|jEqjv= z9R~UlaFW;S#wxvh?a=?Rz=;Zg`5z^LsdpUPbDuuFyeXij$wg*(Iv3$|5V~W0v~x>< z{TL-JL&DP}!o4B38$g%6ex0$Et3^tOm0js(buFZnzE-#(8qGF8?o;KS7Fn|^z#eJH zX@tF(^R@A&f!wq$yu{w5MJ&b&7fp%^Tbv02Nqw7GUfKA@3XEo15n@`Q)_u*arEX*x zh#&LzKdAGf;+ny55}V(fr$5HS=6D*|o&L~}jYslDX0p-A7Ghr0_BZu!yqPSqJkxK1 ze4FA#tv#^m1TRtpR$@-jJS#= zw*N@0mulO90Pd{fJCYKv*-I{&pJ2;>|JuJ$*AV5`!oHX_(H%rKMvGlCzsuzyNfP^# zVEf`v>mxa{(O$`C(x5u>5I77S*ox8yt-7{mEQW63^5EI_Y($zmD30I+FI$~`-^FF6 z4?Ll9mj6bx|Dgtr(sij*yl~UUgoM|0~*RdfjK0{vwJ}N2ssKp5Zh`{j)11&I&33pu*!bcTP=l`HlZ7wobg539V zMuzYBL*;BTY*w_tepEU;$&V?d5uUMO;4S`sEj)U_1TQy0&`r6(mj~Fcouz+{GdU}^k@{12FJ(vFWc!f!?FV`YW(20_};{$3?UMLJ~Dl^_{ z)8_6pHRvhLi#Dq)H8{Bc_efYAOD^UxV+kQXBd%%jV>- zFI!YI6CWf7{6HzO^5XCOEaTObeSr&XwLj`yFg9T4_^x-B2VAWCSaEQ8OQ)LA(sER+ zJa3h5MV`jY0)!A{5|Ipa?eNp?_oi{nt4>-Px+Z_<#1*oyyzHe_Y!=CB;*+08yRbXT z;(Q7=IaGFnP$JG@33#)R!%K#)uRBtnkkZrUt+HW+ge8R?Qy>ZhH8$Ja%AEJoSrXB* zyS*SEn#@LRX~Nql!9^r@VmrwY`OLE<3DsV0{W0P+RzDh*(>fzr;7SM&6*^>BKX$ZP zjNPEM9J$1+jLoCVMYtgxdNj5b)bOLDQHfpWg8Wlj8(?7jBw;4)r12(P>SBwIEi(rv zA>b)8sOed()e*1gE#WAcZdpD`w53K7h=FB)BvDQclkInVhCkcNw%^`S@f*Wt{qr9I zj{hwXWXUYQv)|7+EBz9Dy%;;>o!}v z5da$fsB+`hx_-8*0858zi$_zcRZf3T=(xJ39sUq$nHca1`KFc`TsH}JYp3UT% zr$Kpl4v}ewlhS!_f_IGlfj%v*#DrL*(!{p43UZSCz&ui! zruzd-NP;Er6=M$yjmhSLB-}k1b&eY`F*kq3v(6hh8*3z zOD)VjT}uCnsex;ZWUHAtbF0ddJnSVjRIHusAyrnZGEpFJe5jd-d;qjmZ8>a@O1xb} z;~tIoD77dNRKkN*=*uwTt%?3gQTnbz3$4~HTkKmD6-TC-XuGc)`z~M2ZngODzEQ(a zfbl|QMOqPk9=^Q1X^!eDPOUT_HYk@_*7r>C)$KKR`o_376W;-^1z@p!s?$w#*hIrY zm`eMnn1GH$Ko3vClPvNm>GX#!U5mhV zn)MV`$Ot7dd9;ZR$-oOizh2tC(sK}sY-WBBB2OS4T05u93*6UzJe>7vJlW7_)#aj8 zK4}l|WpEKuGjiQqUHNZ|DAY9#11~yM`}ByxS5w-Z8G@A3EQd&|Cr(xu+YAB|?%Iad zs^YCCkF3_PvOOR1uuZk{Hq#g8IW5cx8eegP%;6B;$FYK9&bv6WT?J9Mxg|8{vOHw& zm#X4OhWCdd=D?~rBr?KT^i;1`D_Q+h0K7g_dP@BbEMKKd>m zA*LGdBi3CFKw;$x+)WERDn~Id{cRXbX$r;S9^0SGYzm zT&3-_-~LJ+`sm#8C1lW+3PnE-2A}~YFnl*UPdRU9I)tex4pFYhh_tSGYf=~F<+#m+ zQ{se;#>A?&uL_{yo;Sq65bi^R??0gh2IX_LF%uVb{QQs?ra@dl7Z(xw{?8hGvt*Eq z_7>xdS?}EyNBio{Y)ofeVk`!N=24lknd9*TjGc`X2g69C5}w9rWD7SX;)O>=&dDeN zj9*^(Qu`|n|2WR>uZ>z+H$ZAMftOu_7-Ot%Mt?{};dncrqxX;t#!S<@%eaSk_O=(k zp)o^+9EF`6)_x5sz2!F{^Eg!uFvJ3xnrmKPIA+DAIQS;+`}=Pw9(M=`|9t)SZ)2nn zxH9K{Pkz@{f27@{z=lF)v|;ax$+GkBrGv6pI0}jv)9Z?}e{wH;vW7)lOC+5PdA+vM zkf2dW`rxLopJJTyXL`3V^ylKwtnIA*%AL}x+W}QCz3De+Zx7llr;T;8@NkIDA3!Fb zUlftJf-3+mv-*c)CzxP%x`w6rv?TK?1FfM|k((bA%>- zIUu&0=Ex0B;c3PYJ+mg8Hn~{}w3`~@s%f^t!3?K?=!px)_?gtkkljg>?@WF&rA7Rzze9M2>)->P&S*qJn9$=X7_b zif_&_>tC6KRf-n~dpPKc+v`wZAR`3gA{$*yxM$SZFpGLCIQ^oJp8;=%OdiNO7q$JP zabx_Vj$iPL|74?}@!i}lW0h*fVRGOz3JOdK2m<%vJ6LUf$mue%n{8L*-mr&soSpRM zeG$(3izqz=_&THLfy6mklH#m0QWwVV6lZ%GRat)O$jCGAU||*lqvHpxAB}t8e8`iS zj-r%oPe_}40h#@bhZ4Z2y;Xs%hL=v!_owslhXvD5(gOTH_1m-KSWTx^XtRij`I-Gg zAIUeJidxyXWWqe1Q02kX)T3eF(1f=Gr#Hsh)*mL($_}B67qP|=%_r2pT9M4Kuq(%o zgTk0v%BE*{Gb6Y1_&0wjb4CX=c7-*3R8n1t+u8Uv`SJ|2S4VlvXecZMYg#H;O;Tjj=wdO@rd3GMgf#Bxf$ ztpvquX%>PV=FyUJ5{Nv)x2`}^+2ykB-R*+|mfWB( z3)*Vs>dxua4f@Lan=%!nQH|zB)4!N@3r_o1&+#FQs<3OK^2ld={H>+p<1BNqvYbW%(_a2va@+ zc3bt`#rG<7$wQ7R2b&ca0b++WW2CN(5tk{KxasJLE0|~c(UT`WLV%5lUV%&bp(h6oHt0hN!Iy(DCrU`0 z^NRX#(W}g6=sL!>)P8vl9ixJ+66s8PoL- zA)mP^faXD=<-?Qs2aD3|{%!9T@9Z8%9K6x6uDLrrv+#B9Pw^2dLtB7v^${d43gz}d z%}6B`VstN?`0QvDv z<3FZ7WzS69ytik+v{&KIMd?NE9Vz}0P6EPy1_|^JkGMVXYmH>X!$I+X^CDAz3zaqy zJTY9t{D)(WdcPEBv2}&p@#y{^_wQdFe-Voc;yqbK#*_~PtK%Z1yS8Zi+8utsXD!gX zZ;ih6nzQowP;<91$Inq1VS@xtP65;V3VdDZmuY6(Q^xBbwie;!eBe~_V_?(VQ{ziO z9uI}Rw4{$5vYC5b^De)deiqcp6U9crD+yysq{qHJi}B%c@(H9)@8t6C8p!H-)ssQ_ z3gQCf{w@ZgiiYcM5bxKMf-IY!Tu<|;FcGjcM8a%!%J_7&=?{;G-%r;(k*@pqTEbdc zvrSb<2#pCkREQk!r)p(;dl3Bn&hBrJq9Xcl`Cn;2XL&f2j6N&Aw5(fT3zny$I{XqA zwXMP#c-Xe&{*l{nQtSxV?*SqOra5#?^sX5LUt?gOu-@_g?$D-i?t{hOa%oee+Hw#~ zqKTEqhV*ogMNc<;Gdi53sMDtJH#lKF&Glb&pkv@P)gxg_7GLgnb65*Bf`v!Z`@Pgl zpS%MrFN;HzruTENO*56EboZP$&a-@c-w2&Fp=)cE?}wowYlj9Tqe!?6IA(}B;i+V{ zMy8a{KU30kC_>BhQa9J&+?z4@T^_CV)J#Lrgqc3>__aX{HG)Z` zQiTFSq-XLdK_dPblIi!nfw~k87X{nLHu?T7o+g&XtK@T*e$U-{d_$$*)E6hli3L0& zol19oWi6NUf7pBPsHV4QT{MJ95$OWbK@kK5M5LD>y`uu6(gZ|`^cFxM0i;P6Y0`^; zAidX6MMQe9q4!<`1oD2k_de&`JMMe$zBleaXN>(9jFDvZHP@Qo{N}f$%bQ>j>rh-5 z`9KyvWWg$?h(fuE3&lVDO{NVedF)A1NZ4hHK^>FX=r<0GSC|JFJOa8*p>6uX0ZOc* z9?0L({8}ZIpfE`kvS8t|HKyi#Yw~FvNp})QCr{!CEamY|zE(N;6feJUIf_j(+;`Zt1O*tK2-@i0=Vwk0py%cs~ zWE`*KWE`!MUn~GQueR5 z2CJV(-yUx5GjBh!=JbzWWEEc@R3;y;wVfVb(v(>)`<5D$)QDEBCTX4bpXM+sRJ7Y! zAG0DvUel}u;kxc^+45kf!-B@ocTaw!1s~$XKXRg-J{p2sF4dMg>~tJXM)2Np6*;0H z1PL9G$KRzute`G^8k0(YT3RPA6R%aOYNoZ4v|YcqT!t+255fh^+8)h3!r*%=YHNOS z`W3p4K5FP(;VuZ1x0>KUxmHt!gt70tuJwJktz8x2SH5Y4RI|Ch91|}DyE`qsEbyw> z8Fx=EgDCJQr~eNA>!3Jy?YMX3^U|Us-%*bl9!x@JcitXsRJiP(9FhyL-)AR#muYYRgD~tPZ`FmC?)N?$@WTa zs!<4qTlcfH;ZVNpu2&DjO)$+hCEkTGzI<9liIJ2f2~`4B+VEvSXT*Kp{x(t%Ya6@7 zFJh9pXoDD8OH1E=Ag^|_@iOYfG9;e6ukIc9*?G6@x5o|o?sRbAY;|Y?QQ+Ew=cSuK zfiEO6l5)j&xUOFs3OYn-g3yFbUt!@eF>JvX!8G`k(C?SSIqPp1zK0R)o-ke7k4~0* zTnKRxx~(72xlz6jVr3z@CG&=$Tbh9YXAvRagu=xeOIx4YiHW9NgCGK(_1&?^BxC5* zDSU5x@SA(|SxgnR`FF+F(?7ssJ0rr$Y=}TyN!JBCeh!xT=RP824q&R>2$3|9bE;em z$_qS@VKuwc*%|H(-W{Sx^~dj_#9O@$>(6gUoND&)hu4zLa)4K+&jJJ(Rx{=<)&>Od zYE@nfs}Qh|$OZFwIm! z$U%j?LAd?1B$rIYDbAx5myHqWO6;J0Dwd)H?1Z1W>9<}l&-plfh}*Z~vO@UqsRk$j z><-e1&IrQ&LPqW&NQR&Bm<1zAC$Uy|7vse&8>I>ovg65$=JN{w*rtb|Y0EP5Tztoz zfL>z6xwQCsYVyq9R)gnk_8So=(%4`pX7*zrIWSj@j+xhPldP)qVGQiNGqM>m4WT7J z?}8$*2lO5eEo@@hDDT7E7tPn`$$sAS$!}A6W~A{_ZiQGfv`}Q@r&@mN5i1K|6Tv@Y zgQXez;_T6NjZ<hzZdCP4}Wezi$RNHNeG!H6-%tjD|B(Kx9C-K-$`CK*Q2fS({Kt3vD;U ztWQpq8gQ!TQwR^YG(*<1L&7u3{m$b}{44_T2)8PC$H}51z4q#YvkFiC8$nywJtNXG|uJ8yo@Du&g#PG|DXl?yxDya?joJ+#Fxb%47 zM%H5YNe!`WOwK)yI(Dz4UuN@mJg9XA`Afi*%`(U4e!C40!lmMW>JGUqr0_a0o7~=s zQU^r}9N?rNHIRa2D*$67sBkZe z7APlSZe?Y77Y6N@Gw38M9!K4Y28GgMCHKUd%!$`jN0Y?w(ME{iK_1G?&c!vR!5g75 zyppCYdoY~hwLs83!aFHGn|vN2721M2nrqQkw?01k$p)$l>O?XJD>uRPh2k|ND|5b8 z2Dx&|&7xAVYA}gY!V3j8aHFCc-j8f%pao|RCSMeQKM$_MM+1qp1WW!jBFU>2YI)pJ zaX2q&{v_45K)ph~T$L3(xKfaDVC(MZ;cWZ3NXkA|MxCYhA%qN{OBS!=LI$u#d~5FJ8ZV{upN!8kr6g`)$jM5_>sI2p)w z*Ub!51<5r}vdw&5Ufo_smQ4sKN?|YU=T)UuiOU|0111ruE@~Wfj7Hz zW^bIwE_D9>4F#gHmDyp_Zr#oKx0;I_AAQpII_?`Es2U?jE*2fh8G=tU8{eD(lr|&KKq!INo};{>MO5n%`&}G$chEY z+`!7l#=Dtho*{CrJ%=px606OAYp=$v%#}cDRo3Obl;BD{uqF%V{TtpleeI$w#uAv` z_~A>sJuo2B+qyxBSc?ric9&@yk)9=F^Y?F#klE#-AQm6=v))a^C(V9moS)2!^I*EP zUrO_;K@vV$#ShR=2^!@yjLkhFiJ1;(o1J;AD+Jg;*?K|~D}*?DxIP?_H>vTU*PE-9 z9&M_TmNxq_NyHcW7q|vMnw+~7&Lvo1zhl{jd}?W#Rb)j|2L~jtJr;87^TQtma!q?8 zLGiZZ7O5N%Yhp->^s^VYF!vy++x^7?l0WfCJ$7H$QddsUQh)||=(i3&APRh%D3z`g z`7!&?>ao!QgEao{ApJW?|IJsDlJPiEoLw4WYEMS+U|io9;%nt)xoquJA_i1o7ss-1 zW`k#s(liz6JUscs=v-;_-4~OBS;UWqYmDr89 z=5uLkA0RA*15>9+ObX0m|NBx~^3cRl;!|5g)4Uk1z*4K#{-^30u52Xj$Daqb=aaR| zy(qUP8fTOXNw$W=Q>vF9@q7a5;XY;yRlN7@6LaHfMD3c(s~TJG;oJmu7rOGaq04Z1z#n<-W|$zm-RSpL~$yEwI>o{A)L=tIGAR! z&9pvN*wM}&U)dUyZY2tbp6rWJT%OkRiVw(z{O=vbzDg#KykoElF`ww(Q|r^VXz=9{&vNXoQ=@rl7rujIQI#~qzj|~+eHrIraX0&ysmhXcTBSVCGM>;E{OXahc?0?ISB0kk zRHy)+mB9p+F~efdtS9DPtD#+tDTlV1`1#!y22z9K2riC?xqT_3pEp$E`Ia~Jr$usk zlZapNn}6%gW;{1(u~TwwqQn)xpRj(PTx_kITlUi{c7>Q^|B0Uj1cUy-B1iwA$i{A% zm>Xw?0uu)$biYkLtLb?ytOOb1OPSY(x?qMMe^h>1wP;kq#7R|Aj^Ogn(r@#(C)U)_ zkMz7l@}a9r?+dSllH@GbHo$k9v0gAg&kws+IAqd@s?<;B!-{?fz_^?p% z)+Xt;M|=*cCM&#~`;q9AU7L^nsjDi`lyCSLRQHqObK_Iw;;0Z@5j4k~#ux*vf0Z+SRL$hE&jJyz?^Ev1weqN{!_BElZO;>NHGQ zOuKMl+2-xQCG01oNelpxc6ongZR$5@g4EN;X69H0ETSrWlSzG3oj*rAwtggHd^Pkns-rS<&!~_^gQ0ci z%%v*_2!-UB&*&*#^RMF$+8L;1ii!PU@wBP-+tr*w7o@L|Ik4OXUky4z7q(ZTQF8d_ zuM7Znd~{dg@lEt#G4CVTxX{^~{fXo(^5%j z5BbiY_!nca!Rc4%a4J?Z%H;%v1!b8UlmV{N;{=~poASQ+7kr5nv2 zyLjLwz)LurJs)1`Id7nhz1m>j3vqc;z^r}9V*RkRnrC+yBU;BJ>_Ky}C!N|EmYSNh z8=ctcBwUxDlOqQ%PQ@W?`cR}hFo@+<1--d*?m8!zc89Sq-A7{a+-NN|VONXsYnHy= zyi6%1?ZlgS?MGRyRy1A(BS92MyW0@UvRhOlW#_9Q3s z{81(c2Sz66hPAp);&3xP;zlT~9f6h~#|5u79QKa=YU#MBg6Njb2CK#KMC`6Lrj+>D zN}Yr1aqQ`pAtzCwY9*q{ejU_mbOYccD(qw+UlBHi`s=k(V$B{IVALyyy9<=Z1>cFZ zTLnR1Vp83f)-kDQJ%)SYJ$H{&RvZb<+(PfAE+X|6E|*icYfssn7>fo^lMSaehTkv! ztc@$0RK_fp=56ib(hZ6tx1|jx&f*A^us7;tS(eU*aV!SQNr)x$h#6-x`30$?TUPi` z8g|}|p^NtXv^0jNZ-NEhe2;TGOVP>?@H>UhT*!war-k`ub-fY?_X5l9DR{k~brfx* z&TXl>jY=X?-)ZhB)Ty5I{jh28(W>{{yN>-{hs?17Bx{uetFofcsxnA;_ANHTDbCzG zpHDP82?Jr6+9|R$Aa?gH*DXS8IbHRf1t<3xI$tDE_`@2K^|LokC9-ZU+5^3W-E>M) zJGt4)K5EOAkq}ZTCd(B9+i{TVg3M+zkMxz`3)7H>t&26Dk4J1+Y{KLEQc6wQ?AH0D z%%JNtjW`Z!<1dOsyvbxWQqNyUE#&Ku<1nrdQsua$!;mlK8FbG!Q9(23u2hoem$&zX z_yT!FD+GXmErM>SJKNtxAFBxco}I7g$}J2D+M$<#+4ux;ViZ=IRXqR$_|HrpO!n8D z81*iFINdCmSDQd~Kms9E@zXV&2w&FDh>r(6&So z$I(u|+zrY}If>89KXDTWQf~G8!;|kHt0vJ4_TtRi=ou5L(+IhyDS_QKx9IwfMX)}d z+FZP=Txt+mzAf#2d?2>EV5pehe8`~ek%;O;E1a2Y}wG*J(^ZRHDvvDSxKhRBmQb@Hylq{EKQ&y`n{>BM}hERa;neJRZ2G3^nYbvhfX^GaU zfvT+|h0hoqOpC(H61_9crznLx;V!l!5{D}>*!$_Cszem|_fO94bQ$D$=vTx#(3~2f zSV~6d3bqY3FFmk8Xmov#SeZT;Zo44y((&fuT&fznQ?CB=h9W*9brE|$b0?{@B9r;D|LMl9 zy+`(lRkwpY=#;2tC=n6~qbQixXPaR#>%Yi+FCJaMhn>`3v7Fiqe3Ua7VWY_x0y#`f zpiio`rC2z0zm{5On7{~SUrF)gw@HqFAu-CSt0^JL@czJ9TXj|7Y>ui~{Lz%|>=Y6B=3Xzc1@Bklsr@SPn-mcA~za<^chEbYiXj3hx6#QhSfk( z`n6s@%%(j$EeIdOGMmjWkX!}cl5^ZkQsRyMKF;b{d%yC==I{qw=HbC_eixpdUrLoP z9OIceaFeumf7DJn8S|N&3A2Gn*c7mSfORfP4e!y(aXfxtd-z?^C7F>Bl(}NFkW*_a z&yvdV9LLeF*9fJ_=5MC?VrnI$9GmUda@aD%p%(1163nWB(RN>bY7;CT?``+bj&cT7 zQwm@9`1EKUbFkkhE1D4QEBw^3b9e$~UjTE&csj+=b7*I3uBpKq4+XqV4iwzS)@}(= zMLW?C_p)oSE8MW+2)K!n!BP}mxWAv*#21$>`4jP{IPTO-?ICW@$4Ip?4sE2Q%rQh` zB`|?f8W}zimIDBJ$7{s!&ycB1s$de=iPI13iZ3LeDqiv&?Y3e?rS^;Iy@Oi#A34PE zaO$#ybR|W@&};Qm>{Eh~&qIU}a5v{XiA79z_qWI0^h;u>w{s*C96_y@m}IB$K$FWQ zjG;-qF^<9BN$UKlq_d^n-t!W2gDZLlYXD1U7-~d{g@T9;L_zL^y^Uk6C(N9g_zO_Slm4oX91C7qkeI;&;rN{QKYFr@ zz5;T=qsw)463yt}kSKMkSuG9h9y ztJaR-R51DAn0JoYK&}20_xnWM%Wi|xDmOE`(_g<7xBePJ14nkOiX>(i>lu{=#RU2# z@t^Bse;tXhLA6H6&eJ`~ng~I?gS74|a)IXGIzogQxR&?ZR&SR`@Yzk(FyE!tU)?;b zyg_!EiFnYGz-jq14QQfH@?wZ(ikJN@%OZz^{Wa^_ zJ-l`;k7rh2My!hgH+}`avaQLN^s72i1iOmSB8UT#tz5ezcx9*|g{I(X=7Ux_^JDG) zm*44McNopzy%5+W@wqYw(v|)k$Be^u2~LzN!2=S?C+i$J;&$P0^zGsJulpW%uV$B# z1aW{c#H1-rbx&^4d%)+pUU3HOZH|idDqo)>1ZfIM9c(jax$&#ze&=O!eZ-z_+H~zr zgAxd%3S!%Jla0Tmmsx`upVuv_TmqJQ30u)-FO`DZ*6M^8gk%us_$cfF3le@`zjq-v z4g;ZyACm4YLS94pM^_x8#tv8>Tt2u1ZAY=OpB`M7WQdz4Y%*O~2_bAWbHaZS;7RYb z-i>ObB$euMm_LG^bkKwFT6c*F5h1UUjLG>$zs6gu`^!j%U0m5fp@z%!(2@oI3CG*! z%Yu0YbFzCl@r?7Fl+e4bPt|BQ|yYmrcCed&iPn6RnI6+mSi+ zG&6NOO6F6%n&_p@=K4-!nQs&zQAiL+j^Rbb+B2$^9cUX({^_^d6d)`mi0~1x&DglG z+<#rR*p2{RK>Kz3y%|H zyf%g2ys?Mph`R17L9<3uVvidDswrry_X4i48W*n4g()2RI?J2&XNskK-oJ93_I2Rw z!X6{(wLRewM1N~G5%@oijW>@}w$oR1E=du?5pW)g_wCbv?rH-zdwz?Z?tnVTu>NytwDvJo^(88b>BCDXaADS za9@_FGM)Cn6#UtZTmyVT#(?BG;`GVdT73H17K9Gm}9SKf89f|i!oP|c%$||71zrhjB_~KQ#(}~ zc{(I9D?Q4KQ$I19ntZ9eskhg(&S%1ka}vNRZ7Y{i2e6~slm9l@J zwv$Vqy_vEhwL=onI5*mvNyGT1+(->@6J@S@F3I`1LP?fI@GY6KQit)~0cQ%3H)!{_ z;Kdag&t!F0;}yqqx`it5ina}0rcZ6alMYlNc)LHGh|vq-^-CdaKFkm7>t9y6$5A5D zJibYKi#PYzQUT|fM*M)VvOFKSq4=uM`7#T86pF=^1-fLO%x?-!uNoY7=)Nb|_^}fk zagj$Jc}Xh-449FFk2nVKCFCW=9y!}iK|dZfAyaCk8b92J6`uy2ufyZSy1an5BZFM9 zr{9v3^n1L!nQ`dH@;L>mnUCh?BqWAH1K&1qOC8@qs(5V&GbZaKq>vfyx!BcPk&QO1 zQXd4iWM7nyuf^Yx`hHjPp{+MY53EbFGtYKwd&3=D_mX}sy7T-84F|VYvMJ!_Ow}qk zwpI#MF;(tt0DeDkLs*#;&0(A#d1)51g0sgZ4GrqS#+d}qFuvzlHF9(0FegErzuU-c zS45(f8Td-dkUCEsa9HwrNMn=k3W4Aa-tn(B;2J@VcvUD^;qzS;LL)&C6$@jU() z5c>ZG2u)HGJe$kta@WByxSpAo>wPbqE;ltQxoki?l2$g62elvf1>DtYj~jZ|mnB?u zL*Fn1q~jW!i|f6k2RCw&2*3u~g&ChA3;xXb=m1$Hs2Tv>fEz>;i2hl_pW6zD!NY(N z>w9UK%7)zMu7#`?>(imrcje37+wWVAmrHq}59QOj^8cR!$ed*PyF=xz&7XBLvp4R$ z`bod;O?fs^eaMj)ov4m#^pGdJJy19}A{{9t{gEhdekHp@DowL7LDBux=9_J1?8m3Y zvYvCXf&hQ%I-Tbm;o6;F(Xuw>T;fC(QRHOVL;#q)8+KO>JKEmUqCiWCAlft5ZIlbq zwa!o|s0AJck{o@xiRUA&-0wP3>cQ(`H5otSy*b4qmt~Ms;KO2rxJjek-dm+bGe~u! zkwzz2#)(uXT`4lT7fOxHDgPi;l%$S1*-H!@AomjYpD@&MrM9|rheeGB0MX~@RL3rh zLX!PM9+Fc4=q_mk`dY(rv-U5WzNB?w%N?#=#4%z?G}BAgYsbIvGed&~RAb4znt8Oxz1u{{u$w5MGA9I4?- z&G3urodWm?Ib|)jrq^PsV=$Tiz2EZr9YQ7FDCWTG1EkF~Ur!i(G1zPM2T{a9=0xZ^ zfakw~yb7YpUk((<5e_*8Vc0=>_5{1p*!S(#aH?duZH$#T5y`m4@OS#Fu)!vWHFmVS zr{yuNEaKC={qruIJP!H$7k>Qwb1>MR2G>MMuq)N~ zoEP^M0*|c}HK!cu%*;pCkQoS-2|5C>@8F4*%Uoaw1c@EC{vrjVvKgyyeR+H6- zirw8VqdvXV-}YkPVs5t1-5{G?v}&83aqiB2{L1;Vh7-WtZ&=(odbUf+H8pbG4~P^? z{+CE`RNmEKC%Twm?r3=zuw=jRI{E^$fmJ71J_*Sbc`^Zu{HT=;b+Hk1GU@-4vpm)!R zG?`f6@XcHzXb{mbTk)qZ=MbXWv6px4Upl=2oXnleFASJzBZ+)HdJ2R}BsVAL*Fd97 z-;!Mj%=&|x8`i!&^}8ot_m*Af)U|+Dm;pui=7oIT>0^C@nKb7h zqP(s9Zl5z*9}AN1p3H7RE}0~L#3)OHBO$b|XI(q)fdF?~!#b&ALpXD){)aYq4vE=t zUuNbPxh*Azk6WByr`yH9S#+>Z=oJ5t?6hIwB&EpQC(1Dx7G8*Ay{*IJugTW}9S^OZ zcj96sMc(@$@dFFuzC?l89&cQ(NM$-OYvz2ZaLj#taPjT=R;{wi+>>mA%^4k~$&m}0 zuXQ1oe|v{r<-bCUblqD!U$_%-J?SK!ml46smeqe^2{^qB#GG922xOq?o!t|AdyPLX zFcs5=@cw#9PrpI|iS!D}KLS;^iP3xB>yXEI@>BHJ3M99F4jony3^2#=9a#0>J!7QN z9)DSVZ-Y;r3K$W48{A7K-kE1~6rkX?8RY-AE3fU+s;u|83&1UaQaxL_$L`Iy2njko z2AEP2r85*ddq64?ig$Kr&J=*v{7;LyjWXJ~g`Q?UpXL;=O8Enaq-9yuxe+~x6RWUM zvxB zaaAv&E&B)TnA&1OD_BJQt$K$w%{Em(uk&@P$HB_^hC0GY<)39)pS1&{i2l(*c}@5p zfV6Dh_xyYmoBI(TD9bgylo?>lwJ6rq%dvr$_JUv!zPPNfi1|&kkiS5>ebl-b+SA47 zWH*}zJlD@n7SFDr=qn;rz#k%%PDiGF|A^)nE~YGf^!sWd95^YB9f0=(L(VHLGoiDD z-d7J$U72|NZg*w*hN$ z#Rx>$X$qE>t`1{)T`VbjY>~F7k5ow$8K>7e<#72m!Ula=w!F(PlG`q0mRkw+l%xLEd4~!u*c#g zA39}CHnOIQF%3W%;L-k=>mwkMv+d*>)nsabOL|RLWj{ysXdcrHPYAuzny0syVHNCP^N`2f00ogrN-kedWe(9fGUVsg#M(*{;$Ys zd>Y8V$!R7)R&`Aj{d3fD>nuk?=`-6GjNmr^htz*F>>k?XV&YK_EH{`wRVe%-?m|WU zPfV{)?dyl}tG<1Tq)h5Yy)QIj5y_Of-}AVH$5HuKRe4STAMj`37P`7Wjy&zx9uQ-I zLTK8;;FtU0oCWN~rn*B3wmOf`#g_2`GHZnTo|Q9Wuv(rUp#<{gqwq0Iru#9uHcGmd z`gm6>Cj6hq30*Y~M&=qI@+2lF21>(5x^y=N1+eP&0fL?xI&pPpiAS|a64(Y(Ev}wZ ztHWZAA(zrX5T)RkY(0DL^d_~UW-t(I#HBzG8Wm;bU=3nHKNXI*T7sn&#nNPP-R~_R z2cUkd@j73VKGQ!B1UaOK5=VVsmB z4s(T^-6DPry$C%@L3)ZeoSr`Xr0KI1cKJ6SC?}j}V*sEc9sUO+Nb>w9MhCU*jA7-; z)vG(}DQ5LHIJrvnz65fVWkyqG!d&CF9o9vFEb?DPakA#q7m_xV3+I|*^>>qpkW?-f zQHpx7&wahBl>^7gb(TL7BYUjU4Pt_6P>d@)r}d0Gos@Lt=QkZcx3OOUTgsoh;QCw7 zj68ZThUx|=8`gWu9!Q~!F}!f5RZli5S>Riea_VOFJ&>rOA4l$ZR>=Im^YXJcJjJ{v z7&K{zDr(W}+{WChJu@U(S2DEu0@DPUm%OC%5BO_c6r_8Oqn@|1p;3RDmHk zNL@z}6OQY>|0HH++X;v!UG^N6r7}+E&>r2iQ}~B9Wv70{n!I*8e?+bc7w*4e zBJMY3mvbmM&lz!qTu{tI5aNsV-TLM7n8dpQ8jsTM^Yf#xL?t?+M?_X?D+?4O47?vu z#e5zpzowU4VUN)$*bm|`2&=XGG?>2P%cl>&S7G_Qqb7iZ(xKyG$BxhbQ{;i>`l2(M zqkCMgqU_xhY>Il4oG($AD5!&ZAgNG1v(D+FvW4QyWf!&B6G*1Q?Wh-bd`C`6)v}-2 zB6<_KI$*-G`>Dz-@g?>lN@=nE&U?bc&xNbl%|^e^-OOjbe6t5Pk;p!P&)YeqZHmN= zy-(@7`+2T&(eo45k@hqZUc0dzEwL?Nm~$ZBP@9~0@+-=P&!}p!L~y3aS{wsBp4L+& z{v@t)SQCPo&3Sl{C&s&^z|YI@-d+IEh`>!+rlQ`W4#o;`6nY9c;U@F4Gu`IZeq#tQ zfSaOZFjd0O*?!zo2#XN!XtN9F_wK-fy5;_~UB7cR*)z|lqEl&3mWlOX{h!?vzZo2e z4V9V7dMv=ZR@)D9Y6WeTr(C3 z*A{KmbLoV$`yl-8Lp$ffsVnyhhN!r&O?%sFiYZC>7xpaULlb@i+YJz%Cj@jiat5uB zv3KA7Vn4sbt8Gi@Jw`h(y@bxD$H&a3JMwTr?<}3o?c875q?`x|WM)*J$*-}qUkc(a ze``wKu%G&M383$y#Dhm{dFys&__pgL;#6i}qMaU37#0HUg~JloLdh&RK838vIhMTX zCUJoCZJ28>A2{}tdKdpBz2$Cduw7FecTfrC$9d9E^JJJb($m}#1#1c&0|B%Ox z(#kv=ZzB;ic_O|?w?}gp=ho0WNeLGlb8qpOJSemHSVBOHIuqc&$^I7-)ep1-(mU7Dv+5FHYJ+h|FWi@)sun#1 z;Hr4^C8|&)=#@~9%dva4F2K9vdy#~tgoofNM{s-|*K&<=Yt=pu2*CbCZP&h}K>p;+ z*Ba>_SmNhpjo2yRNSA_5qk3nQj5LZokFGb)VNp>w4C3>nJ;T$O#f}h?XSO#Nz)1Jg zuG5CL-Bxhz{0_7|F|3*_Fr$SO;I1PkL}ocisgw%m5uY~#k!81-@P4=2ifA;lqzu&Q zPDP&>G@9FY7~1Mg!EYPzG=mK;pkj|09N%SFMn8z5pWX5X0f7|(5(@csx-voy-_Wu7vBFcr!FH% z9l=cthl(?z{cEAhE{64UPxfv~DbNHjiMlVaiZe*NF@ccVs#9*nSXkq$H3q;&>c-Hz zp2M_XXs=DQr|7*$lecNb6B{XlLs3^&tV(u&$DCz$ z&d9V~E5$tncX=iS*~r{ETD0x-bONqb@4ZZ)_`Zu$HVM5fe3)C=rkNx#?K{iCn4C3r z@5i-qS~tCVm(L5AHQfWF1*m$a9u&7S5L6_peZew~?j3#JGijP@@q-O6O7x@KI zjo|ODYzNp4r7xfl?W#w8x?r3_A ztW=i$oCE~Bar&)54TQeKDB%N3za?~d*``ze?8d@)kxNSSO*XN693Y8t^zS-&P?a^n zvud+X$4xR;G(Fb9M01!m0IZwsqaZUzfbi-#gWVrw)_}OJ#!XSISpPJAis9}kRhf3r zpo2Ed=UlqiB)~9c4FKVXWaw(K`Kp1K=%Gex$J~ipZEh;5%X2|NL<23qE>#-+iUrzy z_Fw2$09U28pGSkJcfsl963mf-D>umC8y~Ahp4O`)W$Sg~2&MDRx4lhfPz&=+y$27` ziA0#2@aPwhJ_K$FFt378HH=4Bx5uU@n z`2+d@ooeY7hIEgW%li`ol6Hf-6HX@VO^B%~6Y9&qP5y$7-SQxjDW#9`XzQpqwqKQ* z&%suvwN0-`t6lGIWAkZ)f3xPpQ256Bi4bX*RRk*XMm*u;1cZPNO?Ol2>kfN60AMXI<%!VjJ8L^`#v_0I4&Mxd9 z8uyS(;cVNt4A=vv?ahPz%xVdn-c>Pe|DIxY?~vmnQ-$qU?WTa zab}Hgnc9g?(JDAEEeu*T&v5j~;*~qb;=gIK-7C!djVh?Vy4^#& zLW3V4N+jqN7B7z6u{vk=Mt*S(((RK-`YkEf-COygJwnf%b2NhF@t8w;DFo!u+`ZrE z1=yb^DYb#qB^U!i2tI5wOCO3!-tb=gA)8$7cE#A@-g9`kJ5g|vT$PE1bqPDmeIW$# z-95{hGiZ-Qd3Es~a2eKr(JckYT7W4q7C-+$(&Jfh-mV*XI`<~iW84|r-jJ+w4M}y9 zg$$CHchma&w7}omAt|Q&b3_OHIilm-7@TazKj}%}M3`?Myjqpm504vMyju^@)7*Ze z5<04K7&8ufyFM(1=(?6>#&F@xvj3_fXg8^6g-2$G$A3{cuUzBFbENoeO3DpVj zEd?0XS2<7QxXJLVbRppE%)O8EI)$9(lV07G_a!Q2%&61ucV%=1o)7f?VPdJ~{KLdj z-UgUhF~?qUG+MLiVxBvwd$Bip$XkQ;gm~ZkB?0^_WvsyHVdmQ~s}SQham(ZvHbIJ@ zA&MeWbQn*sZk^73mX^eijfgyN#Nyqe6B4?XnOw z2@X`$KEk+&CF=|R)vd$eYLNcKzpv7|GH=mTmo;k=x+m|xQ5^*Ccjc~E=~eR9EfN_Y z@vrvGKQt?ch!x2GKQybFZAOz4626n^lzCi3On;wWd)&xr9T@d{N=YIs*CkrtW3{{V zt7QJZd&+K?9k{ltXh(t|sp>mohVtOLCgYfbr*?T$Li}%r_x+uGiGS`BlYcVEV!5xZ zEKDb2T7~Qh;BmWL<&c4eNou$M$pCx!UoybH%|HI-kF295bY;F|y5G1x0q~~Cd=~0J zgqFH73w4PcryXIauu!J+tFHF55{#1k`{Vz^1137Z*#R& z3-MkA1L*BwRgoe26?^Bi53}%PIV({`8kaX2jm+0^a%xHkW50~*KLFZGyWl)G0GqU< z;X23pgRym~%7LFSk5+k`a8qX_Lc~zw#`nGHh+DqlZ{V zZo&m1qzvFGIXu<@5Y-Qw^}qh-a(7R5`2i|k3nrZgnKmjFuTn$&MeBln%fP&Z9s<5b zAN%xavzXYyVx#kxyPq#vlxuRHt%xPJh5KJ6+PPh$m)9T5X6MZ}>{yHokoeB4|Bqb;NuJEHWNuDjkoh!xd{SzrXJ*W!E-<+qOhyL)ANVXdJm5$eaqDT>m2up>IuT>SC`>Vic z27gkX_l40gt(YGT=cx`8=e+v$$}LI-7~>Xx5yp966>%c`B@3meSGY0*y$>?b&*>qj z66jOfv;SlPC6$(z=DJ?I{hT-&Z?!bk+&0VR<}TVsdG1QHla;}kXrbHe@v+^+*`mnx zu#G3@scYq#@+g;`h@pLX#rLbKe9Oe9|5Qb?v^xC#6M&7{AAs6XpSsHNG7O zfWGq=s(~pGzaq!qCv|`|(9YEANsS)>nw@`Nx$kRDRF8ie?%T{cO0GmHUf7+1SjYi&i^sR|#ISf!u`<(e zTR@be8uG6w#lOlO|DEMc?gM%`taq5wUxZrxs?N_<2)jSHMZTqlkhWZ75<5KXxc8UJ zO@TCSkeWXrm&5hJUP=p4-gzkBhuoOkxik*`^6Ad&GNd9qj>y@GX5Vp($=Uu=SFX3= z)}q?{J+hx5A7940)%d5H&#vU3hPmU+pHT7=UkCJ-(jw&>C~(?vfmv_&cyGGco_4He2H+f6^38IjuX!UvBm;i?#If;rD@ zK!4}W$29n8E9#<^Lh!C))7vZACi`PzY}Po@=s?Ix(qpgM3XLANaInt}vUosJFd5cG zKa}Z2Ox6nMKq?U{v(sLCKP@pwl^7j;f5}=P^_`fku{|kAm=p=Q9Yv5t&E7?BhsZ-6T8m{Uk;6Af?y8^2r zUsc~LVJeFcTZPOH(tMY5J05$+UxX^J6puhc_1xPD(x}Vw_$x6zI{xYi%gUhRdvdb{ zn%RS1rc#-ANVer7P!F}!*~yasnLrmncPp3IvDpCs>J*uLS2>Nk69e+R&yIt-&yLGH z_LZu^6SZSqWl)Y%geXiOigoK3f7zyrhHKu8MwG0D0CNUtkr@cI;auGMfZXT^=A;`3 zw6;|K2r~k*WIs+>#<*|etlp)-v6g1hg1ik5$L`1jT5h28-uV8{dg9`PTn=7{D;Xz( z!gd`%Q%YG1W1wl6QAmYJzW0m}L_f8F#RM28JqQk8t74Y)Sg^~q_3Jajg%s9ejSiPS z%RA5XwXS|F3m=lclNS-tEQYZNf{YX2|P<5gD=5ftB1`H{k~R zlQrijCqk8D+!M4Q@ZfS`)`6W@w4;+%Kfb1c(kbx<9?LbIvoN50sGtc0=6S>nrWJLx z%waXrjOw}-GDQ!1o`}S6+Q5YY;)RCqcdheYmMIKvlAc-pwSy1NbS;|)h@JO>E}c8q za0UCWb7cX8C0(nOuCD|1R@*;ZYUN6>irh?ncISL7fJ8`tA&@v9k@Gm~4!)90p~H^d%j1~=zYZY*)_MP&in%&DzVlo2RNLJXuNSkr zN8||PPAl&xLQp=`a`|EHA>RTgH+M+N<$?YBX#RBL^EnzoSheZ%%Pz~qWINHXDHO*} zV?O9z-RnQgCBj6~?(#%Ez3#yBP+Rn43o5fc&UT^&)g?eN4nZWc8W9RnFkgGWw=Iw> zyMfs+Xk{W(@@*Q>NnESfL#6S|FBs7f?`X!Q_@qOXUw{0~+UE1~$=8^SAlz5|QI!vO zPs-C}njwH_BjM}Kiz~sx4o8Edrn)P+#-Y5IxKWTKjqb#uqA?W)9PoSc7gZ(F&X=gH z;f<>PC@=ATQ>?#bc@MKQpC-MWdw zt3K@6dt`cP#X*SfuxZ_8XAvD8VP4_@!K*iHat=^{MxzHHsi=+r?vmb^CJ0xd@0Q9P zGJF{y(I>3CC(?6*d5-A!OU?>XIJ4O{lZJBedYErCWY<0oP<`|j#ffpUU7}?=hh5y+ z0R$4O53nbRfzSLrYMz@ITV;~ttC@`;U0F%cfnV3Z%m1l&UV0l~FyhcYfd%$w_Z51IQ}GLFeL1iJ4cJ-jw*k{`cEkJIOsvaE@6(bFa0dWuWZZg34Zp z)HeM=LRk;;#r==G3?r!u5ex`^><@l}#>YXpOMAs$qz+BiLO) zC^7DO-NrYp7>aP#^gk3o*9TOnn~nU)<11bZN@9;muXdgWw|g>nl2;N()r;=8ducW+ z37c$tifIt;v3)m>KXS6Ys;*)eCVczbNf?l(J&x&*q?t*OGsQ~!e{8dxA-PG|6!}MaV6FDmejr($#BXmUy)hD z!k;t>3_%k9w1R*rE4gP>-7P?y)$9j5h$6OIk7#Z9BxgV`RV{PNzkUCK%f?XX*^#xP zf!HIb=fqc8A;aC>>Gfiww{BA?Di<+h@?af3CS3QqvH?C!kogJUi`tZIOnuTak0G;e zrQtLK9Y2~y?7foFFS{4Q-VD%9c|bvrVVfNP_uRwW*d~;pj1b0SZl~GU))UN<&P@^h z0U2K&DTTJWV*Q7HR8?8x#0Y|Xk~Y5pxB6q}{eJwuv{o@K-q@2z2pw!9AXf;@78?!{ zNu#7+ckw~yudpZ)&+a6ozf%WC5@`L8GQ5it%7#dHe5!4)+TZ!@uLntSQhs*XS+0Mq zdepstkR&V&$m~)_*XwBkVt||+b+O(mvUhBN&C(b9g^Nm-A(w(F={p1m{r+x~h4-yX zCWgn)*+I?C!CwF=W)K)qBL?9B%Ek!93?LKVm&j~^4*_(9LEwKr@RSD;B8u!Y9$tOv zKR-wl2Bg6cm+=Ap_5a}mBXVi*EO%1wjlYZi`xmQWAPP{!sgLVF#r{=PlP(01NWad( zh5H}%{rSM-4gw-==bgs*Um5~@uuFx*25M{x{`FrP0<`MirvE$Y|Hl`Mufprr0OE3! z_UpeoJJXpYJs>x48z!Vf3E_17^R!Qw7pR0x|Q)&HBJn>GhP;WcD$tu{LqO; zwHEpQ#CApON0veuo%5^@{p7$ zjId{;F*;h?)k=mY26e73g~qnidhB93nLAT8PK;3rv+EOW zc5uFKQ8=U1Z~i5*n5kyFzO=_Rrnq=Ff(*QlhI>lh5F!AM+UA&P1P_dOv>%u(N9Aj< zB$V4inB(iqEju_j%)mrZe6I>k_7YR8w%E?CyqB4qdyjoazs+Ug_;7n#oIPcxBcjB6 zCCN#{{|nHi6*leQGXdLqORt)DHB!8Z)^!Ix;>o33lT{HH=!J+9F_-nvK(D3t+9;Dv zlU@f4-|Woegl)pH=SKA;g;0ks?G@+$tGzS-hid=h_>dxHX(n6OmMn#mx-piFZX+gZ z6q4v#E6UE;#!`&27G<~@ButFVr9~LrYS5V5s8o}k8M_*@7{)r^!~Nd>;QQnE{B$1Y zd>-fXKA-pUdj9l&FHVZ7Lp&k;&X6t7S?&iyP1fpL`AOO15+bMrCEwqi9={9mfvvO8 zuGhR!Yy`*8)!<9;3#{C_U(&JOD3g%}NDl!{aOaTSaf3biK~0-ZC}j5`|YH#x_}AV!x9y z^XKw-zSe4qFL;m4)*;$<8ArWH-``9;$+O6$M=TG=v=W~F*-Me6PQrs%xRV?nYNjN& zeTX*4IN&A^+fK55=RmtH?^5o}SppO5$@z~sG^Ulp)r+%E@`gI)L&ppKLO{7th6MhY z;W3L>zTLT%>+gF6yCjjQyZcqHB+>GYY$E%)a7B~msscT;lP&Hc15i{lEEyEF_8 zdX*$K=VhryU4s-HqsM6FT$*s)?W1bsX<4o_&m(sUbx z*Il22XQ7?L`U@OPYldQBkq*}0@*Y<`sYkL^w%rBaa@Pe$xsfa|LQA#jSeY{q8jd>e z{dMWD9WrFICloM9)oim&xURD_i!XEakb;Fx(R75uu{$!_N3pj>$FQWz^FL01CwjxI z-BbrI_P^#5A%&dCP@3 z&OEWh!r%7QtwDJRfd-{(+Jc;2|7=Q55KZj&$7;1{4vVSJb|R#!^RW~8f`cZW=~0Yk z@NBU8-SdpqnJ0PGq`RY}(3L+<1-T+yU;%#@A+MTeKW((HliXhx7&cplb#>M6;z8Zz zd#N@F1!Gv+?A#RandQkKDmHzK9(GIu+LjBMa)LtFT^~F$2Q2QDM)! zhnx1wG9A}HRfJE1&At#x-1E&5ES1ujH@NuH^1td0MMc+(2V!@Nf&KUU$uQ!j_T=wV)c2iqt?-uD+_=A z{w_ar@eh!nu0ahrUWN;wba6a=Y9uV6Ef(>4bbZRW0bhnP)ozb)8t0N9JCl@kWp>og zC9Sxo%}>e2ft$$E{J^rrLS*gR5C>V!_K=Q^=ro@3Q0SeAQnx(}<6W3X-llj$NK~l( z7ln>(8ojWU8V~yKuq@RX5<_9#DjexUL`o?w`ok$Y;Jo-9e?=)Lt#df0{(%qBuh>OM zuUYM|uO)U6P{%~oO=Fn-^nyBjBA{P;1?%=%$t2vb89vtaOGurQL!l#onNRmNz&rS$ z<7g!&leFd-!H2tcrO+FQA!NHX!S$JP8;Vcx)+!B0tyMi&NNi(xoVlS492!UhzF#C^ zxBHM(0|2|_yHes8+OPs@I|-wuRRiQFZJ~B^AM@*M^K|K<`=EP@XH^Y!O^av>HzpPC zcA^P0j4Xp{7YCCxcz55*>N9=)()7Mwn9xo`q@2U-roQ8JtVdlj94+rL5Lr87a17ch zb9=#fG~q16*K?86tybVJau{Wmr+Rt@s7yMXHwIwlCZdkTEP<%R%>g1o@~l-GZjHgO z^5?s=48LWYgv^FUQnax51WylO@(MrIULx4o9gnWx0E*Y_M1f;8>bZl@se{u^>)i&p zwmB-duk+GeX2-V1ygwxSiM;bBKsnbyq8Qs`)CQk^<7GyNk)9~=`Jz8#C-Ap9F#c53 z9E2`#)bH-Fsu+LpVL~doj61Yi)!KVDFV~Fq8reEo8Q^slZO9P1NDU&i^f;In8<4YH zO#8UHzi?wADGxinBA-A)6ll||Uo&);Hrcj;#MKu5*hWvNy{mlpc9pyU2c1o>cz+;r zmuR{=dpKsa)2wzV%=RV6`Tn|<2g{}f4dO_iYGhI%D7`_~I`@4BScK>pp41a*KMj@f z-RCI88BzSrlU8T($X`|t-7F6NI_tzs*fGVpcUC$K&Z8Gh($Oov0TiQ-$okxN$|52? zs4*O9=nm|jp#i|szkC1XI}@9L@(FCHt>&Ox{d7igpp0=XnhEQ=>q{+%iyt&a*e+g| zR=JP%I(XC$Ap7mR(sX#-uAnul^m2;0YSH2j9}J&EFZ2e)xpH*RQwQ|weQQ+=k8Inv zrI(yU!$sW-k?T$W&4nDjEVXTCWB^F3&Jg=8ajUG+PA?B3r=7Bch%Zd6tgIT^6W6+^ z+zC3_K03AekY|6~iY@k7&dVVBF)wTX-N~MEf$Xt27>%BdB^R&eOh`1+UzznjqVb*M zd%eMP*^1*Y5ZR-73gbjYsvidJap@JuEg{zV7o4oQAs zHcXqBz@x8EdaG6+H1-0!s7^~8>g>h*LcQ;;QaGt=3!1q12Z==b4j=}EfDr=5lAP&z zKwxwZk|~aju~Ocsm8t^u&_^`W`8uSOk7zw_(#wJ-;N}xpIapb_f=cEIK~OvEC{Ft2 z+JO<+&li0qaXcQ6$?p&_vM8Bdc`J`2OjiR^gFG*<|C7p!B6XsgtlTYzcFF<~K@r1I z^!vyI_C7gChTJVUy6h&s-u`4DEz-5Mzd1q&!x2Wl*{SgHXVWO_Fv*`P&yb5pGI`Jml;olBbG>@MPRnJ?fzXQ|IHf1#clrvD%+Gq^Y$Dc(@Pfk!<$l#K&FB|y^ z#N(8_w{PEm2gF|$LUD3VbSIXcj_+y*+@R^!?W6GtgYmc2Yb!)6a+>7G&P52K<1{9& zaZl`U(xyZ)IR|Z5)#_gn(Xtwl8Psy?jLA0L=Sqx8CBHTUCh%EW%7{ryN|xXUe3Z_C zy)(5wnN%Ihg{J13b8T*Zsz`gH39~ANhE?#Yug!dXOdEy&v?{NXu@H<22V3t9!c)~H z%T+`l@6$db3qpiy-2XZEFu_BUw1nLp4{Q8U9L|}0TXv(G{zIIdU=di%?DrmvYpZYT z+E8v$Gig~e<@45>``ARf=kFcExA9$`zmKW~^m5G1>!_35sGg9G4q|Eu>bshIZ^Y)p z^4frH%(s!ydC#9^r=_NDE^RuHADtAAs3`Rr7MtnvhSp+bBCM|&e7ziBOdE`dQ3Qsa zGuIQBE*2FhtPa@T`ZmuuMK3If)U_!M;_;vMv;O}d`Tv=0MZ*6m aPEue=giF1GU)f3^;6hkA!0UgzmiiwOG$Dil From 7648c3de326ba5099ca40013b6732bee8d7d8c05 Mon Sep 17 00:00:00 2001 From: Jason Joo Date: Tue, 13 Oct 2020 12:50:57 +0800 Subject: [PATCH 25/46] Add Meta_Cluster_Maintenance.md --- Meta_Cluster_Maintenance.md | 79 +++++++++++++++++++++++++++++++++++++ README.md | 4 ++ 2 files changed, 83 insertions(+) create mode 100644 Meta_Cluster_Maintenance.md diff --git a/Meta_Cluster_Maintenance.md b/Meta_Cluster_Maintenance.md new file mode 100644 index 0000000..72767cb --- /dev/null +++ b/Meta_Cluster_Maintenance.md @@ -0,0 +1,79 @@ +# Meta Cluster Maintenance + +## Get Status of Cluster + +```shell +metad-ctl -s ip:port status +``` + +Sample output as: + +```shell +Cluster: +Leader: 3 +Term: 8 +Committed: 4685619 +Applied: 4685619 + +Nodes: +1 Follower 127.0.0.1:2345 StateReplicate 4685619=>4685620 +2 Follower 127.0.0.1:2346 StateReplicate 4685619=>4685620 Vote(3) +3 Leader 127.0.0.1:2347 StateReplicate 4685619=>4685620 Vote(3) +``` + +## Restart Node + +### Restart Follower + +Feel free to restart any follower. The only thing you should take care is that +it's better to restart one node at a time and make sure the status of cluster +become healthy again. + +### Restart Leader + +Restart a leader should follow 2 phases: + +1. Kill the leader +2. Check status of cluster whether a new leader has been elected +3. Start it and now it's a follower + +## Add New Node + +Adding operation should also follow 2 phases step. + +1. Add it into the configuration using `metad-ctl add` specifying `id` and `addr` +2. Start the new, empty meta node + +One node at a time. If you want to add multiple nodes just repeat the below steps. + +## Remove Node + +1. Kill it +2. Remove it from configuration using `metad-ctl remove` + +## Replace Node + +There are two strategies to replace an existing node. + +First is remove-add strategy. +In this strategy you can remove it first follow steps in `Remove Node` and then +add a new node follow steps in `Add New Node`. The core point is that you can +use the same **address** / **id** of the removed one. + +Another is add-remove strategy. +In this strategy you first add a new node into cluster and then remove the old +one. The core point is that it maybe safer compared to first strategy. But you +can't use the same id or address because they will both up for a while. + +## Recover from Disaster + +If something bad happened and the cluster wouldn't achieve consensus anymore or +there was other reason which caused cluster can't work anymore, here is how to get +them back. + +First you should check which storage of node you want to recover with. Use commend +`metad -config -dump a.db` to dump the storage to the file `a.db`. + +Second you can boot up the first node using it through `metad -config -restore a.db`. +Now you have a single-instance cluster. Then you can follow the `Add New Node` steps +to add the rest one by one. diff --git a/README.md b/README.md index c55a438..ad1e68c 100644 --- a/README.md +++ b/README.md @@ -175,6 +175,10 @@ Pay attention to the following rules: - Append only. - Carefully set retention policies. +## Maintenance + +Maintain meta cluster please check [Meta Cluster Maintenance](Meta_Cluster_Maintenance.md) + ## License Chronus is under the MIT license. See the [LICENSE](LICENSE) file for details. From 344cc19d06e63f74ff37348795f51176c5593677 Mon Sep 17 00:00:00 2001 From: Jason Joo Date: Tue, 13 Oct 2020 13:53:56 +0800 Subject: [PATCH 26/46] Add documentation of maintenance on data cluster Signed-off-by: Jason Joo --- Data_Cluster_Maintainance.md | 93 ++++++++++++++++++++++++++++++++++++ README.md | 2 + 2 files changed, 95 insertions(+) create mode 100644 Data_Cluster_Maintainance.md diff --git a/Data_Cluster_Maintainance.md b/Data_Cluster_Maintainance.md new file mode 100644 index 0000000..58438cf --- /dev/null +++ b/Data_Cluster_Maintainance.md @@ -0,0 +1,93 @@ +# Data Cluster Maintenance + +## Get Status of Cluster + +### Node List + +Use following to list all data nodes in cluster (no matter alive or dead): + +```shell +influxd-ctl -s ip:port node list +``` + +Where `ip:port` is any **TCP address** of **alive** node in this cluster. + +Sample output as: + +```shell +Nodes: +4 http://:8092 tcp://127.0.0.1:8082 +5 http://:8093 tcp://127.0.0.1:8083 +6 http://:8094 tcp://127.0.0.1:8084 +7 http://:8095 tcp://127.0.0.1:8085 +8 http://:8096 tcp://127.0.0.1:8086 +9 http://:8091 tcp://127.0.0.1:8081 +15 http://:8097 tcp://127.0.0.1:8087 +``` + +### Shards on Node + +Use following to list all available shards (only id) on specific node: + +```shell +influxd-ctl -s ip:addr shard node +``` + +Output: + +```shell +Shards on node 15: +[513 549 556 575 578 580 582 585 593 594] +``` + +### Shards of Retention Policy + +```shell +influxd-ctl -s ip:port shard list +``` + +### Single Shard Info + +```shell +influxd-ctl -s ip:port shard info +``` + +Output: + +```shell +Shard: 594 +Database: _internal +Retention Policy: monitor +Nodes: [15] +``` + +## Restart Node + +Feel free to restart any node if you have **handoff hinted**(hh) service enabled +on every other node. New replicated data blocked will be cached and retry to +replicate when the node was back online. Any failed query sent to this node will +be retried to other replicas. + +## Add New Node + +Adding operation is simple. Configure it and start it then it will appear in +node list. + +## Remove Node + +1. Remove it from configuration through `influxd-ctl node remove` +2. Stop the instance + +## Replace Node + +Replacement is more complicated. For instance we call the instance to be replaced +as `A` and the new one as `B`. + +1. Add B into cluster +2. Freeze both A and B through `influxd-ctl node freeze` +3. Truncate shards and wait a while to make sure no further writes on A and B +4. Get all shards through `influxd-ctl shard node` +5. Copy them from A to B through `influxd-ctl shard copy` +6. Better to verify the actual data directories are copied correctly +7. Remove A from cluster +8. Unfreeze B to let it accept creation of new shards diff --git a/README.md b/README.md index ad1e68c..6aa295f 100644 --- a/README.md +++ b/README.md @@ -179,6 +179,8 @@ Pay attention to the following rules: Maintain meta cluster please check [Meta Cluster Maintenance](Meta_Cluster_Maintenance.md) +Maintain data cluster please check [Data Cluster Maintenance](Data_Cluster_Maintenance.md) + ## License Chronus is under the MIT license. See the [LICENSE](LICENSE) file for details. From f055d142465cd9626936b44982da06605efc7522 Mon Sep 17 00:00:00 2001 From: Jason Joo Date: Tue, 13 Oct 2020 13:55:42 +0800 Subject: [PATCH 27/46] Fix typo Signed-off-by: Jason Joo --- Data_Cluster_Maintainance.md => Data_Cluster_Maintenance.md | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename Data_Cluster_Maintainance.md => Data_Cluster_Maintenance.md (100%) diff --git a/Data_Cluster_Maintainance.md b/Data_Cluster_Maintenance.md similarity index 100% rename from Data_Cluster_Maintainance.md rename to Data_Cluster_Maintenance.md From 89460a0a3fec40a9d4c673dea526bb079204376c Mon Sep 17 00:00:00 2001 From: Jason Joo Date: Tue, 13 Oct 2020 13:56:44 +0800 Subject: [PATCH 28/46] Polish documentation Signed-off-by: Jason Joo --- Data_Cluster_Maintenance.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Data_Cluster_Maintenance.md b/Data_Cluster_Maintenance.md index 58438cf..af33476 100644 --- a/Data_Cluster_Maintenance.md +++ b/Data_Cluster_Maintenance.md @@ -88,6 +88,7 @@ as `A` and the new one as `B`. 3. Truncate shards and wait a while to make sure no further writes on A and B 4. Get all shards through `influxd-ctl shard node` 5. Copy them from A to B through `influxd-ctl shard copy` -6. Better to verify the actual data directories are copied correctly -7. Remove A from cluster -8. Unfreeze B to let it accept creation of new shards +6. Progress can be checked through `influxd-ctl shard status` +7. Better to verify the actual data directories are copied correctly +8. Remove A from cluster +9. Unfreeze B to let it accept creation of new shards From b2b3e5b1dd2ab96c188ec7d85f0efd349a682d97 Mon Sep 17 00:00:00 2001 From: Jason Joo Date: Tue, 13 Oct 2020 17:21:58 +0800 Subject: [PATCH 29/46] Show more information of shard in influxd-ctl Signed-off-by: Jason Joo --- cmd/influxd-ctl/action/action.go | 8 +++++++- services/controller/service.go | 28 ++++++++++++++++++---------- 2 files changed, 25 insertions(+), 11 deletions(-) diff --git a/cmd/influxd-ctl/action/action.go b/cmd/influxd-ctl/action/action.go index d4acca8..543bb25 100644 --- a/cmd/influxd-ctl/action/action.go +++ b/cmd/influxd-ctl/action/action.go @@ -171,7 +171,13 @@ func GetShard(addr, shard string) error { color.Set(color.Bold) fmt.Println(color.GreenString("Database:"), resp.DB) color.Set(color.Bold) - fmt.Println(color.GreenString("Retenion Policy:"), resp.Rp) + fmt.Println(color.GreenString("Retention Policy:"), resp.Rp) + color.Set(color.Bold) + fmt.Println(color.GreenString("Begin:"), formatTimeStamp(resp.Begin)) + color.Set(color.Bold) + fmt.Println(color.GreenString("End:"), formatTimeStamp(resp.End)) + color.Set(color.Bold) + fmt.Println(color.GreenString("Truncated:"), formatTimeStamp(resp.Truncated)) color.Set(color.Bold) fmt.Print(color.GreenString("Nodes: ")) fmt.Printf("%v\n", resp.Nodes) diff --git a/services/controller/service.go b/services/controller/service.go index a702301..dd52540 100644 --- a/services/controller/service.go +++ b/services/controller/service.go @@ -163,8 +163,8 @@ func (s *Service) handleConn(conn net.Conn) error { shards, err := s.handleNodeShards(conn) s.nodeShardsResponse(conn, shards, err) case RequestShard: - db, rp, info, err := s.handleShard(conn) - s.shardResponse(conn, db, rp, info, err) + db, rp, info, groupInfo, err := s.handleShard(conn) + s.shardResponse(conn, db, rp, info, groupInfo, err) case RequestFreezeDataNode: err = s.handleFreezeDataNode(conn) s.freezeDataNodeResponse(conn, err) @@ -493,7 +493,7 @@ func (s *Service) nodeShardsResponse(w io.Writer, shards []uint64, e error) { s.writeResponse(w, ResponseNodeShards, &resp) } -func (s *Service) handleShard(conn net.Conn) (string, string, *meta.ShardInfo, error) { +func (s *Service) handleShard(conn net.Conn) (string, string, *meta.ShardInfo, *meta.ShardGroupInfo, error) { var ( req GetShardRequest err error @@ -515,14 +515,14 @@ func (s *Service) handleShard(conn net.Conn) (string, string, *meta.ShardInfo, e } for _, shard := range groupInfo.Shards { if shard.ID == req.ShardID { - return db, rp, &shard, nil + return db, rp, &shard, groupInfo, nil } } NOT_FOUND: - return "", "", nil, err + return "", "", nil, nil, err } -func (s *Service) shardResponse(w io.Writer, db, rp string, shard *meta.ShardInfo, e error) { +func (s *Service) shardResponse(w io.Writer, db, rp string, shard *meta.ShardInfo, groupInfo *meta.ShardGroupInfo, e error) { var resp ShardResponse setError(&resp.CommonResp, e) if e == nil { @@ -530,6 +530,10 @@ func (s *Service) shardResponse(w io.Writer, db, rp string, shard *meta.ShardInf resp.DB = db resp.Rp = rp resp.Nodes = fromMetaOwners(shard.Owners) + resp.GroupID = groupInfo.ID + resp.Begin = groupInfo.StartTime.UnixNano() / MILLISECOND + resp.End = groupInfo.EndTime.UnixNano() / MILLISECOND + resp.Truncated = groupInfo.TruncatedAt.UnixNano() / MILLISECOND } s.writeResponse(w, ResponseShard, &resp) } @@ -686,10 +690,14 @@ type NodeShardsResponse struct { type ShardResponse struct { CommonResp - ID uint64 `json:"id"` - DB string `json:"db"` - Rp string `json:"rp"` - Nodes []uint64 `json:"nodes"` + ID uint64 `json:"id"` + DB string `json:"db"` + Rp string `json:"rp"` + Nodes []uint64 `json:"nodes"` + GroupID uint64 `json:"groupId"` + Begin int64 `json:"begin"` + End int64 `json:"end"` + Truncated int64 `json:"truncated"` } type ShowDataNodesResponse struct { From c4866dceacecf146e1eb82cee3ed2ccfe2e68b0b Mon Sep 17 00:00:00 2001 From: Jason Joo Date: Thu, 15 Oct 2020 14:14:41 +0800 Subject: [PATCH 30/46] - Introduce storage info to metad-ctl to check space consumption of each level - Improve the leases in metad cluster to be eventually consistent Signed-off-by: Jason Joo --- cmd/metad-ctl/cmds/parse.go | 11 ++++ cmd/metad-ctl/cmds/status.go | 3 +- cmd/metad-ctl/cmds/storage.go | 97 +++++++++++++++++++++++++++++++++ cmd/metad-ctl/cmds/update.go | 3 + cmd/metad-ctl/main.go | 10 +--- go.mod | 2 + go.sum | 2 + raftmeta/apply.go | 62 +++++++++++---------- raftmeta/cluster_leases.go | 95 ++++++++++++++++++++++++++++++++ raftmeta/cluster_leases_test.go | 37 +++++++++++++ raftmeta/meta_service.go | 28 +++++++--- raftmeta/node.go | 17 ++++-- 12 files changed, 315 insertions(+), 52 deletions(-) create mode 100644 cmd/metad-ctl/cmds/storage.go create mode 100644 raftmeta/cluster_leases.go create mode 100644 raftmeta/cluster_leases_test.go diff --git a/cmd/metad-ctl/cmds/parse.go b/cmd/metad-ctl/cmds/parse.go index bb1931a..adb79a1 100644 --- a/cmd/metad-ctl/cmds/parse.go +++ b/cmd/metad-ctl/cmds/parse.go @@ -7,6 +7,17 @@ import ( "strings" "github.com/angopher/chronus/raftmeta" + "github.com/urfave/cli/v2" +) + +var ( + FLAG_ADDR = &cli.StringFlag{ + Name: "metad", + Aliases: []string{"s"}, + Required: true, + Usage: "Node address in cluster, ip:port", + Destination: &MetadAddress, + } ) func parseNodeAddr(arg string) (string, error) { diff --git a/cmd/metad-ctl/cmds/status.go b/cmd/metad-ctl/cmds/status.go index a9688c7..8f3e450 100644 --- a/cmd/metad-ctl/cmds/status.go +++ b/cmd/metad-ctl/cmds/status.go @@ -4,7 +4,6 @@ import ( "encoding/json" "errors" "fmt" - "os" "sort" "github.com/angopher/chronus/cmd/metad-ctl/util" @@ -46,6 +45,7 @@ func StatusCommand() *cli.Command { Usage: "status of cluster", Description: "Get the cluster status", Action: clusterStatus, + Flags: []cli.Flag{FLAG_ADDR}, } } @@ -69,6 +69,5 @@ func clusterStatus(ctx *cli.Context) (err error) { return nil ERR: - fmt.Fprintln(os.Stderr, err.Error()) return err } diff --git a/cmd/metad-ctl/cmds/storage.go b/cmd/metad-ctl/cmds/storage.go new file mode 100644 index 0000000..d48231d --- /dev/null +++ b/cmd/metad-ctl/cmds/storage.go @@ -0,0 +1,97 @@ +package cmds + +import ( + "errors" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "sort" + + "github.com/angopher/chronus/cmd/metad-ctl/util" + "github.com/angopher/chronus/raftmeta" + "github.com/dgraph-io/badger/v2" + "github.com/dgraph-io/badger/v2/table" + "github.com/dustin/go-humanize" + "github.com/fatih/color" + "github.com/urfave/cli/v2" +) + +func dumpStatus1(resp *raftmeta.StatusClusterResp) { + color.Set(color.Bold) + color.Green("Cluster:\n") + fmt.Println("Leader:", resp.Leader) + fmt.Println("Term:", resp.Term) + fmt.Println("Committed:", resp.Commit) + fmt.Println("Applied:", resp.Applied) + fmt.Println() + + color.Set(color.Bold) + color.Yellow("Nodes:\n") + sort.Slice(resp.Nodes, func(i, j int) bool { + return resp.Nodes[i].ID < resp.Nodes[j].ID + }) + for _, n := range resp.Nodes { + fmt.Print(util.PadRight(fmt.Sprint(n.ID), 6)) + fmt.Print(" ", util.PadRight(n.Role, 15)) + fmt.Print(util.PadRight(n.Addr, 23)) + fmt.Print(util.PadRight(n.Progress, 15)) + fmt.Print(util.PadRight(fmt.Sprint(n.Match, "=>", n.Next), 20)) + if n.Vote > 0 { + fmt.Print("Vote(", n.Vote, ")") + } + fmt.Println() + } +} + +func StorageCommand() *cli.Command { + return &cli.Command{ + Name: "storage", + Usage: "View storage info", + Description: "Get the storage info", + Action: storageInfo, + } +} + +func storageInfo(ctx *cli.Context) (err error) { + if ctx.Args().Len() < 1 { + return errors.New("Usage: metad-ctl storage ") + } + + walDir := ctx.Args().First() + fmt.Println("Dump information of", walDir) + + fileinfos, err := ioutil.ReadDir(walDir) + if err != nil { + return err + } + fmt.Println("Collect files ...") + fileinfoByName := make(map[string]os.FileInfo) + for _, info := range fileinfos { + fileinfoByName[info.Name()] = info + } + + fmt.Println("Reading", badger.ManifestFilename, "...") + fp, err := os.Open(filepath.Join(walDir, badger.ManifestFilename)) + if err != nil { + return err + } + defer fp.Close() + + manifest, _, err := badger.ReplayManifestFile(fp) + fmt.Println() + color.Set(color.Bold) + color.Green("Disk space cost by level:") + for level, lm := range manifest.Levels { + sz := int64(0) + for id := range lm.Tables { + tableFile := table.IDToFilename(id) + if info, ok := fileinfoByName[tableFile]; ok { + sz += info.Size() + } + } + fmt.Print("Level ", level, ": ", humanize.Bytes(uint64(sz)), "\n") + } + + return nil +} diff --git a/cmd/metad-ctl/cmds/update.go b/cmd/metad-ctl/cmds/update.go index 1a3ef17..9a00618 100644 --- a/cmd/metad-ctl/cmds/update.go +++ b/cmd/metad-ctl/cmds/update.go @@ -17,6 +17,7 @@ func AddCommand() *cli.Command { Description: "Introdue new node to cluster should follow two phases operation:\n 1. Add node to configuration using `metad-ctl add`\n 2. Boot up new node with confugration up to date", ArgsUsage: " ", Action: clusterAdd, + Flags: []cli.Flag{FLAG_ADDR}, } } @@ -27,6 +28,7 @@ func UpdateCommand() *cli.Command { Description: "Update address of a node which is already in cluster should follow two phases operation:\n 1. Stop the node\n 2. Update address\n 3. Boot up node with confugration up to date", ArgsUsage: " ", Action: clusterUpdate, + Flags: []cli.Flag{FLAG_ADDR}, } } @@ -37,6 +39,7 @@ func RemoveCommand() *cli.Command { Description: "Remove specified node from config.", ArgsUsage: "", Action: clusterRemove, + Flags: []cli.Flag{FLAG_ADDR}, } } diff --git a/cmd/metad-ctl/main.go b/cmd/metad-ctl/main.go index 18a53cf..edaec7a 100644 --- a/cmd/metad-ctl/main.go +++ b/cmd/metad-ctl/main.go @@ -26,15 +26,7 @@ func main() { cmds.AddCommand(), cmds.UpdateCommand(), cmds.RemoveCommand(), - } - app.Flags = []cli.Flag{ - &cli.StringFlag{ - Name: "metad", - Aliases: []string{"s"}, - Required: true, - Usage: "Node address in cluster, ip:port", - Destination: &cmds.MetadAddress, - }, + cmds.StorageCommand(), } app.Run(os.Args) } diff --git a/go.mod b/go.mod index 19dcbc4..e88330c 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ require ( github.com/BurntSushi/toml v0.3.1 github.com/dgraph-io/badger/v2 v2.2007.2 github.com/dgraph-io/dgraph v1.2.7 + github.com/dustin/go-humanize v1.0.0 github.com/fatih/color v1.9.0 github.com/gogo/protobuf v1.3.1 github.com/golang/snappy v0.0.2 @@ -14,6 +15,7 @@ require ( github.com/jsternberg/zap-logfmt v1.2.0 github.com/klauspost/compress v1.11.1 // indirect github.com/klauspost/pgzip v1.2.5 // indirect + github.com/kr/pretty v0.2.0 // indirect github.com/pkg/errors v0.9.1 github.com/stretchr/testify v1.6.1 // test github.com/urfave/cli/v2 v2.2.0 diff --git a/go.sum b/go.sum index 4f14672..5c5f1d8 100644 --- a/go.sum +++ b/go.sum @@ -288,6 +288,8 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxv github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= +github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.0.0/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= diff --git a/raftmeta/apply.go b/raftmeta/apply.go index 2644e36..723bcc8 100644 --- a/raftmeta/apply.go +++ b/raftmeta/apply.go @@ -31,7 +31,7 @@ func (s *RaftNode) applyCommitted(proposal *internal.Proposal, index uint64) err var req CreateDatabaseReq err := json.Unmarshal(proposal.Data, &req) x.Check(err) - s.Logger.Info(fmt.Sprintf("apply create database %+v", req)) + s.SugaredLogger.Infof("apply create database %+v", req) db, err := s.MetaCli.CreateDatabase(req.Name) pctx.err = err if err == nil && pctx.retData != nil { @@ -43,19 +43,19 @@ func (s *RaftNode) applyCommitted(proposal *internal.Proposal, index uint64) err var req DropDatabaseReq err := json.Unmarshal(proposal.Data, &req) x.Check(err) - s.Logger.Debug(fmt.Sprintf("req %+v", req)) + s.SugaredLogger.Debugf("req %+v", req) return s.MetaCli.DropDatabase(req.Name) case internal.DropRetentionPolicy: var req DropRetentionPolicyReq err := json.Unmarshal(proposal.Data, &req) x.Check(err) - s.Logger.Debug(fmt.Sprintf("req %+v", req)) + s.SugaredLogger.Debugf("req %+v", req) return s.MetaCli.DropRetentionPolicy(req.Database, req.Policy) case internal.CreateShardGroup: var req CreateShardGroupReq err := json.Unmarshal(proposal.Data, &req) x.Check(err) - s.Logger.Debug(fmt.Sprintf("req %+v", req)) + s.SugaredLogger.Debugf("req %+v", req) sg, err := s.MetaCli.CreateShardGroup(req.Database, req.Policy, time.Unix(req.Timestamp, 0)) if err == nil && pctx.retData != nil { if sg != nil { @@ -69,7 +69,7 @@ func (s *RaftNode) applyCommitted(proposal *internal.Proposal, index uint64) err var req CreateDataNodeReq err := json.Unmarshal(proposal.Data, &req) x.Check(err) - s.Logger.Debug(fmt.Sprintf("req %+v", req)) + s.SugaredLogger.Debugf("req %+v", req) ni, err := s.MetaCli.CreateDataNode(req.HttpAddr, req.TcpAddr) if err == nil && pctx.retData != nil { x.AssertTrue(ni != nil) @@ -80,13 +80,13 @@ func (s *RaftNode) applyCommitted(proposal *internal.Proposal, index uint64) err var req DeleteDataNodeReq err := json.Unmarshal(proposal.Data, &req) x.Check(err) - s.Logger.Debug(fmt.Sprintf("req %+v", req)) + s.SugaredLogger.Debugf("req %+v", req) return s.MetaCli.DeleteDataNode(req.Id) case internal.CreateRetentionPolicy: var req CreateRetentionPolicyReq err := json.Unmarshal(proposal.Data, &req) x.Check(err) - s.Logger.Debug(fmt.Sprintf("req %+v", req)) + s.SugaredLogger.Debugf("req %+v", req) var duration *time.Duration if req.Rps.Duration > 0 { @@ -110,7 +110,7 @@ func (s *RaftNode) applyCommitted(proposal *internal.Proposal, index uint64) err var req UpdateRetentionPolicyReq err := json.Unmarshal(proposal.Data, &req) x.Check(err) - s.Logger.Debug(fmt.Sprintf("req %+v", req)) + s.SugaredLogger.Debugf("req %+v", req) var duration *time.Duration if req.Rps.Duration > 0 { @@ -139,7 +139,7 @@ func (s *RaftNode) applyCommitted(proposal *internal.Proposal, index uint64) err var req CreateDatabaseWithRetentionPolicyReq err := json.Unmarshal(proposal.Data, &req) x.Check(err) - s.Logger.Debug(fmt.Sprintf("req %+v", req)) + s.SugaredLogger.Debugf("req %+v", req) var duration *time.Duration if req.Rps.Duration > 0 { @@ -163,7 +163,7 @@ func (s *RaftNode) applyCommitted(proposal *internal.Proposal, index uint64) err var req CreateUserReq err := json.Unmarshal(proposal.Data, &req) x.Check(err) - s.Logger.Debug(fmt.Sprintf("req %+v", req)) + s.SugaredLogger.Debugf("req %+v", req) user, err := s.MetaCli.CreateUser(req.Name, req.Password, req.Admin) if err == nil && pctx.retData != nil { x.AssertTrue(user != nil) @@ -175,35 +175,35 @@ func (s *RaftNode) applyCommitted(proposal *internal.Proposal, index uint64) err var req DropUserReq err := json.Unmarshal(proposal.Data, &req) x.Check(err) - s.Logger.Debug(fmt.Sprintf("req %+v", req)) + s.SugaredLogger.Debugf("req %+v", req) return s.MetaCli.DropUser(req.Name) case internal.UpdateUser: var req UpdateUserReq err := json.Unmarshal(proposal.Data, &req) x.Check(err) - s.Logger.Debug(fmt.Sprintf("req %+v", req)) + s.SugaredLogger.Debugf("req %+v", req) return s.MetaCli.UpdateUser(req.Name, req.Password) case internal.SetPrivilege: var req SetPrivilegeReq err := json.Unmarshal(proposal.Data, &req) x.Check(err) - s.Logger.Debug(fmt.Sprintf("req %+v", req)) + s.SugaredLogger.Debugf("req %+v", req) return s.MetaCli.SetPrivilege(req.UserName, req.Database, req.Privilege) case internal.SetAdminPrivilege: var req SetAdminPrivilegeReq err := json.Unmarshal(proposal.Data, &req) x.Check(err) - s.Logger.Debug(fmt.Sprintf("req %+v", req)) + s.SugaredLogger.Debugf("req %+v", req) return s.MetaCli.SetAdminPrivilege(req.UserName, req.Admin) case internal.Authenticate: var req AuthenticateReq err := json.Unmarshal(proposal.Data, &req) x.Check(err) - s.Logger.Debug(fmt.Sprintf("req %+v", req)) + s.SugaredLogger.Debugf("req %+v", req) user, err := s.MetaCli.Authenticate(req.UserName, req.Password) if err == nil { x.AssertTrue(user != nil) @@ -215,28 +215,28 @@ func (s *RaftNode) applyCommitted(proposal *internal.Proposal, index uint64) err var req AddShardOwnerReq err := json.Unmarshal(proposal.Data, &req) x.Check(err) - s.Logger.Debug(fmt.Sprintf("add shard owner req %+v", req)) + s.SugaredLogger.Debugf("add shard owner req %+v", req) return s.MetaCli.AddShardOwner(req.ShardID, req.NodeID) case internal.RemoveShardOwner: var req RemoveShardOwnerReq err := json.Unmarshal(proposal.Data, &req) x.Check(err) - s.Logger.Debug(fmt.Sprintf("remove shard owner req %+v", req)) + s.SugaredLogger.Debugf("remove shard owner req %+v", req) return s.MetaCli.RemoveShardOwner(req.ShardID, req.NodeID) case internal.DropShard: var req DropShardReq err := json.Unmarshal(proposal.Data, &req) x.Check(err) - s.Logger.Debug(fmt.Sprintf("req %+v", req)) + s.SugaredLogger.Debugf("req %+v", req) return s.MetaCli.DropShard(req.Id) case internal.TruncateShardGroups: var req TruncateShardGroupsReq err := json.Unmarshal(proposal.Data, &req) x.Check(err) - s.Logger.Debug(fmt.Sprintf("req %+v", req)) + s.SugaredLogger.Debugf("req %+v", req) return s.MetaCli.TruncateShardGroups(req.Time) case internal.PruneShardGroups: @@ -246,50 +246,54 @@ func (s *RaftNode) applyCommitted(proposal *internal.Proposal, index uint64) err var req DeleteShardGroupReq err := json.Unmarshal(proposal.Data, &req) x.Check(err) - s.Logger.Debug(fmt.Sprintf("req %+v", req)) + s.SugaredLogger.Debugf("req %+v", req) return s.MetaCli.DeleteShardGroup(req.Database, req.Policy, req.Id) case internal.PrecreateShardGroups: var req PrecreateShardGroupsReq err := json.Unmarshal(proposal.Data, &req) x.Check(err) - s.Logger.Debug(fmt.Sprintf("req %+v", req)) + s.SugaredLogger.Debugf("req %+v", req) return s.MetaCli.PrecreateShardGroups(req.From, req.To) case internal.CreateContinuousQuery: var req CreateContinuousQueryReq err := json.Unmarshal(proposal.Data, &req) x.Check(err) - s.Logger.Debug(fmt.Sprintf("req %+v", req)) + s.SugaredLogger.Debugf("req %+v", req) return s.MetaCli.CreateContinuousQuery(req.Database, req.Name, req.Query) case internal.DropContinuousQuery: var req DropContinuousQueryReq err := json.Unmarshal(proposal.Data, &req) x.Check(err) - s.Logger.Debug(fmt.Sprintf("req %+v", req)) + s.SugaredLogger.Debugf("req %+v", req) return s.MetaCli.DropContinuousQuery(req.Database, req.Name) case internal.CreateSubscription: var req CreateSubscriptionReq err := json.Unmarshal(proposal.Data, &req) x.Check(err) - s.Logger.Debug(fmt.Sprintf("req %+v", req)) + s.SugaredLogger.Debugf("req %+v", req) return s.MetaCli.CreateSubscription(req.Database, req.Rp, req.Name, req.Mode, req.Destinations) case internal.DropSubscription: var req DropSubscriptionReq err := json.Unmarshal(proposal.Data, &req) x.Check(err) - s.Logger.Debug(fmt.Sprintf("req %+v", req)) + s.SugaredLogger.Debug("req %+v", req) return s.MetaCli.DropSubscription(req.Database, req.Rp, req.Name) case internal.AcquireLease: var req AcquireLeaseReq err := json.Unmarshal(proposal.Data, &req) x.Check(err) - s.Logger.Debug(fmt.Sprintf("req %+v", req)) - lease, err := s.leases.Acquire(req.Name, req.NodeId) + s.SugaredLogger.Debugf("req %+v", req) + if req.RequestTime == 0 { + // XXX compatible with old data format + req.RequestTime = time.Now().UnixNano() / 1e6 + } + lease, err := s.leases.Acquire(req.Name, req.NodeId, req.RequestTime) if err == nil && pctx.retData != nil { x.AssertTrue(lease != nil) *pctx.retData.(*meta.Lease) = *lease @@ -372,14 +376,14 @@ func (s *RaftNode) applyCommitted(proposal *internal.Proposal, index uint64) err var req FreezeDataNodeReq err := json.Unmarshal(proposal.Data, &req) x.Check(err) - s.Logger.Debug(fmt.Sprintf("req %+v", req)) + s.SugaredLogger.Debugf("req %+v", req) if req.Freeze { return s.MetaCli.FreezeDataNode(req.Id) } else { return s.MetaCli.UnfreezeDataNode(req.Id) } default: - return fmt.Errorf("Unkown msg type:%d", proposal.Type) + return fmt.Errorf("Unknown msg type:%d", proposal.Type) } return nil diff --git a/raftmeta/cluster_leases.go b/raftmeta/cluster_leases.go new file mode 100644 index 0000000..d579c63 --- /dev/null +++ b/raftmeta/cluster_leases.go @@ -0,0 +1,95 @@ +package raftmeta + +import ( + "errors" + "sync" + "time" + + "github.com/influxdata/influxdb/services/meta" + "go.uber.org/zap" +) + +var ( + ErrSkipExpired = errors.New("Skip an expired lease") + ErrAcquiredByOther = errors.New("Another node has the lease") +) + +// ClusterLeases is a concurrency-safe collection of leases keyed by name. +type ClusterLeases struct { + mu sync.Mutex + m map[string]*meta.Lease + d time.Duration + sugar *zap.SugaredLogger +} + +// NewLeases returns a new instance of Leases. +func NewClusterLeases(d time.Duration, logger *zap.Logger) *ClusterLeases { + return &ClusterLeases{ + m: make(map[string]*meta.Lease), + d: d, + sugar: logger.Sugar(), + } +} + +func (leases *ClusterLeases) Get(name string) *meta.Lease { + leases.mu.Lock() + defer leases.mu.Unlock() + + return leases.m[name] +} + +// CanAcquire returns whether it's necessary actually trying to acquire for now. +func (leases *ClusterLeases) ShouldTry(name string, nodeID uint64) bool { + leases.mu.Lock() + defer leases.mu.Unlock() + + now := time.Now() + l := leases.m[name] + if l != nil { + if now.After(l.Expiration) || l.Owner == nodeID { + return true + } + return false + } + + return true +} + +// Acquire acquires a lease with the given name for the given nodeID. +// If the lease doesn't exist or exists but is expired, a valid lease is returned. +// If nodeID already owns the named and unexpired lease, the lease expiration is extended. +// If a different node owns the lease, an error is returned. +func (leases *ClusterLeases) Acquire(name string, nodeID uint64, requestTime int64) (*meta.Lease, error) { + leases.mu.Lock() + defer leases.mu.Unlock() + + now := time.Now() + beginTime := time.Unix(requestTime/1000, (requestTime%1000)*1e6) + expireTime := beginTime.Add(leases.d) + + if now.After(expireTime) { + // skip + leases.sugar.Warnf("Skip expired lock: key=%s, node=%d", name, nodeID) + return nil, ErrSkipExpired + } + + l := leases.m[name] + if l != nil { + if now.After(l.Expiration) || l.Owner == nodeID { + l.Expiration = expireTime + l.Owner = nodeID + return l, nil + } + return l, ErrAcquiredByOther + } + + l = &meta.Lease{ + Name: name, + Expiration: expireTime, + Owner: nodeID, + } + + leases.m[name] = l + + return l, nil +} diff --git a/raftmeta/cluster_leases_test.go b/raftmeta/cluster_leases_test.go new file mode 100644 index 0000000..1805e3a --- /dev/null +++ b/raftmeta/cluster_leases_test.go @@ -0,0 +1,37 @@ +package raftmeta_test + +import ( + "testing" + "time" + + "github.com/angopher/chronus/raftmeta" + "github.com/stretchr/testify/assert" + "go.uber.org/zap" +) + +func TestLease(t *testing.T) { + leases := raftmeta.NewClusterLeases(time.Second, zap.NewNop()) + KEY1 := "k1" + NODE1 := uint64(1) + NODE2 := uint64(2) + + l, err := leases.Acquire(KEY1, NODE1, 0) + assert.Equal(t, raftmeta.ErrSkipExpired, err) + assert.Nil(t, l) + + l, err = leases.Acquire(KEY1, NODE1, time.Now().UnixNano()/1e6) + assert.Nil(t, err) + assert.NotNil(t, l) + + l, err = leases.Acquire(KEY1, NODE2, time.Now().UnixNano()/1e6) + assert.Equal(t, raftmeta.ErrAcquiredByOther, err) + assert.NotNil(t, l) + + assert.True(t, leases.ShouldTry(KEY1, NODE1)) + assert.False(t, leases.ShouldTry(KEY1, NODE2)) + + l = leases.Get(KEY1) + assert.NotNil(t, l) + assert.Equal(t, KEY1, l.Name) + assert.Equal(t, NODE1, l.Owner) +} diff --git a/raftmeta/meta_service.go b/raftmeta/meta_service.go index 1553621..3542bd2 100644 --- a/raftmeta/meta_service.go +++ b/raftmeta/meta_service.go @@ -1317,8 +1317,9 @@ func (s *MetaService) DropSubscription(w http.ResponseWriter, r *http.Request) { } type AcquireLeaseReq struct { - Name string - NodeId uint64 + Name string + NodeId uint64 + RequestTime int64 // timestamp in millis } type AcquireLeaseResp struct { @@ -1346,15 +1347,27 @@ func (s *MetaService) AcquireLease(w http.ResponseWriter, r *http.Request) { return } + // do pre-check before proposing + if !s.Node.ShouldTryAcquireLease(req.Name, req.NodeId) { + resp.RetMsg = ErrAcquiredByOther.Error() + return + } + + req.RequestTime = time.Now().UnixNano() / 1e6 + data, _ = json.Marshal(&req) + lease := &meta.Lease{} err = s.ProposeAndWait(internal.AcquireLease, data, lease) if err != nil { resp.RetMsg = err.Error() if !strings.Contains(err.Error(), "another node has the lease") { - s.Logger.Error("AcquireLease fail", - zap.String("Name", req.Name), - zap.Uint64("NodeId", req.NodeId), - zap.Error(err)) + s.Logger.Error( + fmt.Sprintf( + "AcquireLease fail, name=%s, node=%d", + req.Name, req.NodeId, + ), + zap.Error(err), + ) } return } @@ -1364,7 +1377,8 @@ func (s *MetaService) AcquireLease(w http.ResponseWriter, r *http.Request) { resp.RetMsg = "ok" s.Logger.Debug("AcquireLease ok", zap.String("Name", req.Name), - zap.Uint64("NodeId", req.NodeId)) + zap.Uint64("NodeId", req.NodeId), + ) } type DataResp struct { diff --git a/raftmeta/node.go b/raftmeta/node.go index 6ed4711..4e2f9a9 100644 --- a/raftmeta/node.go +++ b/raftmeta/node.go @@ -125,7 +125,7 @@ type RaftNode struct { MetaCli MetaClient //用于Continuous query - leases *meta.Leases + leases *ClusterLeases //raft集群内部配置状态 RaftConfState *raftpb.ConfState @@ -176,7 +176,8 @@ type RaftNode struct { //only for test ApplyCallBack func(proposal *internal.Proposal, index uint64) - Logger *zap.Logger + Logger *zap.Logger + SugaredLogger *zap.SugaredLogger } func NewRaftNode(config Config, logger *zap.Logger) *RaftNode { @@ -219,11 +220,13 @@ func NewRaftNode(config Config, logger *zap.Logger) *RaftNode { //storage := raft.NewMemoryStorage() storage := raftwal.Init(walStore, c.ID, 0) c.Storage = storage + selfLogger := logger.With(zap.String("raftmeta", "RaftNode")) return &RaftNode{ - leases: meta.NewLeases(meta.DefaultLeaseDuration), + leases: NewClusterLeases(meta.DefaultLeaseDuration, selfLogger), ID: c.ID, RaftConfig: c, - Logger: logger.With(zap.String("raftmeta", "RaftNode")), + Logger: selfLogger, + SugaredLogger: selfLogger.Sugar(), Config: config, RaftCtx: rc, Storage: storage, @@ -389,7 +392,7 @@ func (s *RaftNode) reclaimDiskSpace() { for { select { case <-ticker.C: - s.walStore.RunValueLogGC(0.1) + s.walStore.RunValueLogGC(0.5) case <-s.Done: return } @@ -752,3 +755,7 @@ func (s *RaftNode) PastLife() (idx uint64, restart bool, rerr error) { } return } + +func (s *RaftNode) ShouldTryAcquireLease(name string, nodeId uint64) bool { + return s.leases.ShouldTry(name, nodeId) +} From fb734ef0d43ea60bc0a345161cdc7892c56adcde Mon Sep 17 00:00:00 2001 From: Jason Joo Date: Thu, 15 Oct 2020 15:09:49 +0800 Subject: [PATCH 31/46] Polish logging Signed-off-by: Jason Joo --- coordinator/cluster_meta_client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/coordinator/cluster_meta_client.go b/coordinator/cluster_meta_client.go index ec29e64..bf6f5d8 100644 --- a/coordinator/cluster_meta_client.go +++ b/coordinator/cluster_meta_client.go @@ -92,7 +92,7 @@ func (me *ClusterMetaClient) syncLoop() { // normal if printLimiter.Allow() { // one log in 10 seconds - me.Logger.Info(fmt.Sprintf("index=%d local_index=%d", index, me.cache.Data().Index)) + me.Logger.Debug(fmt.Sprintf("index=%d local_index=%d", index, me.cache.Data().Index)) } } } From 34b377edeacbea7135d701988d127a56eb076d8a Mon Sep 17 00:00:00 2001 From: Jason Joo Date: Thu, 15 Oct 2020 16:54:14 +0800 Subject: [PATCH 32/46] Change default configuration of influxd Signed-off-by: Jason Joo --- cmd/influxd/run/config.go | 20 +++----------------- 1 file changed, 3 insertions(+), 17 deletions(-) diff --git a/cmd/influxd/run/config.go b/cmd/influxd/run/config.go index 53f32ad..da4120d 100644 --- a/cmd/influxd/run/config.go +++ b/cmd/influxd/run/config.go @@ -4,9 +4,6 @@ import ( "fmt" "io/ioutil" "log" - "os" - "os/user" - "path/filepath" "regexp" "strings" "time" @@ -110,20 +107,9 @@ func NewConfig() *Config { func NewDemoConfig() (*Config, error) { c := NewConfig() - var homeDir string - // By default, store meta and data files in current users home directory - u, err := user.Current() - if err == nil { - homeDir = u.HomeDir - } else if os.Getenv("HOME") != "" { - homeDir = os.Getenv("HOME") - } else { - return nil, fmt.Errorf("failed to determine current user for storage") - } - - c.Meta.Dir = filepath.Join(homeDir, ".influxdb/meta") - c.Data.Dir = filepath.Join(homeDir, ".influxdb/data") - c.Data.WALDir = filepath.Join(homeDir, ".influxdb/wal") + c.Meta.Dir = "./meta" + c.Data.Dir = "./data" + c.Data.WALDir = "./wal" return c, nil } From 28130a57dcc929b665fe2b902a3d404f76e887c6 Mon Sep 17 00:00:00 2001 From: Jason Joo Date: Thu, 15 Oct 2020 17:12:45 +0800 Subject: [PATCH 33/46] Adjust documentation for latest command tool Signed-off-by: Jason Joo --- Meta_Cluster_Maintenance.md | 2 +- README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Meta_Cluster_Maintenance.md b/Meta_Cluster_Maintenance.md index 72767cb..06a6a5e 100644 --- a/Meta_Cluster_Maintenance.md +++ b/Meta_Cluster_Maintenance.md @@ -3,7 +3,7 @@ ## Get Status of Cluster ```shell -metad-ctl -s ip:port status +metad-ctl status -s ip:port ``` Sample output as: diff --git a/README.md b/README.md index 6aa295f..b19cdf9 100644 --- a/README.md +++ b/README.md @@ -90,7 +90,7 @@ Then a single node meta cluster is done. Using: ```shell -metad-ctl -s ip:port status +metad-ctl status -s ip:port ``` You can specify any **alive** address in cluster by `-s ip:port` and it's From 27ecfc36595aff81401429f3ee7ecee7a9c9d8d7 Mon Sep 17 00:00:00 2001 From: Jason Joo Date: Tue, 27 Oct 2020 17:33:29 +0800 Subject: [PATCH 34/46] Fix inconsistent issue when creating/updating user --- cmd/metad/main.go | 2 +- coordinator/cluster_meta_client.go | 10 ++- raftmeta/apply.go | 65 ++++++++++--------- raftmeta/meta_service.go | 35 ++++++---- raftmeta/{meta_client.go => meta_store.go} | 6 +- raftmeta/node.go | 4 +- raftmeta/raft_test.go | 2 +- services/meta/{client.go => store.go} | 31 +++------ .../meta/{client_test.go => store_test.go} | 12 +++- 9 files changed, 84 insertions(+), 83 deletions(-) rename raftmeta/{meta_client.go => meta_store.go} (92%) rename services/meta/{client.go => store.go} (96%) rename services/meta/{client_test.go => store_test.go} (98%) diff --git a/cmd/metad/main.go b/cmd/metad/main.go index c700376..e31be32 100644 --- a/cmd/metad/main.go +++ b/cmd/metad/main.go @@ -60,7 +60,7 @@ func main() { x.Check(err) node := raftmeta.NewRaftNode(config, log) - node.MetaCli = metaCli + node.MetaStore = metaCli // dump only if *dumpFile != "" { diff --git a/coordinator/cluster_meta_client.go b/coordinator/cluster_meta_client.go index bf6f5d8..e2e3088 100644 --- a/coordinator/cluster_meta_client.go +++ b/coordinator/cluster_meta_client.go @@ -223,11 +223,9 @@ func (me *ClusterMetaClient) CreateSubscription(database, rp, name, mode string, return me.cache.CreateSubscription(database, rp, name, mode, destinations) } -func (me *ClusterMetaClient) CreateUser(name, password string, admin bool) (meta.User, error) { - if u, err := me.metaCli.CreateUser(name, password, admin); err != nil { - return u, err - } - return me.cache.CreateUser(name, password, admin) +func (me *ClusterMetaClient) CreateUser(name, password string, admin bool) (u meta.User, err error) { + u, err = me.metaCli.CreateUser(name, password, admin) + return } func (me *ClusterMetaClient) Databases() []meta.DatabaseInfo { @@ -337,7 +335,7 @@ func (me *ClusterMetaClient) UpdateUser(name, password string) error { if err := me.metaCli.UpdateUser(name, password); err != nil { return err } - return me.cache.UpdateUser(name, password) + return nil } func (me *ClusterMetaClient) UserPrivilege(username, database string) (*influxql.Privilege, error) { diff --git a/raftmeta/apply.go b/raftmeta/apply.go index 723bcc8..a4b672f 100644 --- a/raftmeta/apply.go +++ b/raftmeta/apply.go @@ -32,7 +32,7 @@ func (s *RaftNode) applyCommitted(proposal *internal.Proposal, index uint64) err err := json.Unmarshal(proposal.Data, &req) x.Check(err) s.SugaredLogger.Infof("apply create database %+v", req) - db, err := s.MetaCli.CreateDatabase(req.Name) + db, err := s.MetaStore.CreateDatabase(req.Name) pctx.err = err if err == nil && pctx.retData != nil { x.AssertTrue(db != nil) @@ -44,19 +44,19 @@ func (s *RaftNode) applyCommitted(proposal *internal.Proposal, index uint64) err err := json.Unmarshal(proposal.Data, &req) x.Check(err) s.SugaredLogger.Debugf("req %+v", req) - return s.MetaCli.DropDatabase(req.Name) + return s.MetaStore.DropDatabase(req.Name) case internal.DropRetentionPolicy: var req DropRetentionPolicyReq err := json.Unmarshal(proposal.Data, &req) x.Check(err) s.SugaredLogger.Debugf("req %+v", req) - return s.MetaCli.DropRetentionPolicy(req.Database, req.Policy) + return s.MetaStore.DropRetentionPolicy(req.Database, req.Policy) case internal.CreateShardGroup: var req CreateShardGroupReq err := json.Unmarshal(proposal.Data, &req) x.Check(err) s.SugaredLogger.Debugf("req %+v", req) - sg, err := s.MetaCli.CreateShardGroup(req.Database, req.Policy, time.Unix(req.Timestamp, 0)) + sg, err := s.MetaStore.CreateShardGroup(req.Database, req.Policy, time.Unix(req.Timestamp, 0)) if err == nil && pctx.retData != nil { if sg != nil { *pctx.retData.(*meta.ShardGroupInfo) = *sg @@ -70,7 +70,7 @@ func (s *RaftNode) applyCommitted(proposal *internal.Proposal, index uint64) err err := json.Unmarshal(proposal.Data, &req) x.Check(err) s.SugaredLogger.Debugf("req %+v", req) - ni, err := s.MetaCli.CreateDataNode(req.HttpAddr, req.TcpAddr) + ni, err := s.MetaStore.CreateDataNode(req.HttpAddr, req.TcpAddr) if err == nil && pctx.retData != nil { x.AssertTrue(ni != nil) *pctx.retData.(*meta.NodeInfo) = *ni @@ -81,7 +81,7 @@ func (s *RaftNode) applyCommitted(proposal *internal.Proposal, index uint64) err err := json.Unmarshal(proposal.Data, &req) x.Check(err) s.SugaredLogger.Debugf("req %+v", req) - return s.MetaCli.DeleteDataNode(req.Id) + return s.MetaStore.DeleteDataNode(req.Id) case internal.CreateRetentionPolicy: var req CreateRetentionPolicyReq err := json.Unmarshal(proposal.Data, &req) @@ -99,7 +99,7 @@ func (s *RaftNode) applyCommitted(proposal *internal.Proposal, index uint64) err Duration: duration, ShardGroupDuration: req.Rps.ShardGroupDuration, } - rpi, err := s.MetaCli.CreateRetentionPolicy(req.Database, &spec, req.MakeDefault) + rpi, err := s.MetaStore.CreateRetentionPolicy(req.Database, &spec, req.MakeDefault) if err == nil && pctx.retData != nil { x.AssertTrue(rpi != nil) *pctx.retData.(*meta.RetentionPolicyInfo) = *rpi @@ -133,7 +133,7 @@ func (s *RaftNode) applyCommitted(proposal *internal.Proposal, index uint64) err Duration: duration, ShardGroupDuration: sduration, } - return s.MetaCli.UpdateRetentionPolicy(req.Database, req.Name, &up, req.MakeDefault) + return s.MetaStore.UpdateRetentionPolicy(req.Database, req.Name, &up, req.MakeDefault) case internal.CreateDatabaseWithRetentionPolicy: var req CreateDatabaseWithRetentionPolicyReq @@ -152,7 +152,7 @@ func (s *RaftNode) applyCommitted(proposal *internal.Proposal, index uint64) err Duration: duration, ShardGroupDuration: req.Rps.ShardGroupDuration, } - db, err := s.MetaCli.CreateDatabaseWithRetentionPolicy(req.Name, &spec) + db, err := s.MetaStore.CreateDatabaseWithRetentionPolicy(req.Name, &spec) if err == nil && pctx.retData != nil { x.AssertTrue(db != nil) *pctx.retData.(*meta.DatabaseInfo) = *db @@ -164,7 +164,7 @@ func (s *RaftNode) applyCommitted(proposal *internal.Proposal, index uint64) err err := json.Unmarshal(proposal.Data, &req) x.Check(err) s.SugaredLogger.Debugf("req %+v", req) - user, err := s.MetaCli.CreateUser(req.Name, req.Password, req.Admin) + user, err := s.MetaStore.CreateUser(req.Name, req.Password, req.Admin) if err == nil && pctx.retData != nil { x.AssertTrue(user != nil) *pctx.retData.(*meta.UserInfo) = *(user.(*meta.UserInfo)) @@ -176,36 +176,37 @@ func (s *RaftNode) applyCommitted(proposal *internal.Proposal, index uint64) err err := json.Unmarshal(proposal.Data, &req) x.Check(err) s.SugaredLogger.Debugf("req %+v", req) - return s.MetaCli.DropUser(req.Name) + return s.MetaStore.DropUser(req.Name) case internal.UpdateUser: var req UpdateUserReq err := json.Unmarshal(proposal.Data, &req) x.Check(err) s.SugaredLogger.Debugf("req %+v", req) - return s.MetaCli.UpdateUser(req.Name, req.Password) + // XXX password should be hashed before to keep consistent between nodes + return s.MetaStore.UpdateUser(req.Name, req.Password) case internal.SetPrivilege: var req SetPrivilegeReq err := json.Unmarshal(proposal.Data, &req) x.Check(err) s.SugaredLogger.Debugf("req %+v", req) - return s.MetaCli.SetPrivilege(req.UserName, req.Database, req.Privilege) + return s.MetaStore.SetPrivilege(req.UserName, req.Database, req.Privilege) case internal.SetAdminPrivilege: var req SetAdminPrivilegeReq err := json.Unmarshal(proposal.Data, &req) x.Check(err) s.SugaredLogger.Debugf("req %+v", req) - return s.MetaCli.SetAdminPrivilege(req.UserName, req.Admin) + return s.MetaStore.SetAdminPrivilege(req.UserName, req.Admin) case internal.Authenticate: var req AuthenticateReq err := json.Unmarshal(proposal.Data, &req) x.Check(err) s.SugaredLogger.Debugf("req %+v", req) - user, err := s.MetaCli.Authenticate(req.UserName, req.Password) - if err == nil { + user, err := s.MetaStore.Authenticate(req.UserName, req.Password) + if err == nil && pctx != nil && pctx.retData != nil { x.AssertTrue(user != nil) *pctx.retData.(*meta.UserInfo) = *(user.(*meta.UserInfo)) } @@ -216,73 +217,73 @@ func (s *RaftNode) applyCommitted(proposal *internal.Proposal, index uint64) err err := json.Unmarshal(proposal.Data, &req) x.Check(err) s.SugaredLogger.Debugf("add shard owner req %+v", req) - return s.MetaCli.AddShardOwner(req.ShardID, req.NodeID) + return s.MetaStore.AddShardOwner(req.ShardID, req.NodeID) case internal.RemoveShardOwner: var req RemoveShardOwnerReq err := json.Unmarshal(proposal.Data, &req) x.Check(err) s.SugaredLogger.Debugf("remove shard owner req %+v", req) - return s.MetaCli.RemoveShardOwner(req.ShardID, req.NodeID) + return s.MetaStore.RemoveShardOwner(req.ShardID, req.NodeID) case internal.DropShard: var req DropShardReq err := json.Unmarshal(proposal.Data, &req) x.Check(err) s.SugaredLogger.Debugf("req %+v", req) - return s.MetaCli.DropShard(req.Id) + return s.MetaStore.DropShard(req.Id) case internal.TruncateShardGroups: var req TruncateShardGroupsReq err := json.Unmarshal(proposal.Data, &req) x.Check(err) s.SugaredLogger.Debugf("req %+v", req) - return s.MetaCli.TruncateShardGroups(req.Time) + return s.MetaStore.TruncateShardGroups(req.Time) case internal.PruneShardGroups: - return s.MetaCli.PruneShardGroups() + return s.MetaStore.PruneShardGroups() case internal.DeleteShardGroup: var req DeleteShardGroupReq err := json.Unmarshal(proposal.Data, &req) x.Check(err) s.SugaredLogger.Debugf("req %+v", req) - return s.MetaCli.DeleteShardGroup(req.Database, req.Policy, req.Id) + return s.MetaStore.DeleteShardGroup(req.Database, req.Policy, req.Id) case internal.PrecreateShardGroups: var req PrecreateShardGroupsReq err := json.Unmarshal(proposal.Data, &req) x.Check(err) s.SugaredLogger.Debugf("req %+v", req) - return s.MetaCli.PrecreateShardGroups(req.From, req.To) + return s.MetaStore.PrecreateShardGroups(req.From, req.To) case internal.CreateContinuousQuery: var req CreateContinuousQueryReq err := json.Unmarshal(proposal.Data, &req) x.Check(err) s.SugaredLogger.Debugf("req %+v", req) - return s.MetaCli.CreateContinuousQuery(req.Database, req.Name, req.Query) + return s.MetaStore.CreateContinuousQuery(req.Database, req.Name, req.Query) case internal.DropContinuousQuery: var req DropContinuousQueryReq err := json.Unmarshal(proposal.Data, &req) x.Check(err) s.SugaredLogger.Debugf("req %+v", req) - return s.MetaCli.DropContinuousQuery(req.Database, req.Name) + return s.MetaStore.DropContinuousQuery(req.Database, req.Name) case internal.CreateSubscription: var req CreateSubscriptionReq err := json.Unmarshal(proposal.Data, &req) x.Check(err) s.SugaredLogger.Debugf("req %+v", req) - return s.MetaCli.CreateSubscription(req.Database, req.Rp, req.Name, req.Mode, req.Destinations) + return s.MetaStore.CreateSubscription(req.Database, req.Rp, req.Name, req.Mode, req.Destinations) case internal.DropSubscription: var req DropSubscriptionReq err := json.Unmarshal(proposal.Data, &req) x.Check(err) s.SugaredLogger.Debug("req %+v", req) - return s.MetaCli.DropSubscription(req.Database, req.Rp, req.Name) + return s.MetaStore.DropSubscription(req.Database, req.Rp, req.Name) case internal.AcquireLease: var req AcquireLeaseReq @@ -301,7 +302,7 @@ func (s *RaftNode) applyCommitted(proposal *internal.Proposal, index uint64) err return err case internal.SnapShot: - md, err := s.MetaCli.MarshalBinary() + md, err := s.MetaStore.MarshalBinary() x.Check(err) var sndata internal.SnapshotData sndata.Data = md @@ -316,7 +317,7 @@ func (s *RaftNode) applyCommitted(proposal *internal.Proposal, index uint64) err case internal.CreateChecksumMsg: //TODO:optimize, reduce block time start := time.Now() - mcd := s.MetaCli.Data() + mcd := s.MetaStore.Data() //消除DeleteAt和TruncatedAt对checksum的影响 for i := range mcd.Databases { @@ -369,7 +370,7 @@ func (s *RaftNode) applyCommitted(proposal *internal.Proposal, index uint64) err s.Logger.Info("checksum", zap.Uint64("index", req.Index), zap.String("checksum", s.lastChecksum.checksum)) x.AssertTruef(s.lastChecksum.checksum == req.Checksum, "verify checksum fail, local %s != %s, data=%+v", s.lastChecksum.checksum, req.Checksum, - s.MetaCli.Data(), + s.MetaStore.Data(), ) s.Logger.Info(fmt.Sprintf("verify checksum success. costs %s", time.Now().Sub(start))) case internal.FreezeDataNode: @@ -378,9 +379,9 @@ func (s *RaftNode) applyCommitted(proposal *internal.Proposal, index uint64) err x.Check(err) s.SugaredLogger.Debugf("req %+v", req) if req.Freeze { - return s.MetaCli.FreezeDataNode(req.Id) + return s.MetaStore.FreezeDataNode(req.Id) } else { - return s.MetaCli.UnfreezeDataNode(req.Id) + return s.MetaStore.UnfreezeDataNode(req.Id) } default: return fmt.Errorf("Unknown msg type:%d", proposal.Type) diff --git a/raftmeta/meta_service.go b/raftmeta/meta_service.go index 3542bd2..490900a 100644 --- a/raftmeta/meta_service.go +++ b/raftmeta/meta_service.go @@ -16,6 +16,7 @@ import ( "github.com/influxdata/influxdb/services/meta" "github.com/influxdata/influxql" "go.uber.org/zap" + "golang.org/x/crypto/bcrypt" ) type CommonResp struct { @@ -587,6 +588,15 @@ func (s *MetaService) CreateUser(w http.ResponseWriter, r *http.Request) { s.Logger.Error("CreateUser fail", zap.Error(err)) return } + // regenerate data due to pre-hashed password + hash, err := bcrypt.GenerateFromPassword([]byte(req.Password), bcrypt.DefaultCost) + if err != nil { + resp.RetMsg = err.Error() + s.Logger.Error("Hash user password fail", zap.Error(err)) + return + } + req.Password = string(hash) + data, _ = json.Marshal(&req) user := &meta.UserInfo{} err = s.ProposeAndWait(internal.CreateUser, data, user) @@ -594,7 +604,6 @@ func (s *MetaService) CreateUser(w http.ResponseWriter, r *http.Request) { resp.RetMsg = err.Error() s.Logger.Error("CreateUser fail", zap.String("Name", req.Name), - zap.String("Password", req.Password), zap.Bool("Admin", req.Admin), zap.Error(err)) return @@ -605,7 +614,6 @@ func (s *MetaService) CreateUser(w http.ResponseWriter, r *http.Request) { resp.RetMsg = "ok" s.Logger.Info("CreateUser ok", zap.String("Name", req.Name), - zap.String("Password", req.Password), zap.Bool("Admin", req.Admin)) } @@ -675,12 +683,20 @@ func (s *MetaService) UpdateUser(w http.ResponseWriter, r *http.Request) { s.Logger.Error("UpdateUser fail", zap.Error(err)) return } + // regenerate data due to pre-hashed password + hash, err := bcrypt.GenerateFromPassword([]byte(req.Password), bcrypt.DefaultCost) + if err != nil { + resp.RetMsg = err.Error() + s.Logger.Error("Hash user password fail", zap.Error(err)) + return + } + req.Password = string(hash) + data, _ = json.Marshal(&req) err = s.ProposeAndWait(internal.UpdateUser, data, nil) if err != nil { s.Logger.Error("UpdateUser fail", zap.String("Name", req.Name), - zap.String("Password", req.Password), zap.Error(err)) resp.RetMsg = err.Error() return @@ -689,8 +705,7 @@ func (s *MetaService) UpdateUser(w http.ResponseWriter, r *http.Request) { resp.RetCode = 0 resp.RetMsg = "ok" s.Logger.Info("UpdateUser ok", - zap.String("Name", req.Name), - zap.String("Password", req.Password)) + zap.String("Name", req.Name)) } type SetPrivilegeReq struct { @@ -812,24 +827,18 @@ func (s *MetaService) Authenticate(w http.ResponseWriter, r *http.Request) { s.Logger.Error("Authenticate fail", zap.Error(err)) return } - - user := &meta.UserInfo{} - err = s.ProposeAndWait(internal.Authenticate, data, user) + u, err := s.Node.MetaStore.Authenticate(req.UserName, req.Password) if err != nil { s.Logger.Error("Authenticate fail", zap.String("UserName", req.UserName), - zap.String("Password", req.Password), zap.Error(err)) resp.RetMsg = err.Error() return } - resp.UserInfo = *(user) + resp.UserInfo = *(u.(*meta.UserInfo)) resp.RetCode = 0 resp.RetMsg = "ok" - s.Logger.Info("Authenticate ok", - zap.String("UserName", req.UserName), - zap.String("Password", req.Password)) } type AddShardOwnerReq struct { diff --git a/raftmeta/meta_client.go b/raftmeta/meta_store.go similarity index 92% rename from raftmeta/meta_client.go rename to raftmeta/meta_store.go index c4776df..d50d9c5 100644 --- a/raftmeta/meta_client.go +++ b/raftmeta/meta_store.go @@ -8,7 +8,7 @@ import ( "github.com/influxdata/influxql" ) -type MetaClient interface { +type MetaStore interface { MarshalBinary() ([]byte, error) ReplaceData(data *imeta.Data) error Data() imeta.Data @@ -18,7 +18,7 @@ type MetaClient interface { CreateRetentionPolicy(database string, spec *meta.RetentionPolicySpec, makeDefault bool) (*meta.RetentionPolicyInfo, error) CreateShardGroup(database, policy string, timestamp time.Time) (*meta.ShardGroupInfo, error) CreateSubscription(database, rp, name, mode string, destinations []string) error - CreateUser(name, password string, admin bool) (meta.User, error) + CreateUser(name, hashedPassword string, admin bool) (meta.User, error) CreateDataNode(httpAddr, tcpAddr string) (*meta.NodeInfo, error) DeleteDataNode(id uint64) error IsDataNodeFreezed(id uint64) bool @@ -41,5 +41,5 @@ type MetaClient interface { SetPrivilege(username, database string, p influxql.Privilege) error TruncateShardGroups(t time.Time) error UpdateRetentionPolicy(database, name string, rpu *meta.RetentionPolicyUpdate, makeDefault bool) error - UpdateUser(name, password string) error + UpdateUser(name, hashedPassword string) error } diff --git a/raftmeta/node.go b/raftmeta/node.go index 4e2f9a9..476b7fe 100644 --- a/raftmeta/node.go +++ b/raftmeta/node.go @@ -123,7 +123,7 @@ type RaftNode struct { ID uint64 Node raft.Node - MetaCli MetaClient + MetaStore MetaStore //用于Continuous query leases *ClusterLeases @@ -373,7 +373,7 @@ func (s *RaftNode) restoreFromSnapshot() bool { err = metaData.UnmarshalBinary(sndata.Data) x.Checkf(err, "meta data UnmarshalBinary fail") - err = s.MetaCli.ReplaceData(metaData) + err = s.MetaStore.ReplaceData(metaData) x.Checkf(err, "meta cli ReplaceData fail") return true diff --git a/raftmeta/raft_test.go b/raftmeta/raft_test.go index 177f8e2..e89fe01 100644 --- a/raftmeta/raft_test.go +++ b/raftmeta/raft_test.go @@ -127,7 +127,7 @@ func newService(config raftmeta.Config, t *fakeTransport, cb func(proposal *inte log := logger.New(os.Stderr) node := raftmeta.NewRaftNode(config, log) - node.MetaCli = metaCli + node.MetaStore = metaCli node.ApplyCallBack = cb node.Transport = t diff --git a/services/meta/client.go b/services/meta/store.go similarity index 96% rename from services/meta/client.go rename to services/meta/store.go index ecad42b..e8659d8 100644 --- a/services/meta/client.go +++ b/services/meta/store.go @@ -464,10 +464,6 @@ func (c *Client) user(name string) (meta.User, error) { return nil, meta.ErrUserNotFound } -// bcryptCost is the cost associated with generating password with bcrypt. -// This setting is lowered during testing to improve test suite performance. -var bcryptCost = bcrypt.DefaultCost - // hashWithSalt returns a salted hash of password using salt. func (c *Client) hashWithSalt(salt []byte, password string) []byte { hasher := sha256.New() @@ -487,7 +483,7 @@ func (c *Client) saltedHash(password string) (salt, hash []byte, err error) { } // CreateUser adds a user with the given name and password and admin status. -func (c *Client) CreateUser(name, password string, admin bool) (meta.User, error) { +func (c *Client) CreateUser(name, hashedPassword string, admin bool) (meta.User, error) { c.mu.Lock() defer c.mu.Unlock() @@ -495,19 +491,14 @@ func (c *Client) CreateUser(name, password string, admin bool) (meta.User, error // See if the user already exists. if u, err := c.user(name); err != nil && u != nil { - if err := bcrypt.CompareHashAndPassword([]byte(u.(*meta.UserInfo).Hash), []byte(password)); err != nil || u.(*meta.UserInfo).Admin != admin { + info := u.(*meta.UserInfo) + if info.Hash != hashedPassword || info.Admin != admin { return nil, meta.ErrUserExists } return u, nil } - // Hash the password before serializing it. - hash, err := bcrypt.GenerateFromPassword([]byte(password), bcryptCost) - if err != nil { - return nil, err - } - - if err := data.CreateUser(name, string(hash), admin); err != nil { + if err := data.CreateUser(name, hashedPassword, admin); err != nil { return nil, err } @@ -519,23 +510,17 @@ func (c *Client) CreateUser(name, password string, admin bool) (meta.User, error } // UpdateUser updates the password of an existing user. -func (c *Client) UpdateUser(name, password string) error { +func (c *Client) UpdateUser(name, hashedPassword string) error { c.mu.Lock() defer c.mu.Unlock() data := c.cacheData.Clone() - // Hash the password before serializing it. - hash, err := bcrypt.GenerateFromPassword([]byte(password), bcryptCost) - if err != nil { + if err := data.UpdateUser(name, hashedPassword); err != nil { return err } - if err := data.UpdateUser(name, string(hash)); err != nil { - return err - } - - delete(c.authCache, name) + defer delete(c.authCache, name) return c.commit(data) } @@ -551,6 +536,8 @@ func (c *Client) DropUser(name string) error { return err } + defer delete(c.authCache, name) + if err := c.commit(data); err != nil { return err } diff --git a/services/meta/client_test.go b/services/meta/store_test.go similarity index 98% rename from services/meta/client_test.go rename to services/meta/store_test.go index 7dc0b65..09fb5dd 100644 --- a/services/meta/client_test.go +++ b/services/meta/store_test.go @@ -14,6 +14,7 @@ import ( "github.com/influxdata/influxdb" "github.com/influxdata/influxdb/services/meta" "github.com/influxdata/influxql" + "golang.org/x/crypto/bcrypt" ) func TestMetaClient_CreateDatabaseOnly(t *testing.T) { @@ -575,6 +576,11 @@ func TestMetaClient_DropRetentionPolicy(t *testing.T) { } } +func hashPassword(password string) string { + hash, _ := bcrypt.GenerateFromPassword([]byte(password), bcrypt.DefaultCost) + return string(hash) +} + func TestMetaClient_CreateUser(t *testing.T) { t.Parallel() @@ -583,12 +589,12 @@ func TestMetaClient_CreateUser(t *testing.T) { defer c.Close() // Create an admin user - if _, err := c.CreateUser("fred", "supersecure", true); err != nil { + if _, err := c.CreateUser("fred", hashPassword("supersecure"), true); err != nil { t.Fatal(err) } // Create a non-admin user - if _, err := c.CreateUser("wilma", "password", false); err != nil { + if _, err := c.CreateUser("wilma", hashPassword("password"), false); err != nil { t.Fatal(err) } @@ -621,7 +627,7 @@ func TestMetaClient_CreateUser(t *testing.T) { } // Change password should succeed. - if err := c.UpdateUser("fred", "moresupersecure"); err != nil { + if err := c.UpdateUser("fred", hashPassword("moresupersecure")); err != nil { t.Fatal(err) } From bac19763e720018cf5a653a21ef3920c62bd83ff Mon Sep 17 00:00:00 2001 From: Jason Joo Date: Mon, 2 Nov 2020 13:30:13 +0800 Subject: [PATCH 35/46] Add connection pools monitoring through logs Signed-off-by: Jason Joo --- cmd/influxd/run/server.go | 41 +++++- coordinator/client_pool.go | 91 +++++++++----- coordinator/remote_executor.go | 61 ++++----- coordinator/shard_writer.go | 4 + x/pool.go | 223 +++++++++++++++++++++------------ x/pool_test.go | 131 ++++++++++++++----- 6 files changed, 376 insertions(+), 175 deletions(-) diff --git a/cmd/influxd/run/server.go b/cmd/influxd/run/server.go index 481daeb..fe374a0 100644 --- a/cmd/influxd/run/server.go +++ b/cmd/influxd/run/server.go @@ -74,7 +74,8 @@ type Server struct { BindAddress string Listener net.Listener - Logger *zap.Logger + Logger *zap.Logger + SugarLogger *zap.SugaredLogger Node *influxdb.Node ClusterMetaClient *coordinator.ClusterMetaClient @@ -102,6 +103,8 @@ type Server struct { CPUProfile string MemProfile string + clusterExecutor *coordinator.ClusterExecutor + // httpAPIAddr is the host:port combination for the main HTTP API for querying and writing data httpAPIAddr string @@ -187,7 +190,8 @@ func NewServer(c *Config, buildInfo *BuildInfo, logger *zap.Logger) (*Server, er Node: node, - Logger: logger, + Logger: logger, + SugarLogger: logger.Sugar(), ClusterMetaClient: coordinator.NewMetaClient(c.Meta, c.Coordinator, nodeID), @@ -288,6 +292,7 @@ func NewServer(c *Config, buildInfo *BuildInfo, logger *zap.Logger) (*Server, er clusterExecutor.WithLogger(s.Logger) // Initialize query executor. + s.clusterExecutor = clusterExecutor s.QueryExecutor = query.NewExecutor() s.QueryExecutor.StatementExecutor = &influxdb_coordinator.StatementExecutor{ MetaClient: s.ClusterMetaClient, @@ -579,6 +584,8 @@ func (s *Server) Open() error { go s.startServerReporting() } + go s.poolStatsLoop() + return nil } @@ -688,6 +695,36 @@ func (s *Server) reportServer() { go cl.Save(usage) } +func dumpPoolStats(name string, sugar *zap.SugaredLogger, stats []coordinator.StatEntity) { + sugar.Info("Dump statistics of ", name) + sugar.Info("active/idle/capacity, success/cost(ms), failure/cost(ms), return/close") + for _, stat := range stats { + sugar.Infof("Node %d: %d/%d/%d, %d/%d, %d/%d, %d/%d", + stat.NodeId, + stat.Stat.Active, stat.Stat.Idle, stat.Stat.Capacity, + stat.Stat.GetSuccessCnt, stat.Stat.GetSuccessMillis, + stat.Stat.GetFailureCnt, stat.Stat.GetFailureMillis, + stat.Stat.ReturnCnt, stat.Stat.CloseCnt, + ) + } +} + +func (s *Server) poolStatsLoop() { + ticker := time.NewTicker(120 * time.Second) + +LOOP: + for { + select { + case <-ticker.C: + dumpPoolStats("remote executors", s.SugarLogger, s.clusterExecutor.RemoteNodeExecutor.Stats()) + dumpPoolStats("shard writers", s.SugarLogger, s.ShardWriter.Stats()) + case <-s.closing: + //shutdown + break LOOP + } + } +} + // Service represents a service attached to the server. type Service interface { WithLogger(log *zap.Logger) diff --git a/coordinator/client_pool.go b/coordinator/client_pool.go index 7ca77a6..f3f7e97 100644 --- a/coordinator/client_pool.go +++ b/coordinator/client_pool.go @@ -3,6 +3,7 @@ package coordinator import ( "fmt" "net" + "sort" "sync" "time" @@ -21,6 +22,11 @@ type ClientPool struct { factory PoolFactory } +type StatEntity struct { + NodeId uint64 + Stat x.PoolStatistics +} + type poolEntry struct { pool x.ConnPool lastUse time.Time @@ -37,33 +43,33 @@ func NewClientPool(factory PoolFactory) *ClientPool { return pool } -func (c *ClientPool) Len() int { - c.mu.RLock() +func (clientPool *ClientPool) Len() int { + clientPool.mu.RLock() var size int - for _, p := range c.pools { + for _, p := range clientPool.pools { size += p.pool.Len() } - c.mu.RUnlock() + clientPool.mu.RUnlock() return size } -func (c *ClientPool) Total() int { - c.mu.RLock() +func (clientPool *ClientPool) Total() int { + clientPool.mu.RLock() var size int - for _, p := range c.pools { + for _, p := range clientPool.pools { size += p.pool.Total() } - c.mu.RUnlock() + clientPool.mu.RUnlock() return size } -func (c *ClientPool) idleCheckOnce() { - c.mu.Lock() - defer c.mu.Unlock() +func (clientPool *ClientPool) idleCheckOnce() { + clientPool.mu.Lock() + defer clientPool.mu.Unlock() now := time.Now() var removed []uint64 - for nodeId, entry := range c.pools { + for nodeId, entry := range clientPool.pools { if now.Sub(entry.lastUse) <= 600*time.Second { continue } @@ -71,59 +77,78 @@ func (c *ClientPool) idleCheckOnce() { removed = append(removed, nodeId) } for _, id := range removed { - if entry, ok := c.pools[id]; ok { + if entry, ok := clientPool.pools[id]; ok { entry.pool.Close() - delete(c.pools, id) + delete(clientPool.pools, id) } } } -func (c *ClientPool) idleCheckLoop() { - c.wg.Add(1) - defer c.wg.Done() +func (clientPool *ClientPool) idleCheckLoop() { + clientPool.wg.Add(1) + defer clientPool.wg.Done() ticker := time.NewTicker(60 * time.Second) defer ticker.Stop() for { select { case <-ticker.C: - c.idleCheckOnce() - case <-c.closer: + clientPool.idleCheckOnce() + case <-clientPool.closer: return } } } -func (c *ClientPool) GetConn(nodeID uint64) (x.PooledConn, error) { - c.mu.RLock() - if entry, ok := c.pools[nodeID]; ok { +func (clientPool *ClientPool) GetConn(nodeID uint64) (x.PooledConn, error) { + clientPool.mu.RLock() + if entry, ok := clientPool.pools[nodeID]; ok { entry.lastUse = time.Now() - c.mu.RUnlock() + clientPool.mu.RUnlock() return entry.pool.Get() } - c.mu.RUnlock() + clientPool.mu.RUnlock() // switch to write lock - c.mu.Lock() - defer c.mu.Unlock() + clientPool.mu.Lock() + defer clientPool.mu.Unlock() // create new pool - pool, err := c.factory(nodeID) + pool, err := clientPool.factory(nodeID) if err != nil { return nil, err } - c.pools[nodeID] = &poolEntry{ + clientPool.pools[nodeID] = &poolEntry{ lastUse: time.Now(), pool: pool, } return pool.Get() } -func (c *ClientPool) close() { - c.mu.Lock() - defer c.mu.Unlock() - for _, entry := range c.pools { +// Dump statistics of connection pools in this client +func (clientPool *ClientPool) Stat() []StatEntity { + clientPool.mu.RLock() + defer clientPool.mu.RUnlock() + + stats := make([]StatEntity, 0, len(clientPool.pools)) + for nodeId, pool := range clientPool.pools { + stats = append(stats, StatEntity{ + Stat: pool.pool.Statistics(), + NodeId: nodeId, + }) + } + sort.Slice(stats, func(i, j int) bool { + return stats[i].NodeId < stats[j].NodeId + }) + + return stats +} + +func (clientPool *ClientPool) close() { + clientPool.mu.Lock() + defer clientPool.mu.Unlock() + for _, entry := range clientPool.pools { entry.pool.Close() } - c.pools = nil + clientPool.pools = nil } type ClientConnFactory struct { diff --git a/coordinator/remote_executor.go b/coordinator/remote_executor.go index fcf4847..b9e8d09 100644 --- a/coordinator/remote_executor.go +++ b/coordinator/remote_executor.go @@ -31,6 +31,7 @@ type RemoteNodeExecutor interface { MapType(nodeId uint64, m *influxql.Measurement, field string, shardIds []uint64) (influxql.DataType, error) CreateIterator(nodeId uint64, ctx context.Context, m *influxql.Measurement, opt query.IteratorOptions, shardIds []uint64) (query.Iterator, error) TaskManagerStatement(nodeId uint64, stmt influxql.Statement) (*query.Result, error) + Stats() []StatEntity } type remoteNodeExecutor struct { @@ -41,8 +42,8 @@ type remoteNodeExecutor struct { Logger *zap.Logger } -func (me *remoteNodeExecutor) WithLogger(log *zap.Logger) { - me.Logger = log.With(zap.String("service", "RemoteNodeExecutor")) +func (executor *remoteNodeExecutor) WithLogger(log *zap.Logger) { + executor.Logger = log.With(zap.String("service", "RemoteNodeExecutor")) } func writeTestPacket(conn net.Conn) (err error) { @@ -84,8 +85,8 @@ func getConnWithRetry(pool *ClientPool, nodeId uint64, logger *zap.Logger) (x.Po return nil, err } -func (me *remoteNodeExecutor) CreateIterator(nodeId uint64, ctx context.Context, m *influxql.Measurement, opt query.IteratorOptions, shardIds []uint64) (query.Iterator, error) { - conn, err := getConnWithRetry(me.ClientPool, nodeId, me.Logger) +func (executor *remoteNodeExecutor) CreateIterator(nodeId uint64, ctx context.Context, m *influxql.Measurement, opt query.IteratorOptions, shardIds []uint64) (query.Iterator, error) { + conn, err := getConnWithRetry(executor.ClientPool, nodeId, executor.Logger) if err != nil { return nil, err } @@ -123,18 +124,14 @@ func (me *remoteNodeExecutor) CreateIterator(nodeId uint64, ctx context.Context, return nil, err } - if resp.DataType == influxql.Unknown { - return nil, nil - } - stats := query.IteratorStats{SeriesN: resp.SeriesN} //conn.Close will be invoked when iterator.Close itr := query.NewReaderIterator(ctx, newIteratorReader(conn, resp.Termination), resp.DataType, stats) return itr, nil } -func (me *remoteNodeExecutor) MapType(nodeId uint64, m *influxql.Measurement, field string, shardIds []uint64) (influxql.DataType, error) { - conn, err := getConnWithRetry(me.ClientPool, nodeId, me.Logger) +func (executor *remoteNodeExecutor) MapType(nodeId uint64, m *influxql.Measurement, field string, shardIds []uint64) (influxql.DataType, error) { + conn, err := getConnWithRetry(executor.ClientPool, nodeId, executor.Logger) if err != nil { return influxql.Unknown, err } @@ -171,8 +168,8 @@ func (me *remoteNodeExecutor) MapType(nodeId uint64, m *influxql.Measurement, fi return resp.DataType, nil } -func (me *remoteNodeExecutor) IteratorCost(nodeId uint64, m *influxql.Measurement, opt query.IteratorOptions, shardIds []uint64) (query.IteratorCost, error) { - conn, err := getConnWithRetry(me.ClientPool, nodeId, me.Logger) +func (executor *remoteNodeExecutor) IteratorCost(nodeId uint64, m *influxql.Measurement, opt query.IteratorOptions, shardIds []uint64) (query.IteratorCost, error) { + conn, err := getConnWithRetry(executor.ClientPool, nodeId, executor.Logger) if err != nil { return query.IteratorCost{}, err } @@ -204,8 +201,8 @@ func (me *remoteNodeExecutor) IteratorCost(nodeId uint64, m *influxql.Measuremen return resp.Cost, nil } -func (me *remoteNodeExecutor) FieldDimensions(nodeId uint64, m *influxql.Measurement, shardIds []uint64) (fields map[string]influxql.DataType, dimensions map[string]struct{}, err error) { - conn, err := getConnWithRetry(me.ClientPool, nodeId, me.Logger) +func (executor *remoteNodeExecutor) FieldDimensions(nodeId uint64, m *influxql.Measurement, shardIds []uint64) (fields map[string]influxql.DataType, dimensions map[string]struct{}, err error) { + conn, err := getConnWithRetry(executor.ClientPool, nodeId, executor.Logger) if err != nil { return nil, nil, err } @@ -236,8 +233,8 @@ func (me *remoteNodeExecutor) FieldDimensions(nodeId uint64, m *influxql.Measure return resp.Fields, resp.Dimensions, nil } -func (me *remoteNodeExecutor) TaskManagerStatement(nodeId uint64, stmt influxql.Statement) (*query.Result, error) { - conn, err := getConnWithRetry(me.ClientPool, nodeId, me.Logger) +func (executor *remoteNodeExecutor) TaskManagerStatement(nodeId uint64, stmt influxql.Statement) (*query.Result, error) { + conn, err := getConnWithRetry(executor.ClientPool, nodeId, executor.Logger) if err != nil { return nil, err } @@ -270,8 +267,8 @@ func (me *remoteNodeExecutor) TaskManagerStatement(nodeId uint64, stmt influxql. return result, nil } -func (me *remoteNodeExecutor) SeriesCardinality(nodeId uint64, database string) (int64, error) { - conn, err := getConnWithRetry(me.ClientPool, nodeId, me.Logger) +func (executor *remoteNodeExecutor) SeriesCardinality(nodeId uint64, database string) (int64, error) { + conn, err := getConnWithRetry(executor.ClientPool, nodeId, executor.Logger) if err != nil { return -1, err } @@ -303,8 +300,8 @@ func (me *remoteNodeExecutor) SeriesCardinality(nodeId uint64, database string) return n, nil } -func (me *remoteNodeExecutor) MeasurementNames(nodeId uint64, database string, cond influxql.Expr) ([][]byte, error) { - conn, err := getConnWithRetry(me.ClientPool, nodeId, me.Logger) +func (executor *remoteNodeExecutor) MeasurementNames(nodeId uint64, database string, cond influxql.Expr) ([][]byte, error) { + conn, err := getConnWithRetry(executor.ClientPool, nodeId, executor.Logger) if err != nil { return nil, err } @@ -337,8 +334,8 @@ func (me *remoteNodeExecutor) MeasurementNames(nodeId uint64, database string, c return names, nil } -func (me *remoteNodeExecutor) TagValues(nodeId uint64, shardIDs []uint64, cond influxql.Expr) ([]tsdb.TagValues, error) { - conn, err := getConnWithRetry(me.ClientPool, nodeId, me.Logger) +func (executor *remoteNodeExecutor) TagValues(nodeId uint64, shardIDs []uint64, cond influxql.Expr) ([]tsdb.TagValues, error) { + conn, err := getConnWithRetry(executor.ClientPool, nodeId, executor.Logger) if err != nil { return nil, err } @@ -373,8 +370,8 @@ func (me *remoteNodeExecutor) TagValues(nodeId uint64, shardIDs []uint64, cond i return tagValues, nil } -func (me *remoteNodeExecutor) TagKeys(nodeId uint64, shardIDs []uint64, cond influxql.Expr) ([]tsdb.TagKeys, error) { - conn, err := getConnWithRetry(me.ClientPool, nodeId, me.Logger) +func (executor *remoteNodeExecutor) TagKeys(nodeId uint64, shardIDs []uint64, cond influxql.Expr) ([]tsdb.TagKeys, error) { + conn, err := getConnWithRetry(executor.ClientPool, nodeId, executor.Logger) if err != nil { return nil, err } @@ -407,8 +404,8 @@ func (me *remoteNodeExecutor) TagKeys(nodeId uint64, shardIDs []uint64, cond inf return tagKeys, nil } -func (me *remoteNodeExecutor) DeleteMeasurement(nodeId uint64, database, name string) error { - conn, err := getConnWithRetry(me.ClientPool, nodeId, me.Logger) +func (executor *remoteNodeExecutor) DeleteMeasurement(nodeId uint64, database, name string) error { + conn, err := getConnWithRetry(executor.ClientPool, nodeId, executor.Logger) if err != nil { return err } @@ -439,8 +436,8 @@ func (me *remoteNodeExecutor) DeleteMeasurement(nodeId uint64, database, name st return nil } -func (me *remoteNodeExecutor) DeleteDatabase(nodeId uint64, database string) error { - conn, err := getConnWithRetry(me.ClientPool, nodeId, me.Logger) +func (executor *remoteNodeExecutor) DeleteDatabase(nodeId uint64, database string) error { + conn, err := getConnWithRetry(executor.ClientPool, nodeId, executor.Logger) if err != nil { return err } @@ -470,8 +467,8 @@ func (me *remoteNodeExecutor) DeleteDatabase(nodeId uint64, database string) err return nil } -func (me *remoteNodeExecutor) DeleteSeries(nodeId uint64, database string, sources []influxql.Source, cond influxql.Expr) error { - conn, err := getConnWithRetry(me.ClientPool, nodeId, me.Logger) +func (executor *remoteNodeExecutor) DeleteSeries(nodeId uint64, database string, sources []influxql.Source, cond influxql.Expr) error { + conn, err := getConnWithRetry(executor.ClientPool, nodeId, executor.Logger) if err != nil { return err } @@ -503,6 +500,10 @@ func (me *remoteNodeExecutor) DeleteSeries(nodeId uint64, database string, sourc return nil } +func (executor *remoteNodeExecutor) Stats() []StatEntity { + return executor.ClientPool.Stat() +} + type iteratorReader struct { io.ReadCloser terminator []byte diff --git a/coordinator/shard_writer.go b/coordinator/shard_writer.go index 410bbf4..66e15fc 100644 --- a/coordinator/shard_writer.go +++ b/coordinator/shard_writer.go @@ -151,3 +151,7 @@ func (w *ShardWriter) Close() error { w.pool = nil return nil } + +func (w *ShardWriter) Stats() []StatEntity { + return w.pool.Stat() +} diff --git a/x/pool.go b/x/pool.go index 1f0bd9c..b06e2ef 100644 --- a/x/pool.go +++ b/x/pool.go @@ -14,6 +14,13 @@ var ( ErrClosed = errors.New("pool is closed") ) +type PoolStatistics struct { + Active, Idle, Capacity int + GetSuccessCnt, GetSuccessMillis uint64 + GetFailureCnt, GetFailureMillis uint64 + ReturnCnt, CloseCnt uint64 +} + type ConnPool interface { // Get returns a new connection from the pool. Closing the connections puts // it back to the Pool. Closing it when the pool is destroyed or full will @@ -28,6 +35,7 @@ type ConnPool interface { Len() int // Total returns total connection count managed by pool Total() int + Statistics() PoolStatistics } type PooledConn interface { @@ -53,6 +61,11 @@ type boundedPool struct { total int32 // net.Conn generator factory Factory + + // statistics + getSuccessCnt, getSuccessCost uint64 + getFailureCnt, getFailureCost uint64 + returnCnt, closeCnt uint64 } // Factory is a function to create new connections. @@ -71,7 +84,7 @@ func NewBoundedPool(initialCnt, maxCap int, idleTimeout time.Duration, waitTimeo return nil, errors.New("invalid capacity settings") } - c := &boundedPool{ + pool := &boundedPool{ closer: make(chan int), conns: make(chan *pooledConn, maxCap), factory: factory, @@ -85,26 +98,26 @@ func NewBoundedPool(initialCnt, maxCap int, idleTimeout time.Duration, waitTimeo for i := 0; i < initialCnt; i++ { conn, err := factory() if err != nil { - c.Close() + pool.Close() return nil, fmt.Errorf("factory is not able to fill the pool: %s", err) } - c.conns <- c.wrapConn(conn) - atomic.AddInt32(&c.total, 1) + pool.conns <- pool.wrapConn(conn) + atomic.AddInt32(&pool.total, 1) } - go c.idleChecker() + go pool.idleChecker() - return c, nil + return pool, nil } -func (c *boundedPool) checkOnce() { +func (pool *boundedPool) checkOnce() { now := time.Now() - for int(atomic.LoadInt32(&c.total)) > c.initialCnt && len(c.conns) > 0 { + for int(atomic.LoadInt32(&pool.total)) > pool.initialCnt && len(pool.conns) > 0 { select { - case conn := <-c.conns: - if now.Sub(conn.lastUse) <= c.idleTimeout { + case conn := <-pool.conns: + if now.Sub(conn.lastUse) <= pool.idleTimeout { select { - case c.conns <- conn: + case pool.conns <- conn: default: // can't be returned, drop conn.MarkUnusable() @@ -123,17 +136,17 @@ func (c *boundedPool) checkOnce() { } // idleChecker checks the connections in pool and closes those haven't been used for idleTimeout and more than initialCnt. -func (c *boundedPool) idleChecker() { - c.wg.Add(1) - defer c.wg.Done() +func (pool *boundedPool) idleChecker() { + pool.wg.Add(1) + defer pool.wg.Done() ticker := time.NewTicker(60 * time.Second) defer ticker.Stop() LOOP: for { select { case <-ticker.C: - c.checkOnce() - case <-c.closer: + pool.checkOnce() + case <-pool.closer: break LOOP } } @@ -142,31 +155,48 @@ LOOP: // Get implements the Pool interfaces Get() method. If there is no new // connection available in the pool, a new connection will be created via the // Factory() method. -func (c *boundedPool) Get() (PooledConn, error) { - conns := c.conns +func (pool *boundedPool) Get() (newConn PooledConn, errOccurred error) { + conns := pool.conns if conns == nil { - return nil, ErrClosed + errOccurred = ErrClosed + return } + // statistics + begin := time.Now().Nanosecond() / 1e6 + defer func() { + cost := uint64(time.Now().Nanosecond()/1e6 - begin) + if errOccurred == nil { + atomic.AddUint64(&pool.getSuccessCnt, 1) + atomic.AddUint64(&pool.getSuccessCost, cost) + } else if errOccurred != ErrClosed { + atomic.AddUint64(&pool.getFailureCnt, 1) + atomic.AddUint64(&pool.getFailureCost, cost) + } + }() + // Try and grab a connection from the pool select { case conn := <-conns: if conn == nil { - return nil, ErrClosed + errOccurred = ErrClosed + return } - return conn, nil + newConn = conn + return default: // Could not get connection, can we create a new one? - total := atomic.LoadInt32(&c.total) + total := atomic.LoadInt32(&pool.total) capacity := int32(cap(conns)) - if total < capacity && atomic.AddInt32(&c.total, 1) <= capacity { - conn, err := c.factory() + if total < capacity && atomic.AddInt32(&pool.total, 1) <= capacity { + conn, err := pool.factory() if err != nil { - atomic.AddInt32(&c.total, -1) - return nil, err + atomic.AddInt32(&pool.total, -1) + errOccurred = err + return } - - return c.wrapConn(conn), nil + newConn = pool.wrapConn(conn) + return } } @@ -175,33 +205,35 @@ func (c *boundedPool) Get() (PooledConn, error) { select { case conn := <-conns: if conn == nil { - return nil, ErrClosed + errOccurred = ErrClosed + return } - return conn, nil - case <-time.After(c.waitTimeout): - return nil, fmt.Errorf("timed out waiting for free connection") + newConn = conn + return + case <-time.After(pool.waitTimeout): + errOccurred = fmt.Errorf("timed out waiting for free connection") + return } - } // put puts the connection back to the pool. If the pool is full or closed, // conn is simply closed. A nil conn will be rejected. -func (c *boundedPool) put(conn *pooledConn) error { +func (pool *boundedPool) put(conn *pooledConn) error { if conn == nil { return errors.New("connection is nil. rejecting") } - c.mu.RLock() - defer c.mu.RUnlock() + pool.mu.RLock() + defer pool.mu.RUnlock() - if c.conns == nil { + if pool.conns == nil { // pool is closed, close passed connection goto DROP } // put the resource back into the pool. If the pool is full, drop it select { - case c.conns <- conn: + case pool.conns <- conn: return nil default: // pool is full, close passed connection @@ -210,44 +242,72 @@ func (c *boundedPool) put(conn *pooledConn) error { DROP: conn.MarkUnusable() - return conn.Close() + return conn.close() } -func (c *boundedPool) Close() { - close(c.closer) - c.wg.Wait() - c.mu.Lock() - conns := c.conns - c.conns = nil - c.factory = nil - c.mu.Unlock() +func closeIdleConnWhenShutdown(conn *pooledConn) { + conn.MarkUnusable() + conn.mu.Lock() + defer conn.mu.Unlock() + conn.close() +} + +func (pool *boundedPool) Close() { + close(pool.closer) + pool.wg.Wait() + pool.mu.Lock() + conns := pool.conns + pool.conns = nil + pool.factory = nil + pool.mu.Unlock() if conns == nil { return } close(conns) - for conn := range conns { - conn.Close() + for { + conn := <-conns + if conn == nil { + break + } + closeIdleConnWhenShutdown(conn) } } // Len returns current idled connection count in pool -func (c *boundedPool) Len() int { - return len(c.conns) +func (pool *boundedPool) Len() int { + return len(pool.conns) } // Total returns total connection count managed by pool -func (c *boundedPool) Total() int { - return int(atomic.LoadInt32(&c.total)) +func (pool *boundedPool) Total() int { + return int(atomic.LoadInt32(&pool.total)) } // newConn wraps a standard net.Conn to a poolConn net.Conn. -func (c *boundedPool) wrapConn(conn net.Conn) *pooledConn { - p := &pooledConn{c: c} - p.Conn = conn - p.lastUse = time.Now() - return p +func (pool *boundedPool) wrapConn(conn net.Conn) *pooledConn { + c := &pooledConn{pool: pool} + c.Conn = conn + c.lastUse = time.Now() + return c +} + +func (pool *boundedPool) Statistics() PoolStatistics { + stat := PoolStatistics{} + stat.Capacity = cap(pool.conns) + stat.Idle = len(pool.conns) + total := atomic.LoadInt32(&pool.total) + stat.Active = int(total) - stat.Idle + + stat.GetSuccessCnt = atomic.LoadUint64(&pool.getSuccessCnt) + stat.GetSuccessMillis = atomic.LoadUint64(&pool.getSuccessCost) + stat.GetFailureCnt = atomic.LoadUint64(&pool.getFailureCnt) + stat.GetFailureMillis = atomic.LoadUint64(&pool.getFailureCost) + stat.ReturnCnt = atomic.LoadUint64(&pool.returnCnt) + stat.CloseCnt = atomic.LoadUint64(&pool.closeCnt) + + return stat } // pooledConn is a wrapper around net.Conn to modify the the behavior of @@ -255,42 +315,51 @@ func (c *boundedPool) wrapConn(conn net.Conn) *pooledConn { type pooledConn struct { net.Conn mu sync.Mutex - c *boundedPool + pool *boundedPool lastUse time.Time unusable bool } // Close() puts the given connects back to the pool instead of closing it. -func (p *pooledConn) Close() error { - p.mu.Lock() - defer p.mu.Unlock() - if p.Conn == nil { +func (c *pooledConn) Close() error { + c.mu.Lock() + defer c.mu.Unlock() + return c.close() +} + +// inner close method, no MUTEX lock inside +func (c *pooledConn) close() error { + if c.Conn == nil { return nil } defer func() { - p.Conn = nil + c.Conn = nil }() - if p.unusable { - if p.Conn != nil { - return p.Conn.Close() + + if c.unusable { + if c.pool != nil { + atomic.AddUint64(&c.pool.closeCnt, 1) } - return nil + return c.Conn.Close() } - return p.c.put(&pooledConn{ - Conn: p.Conn, - c: p.c, + if c.pool != nil { + atomic.AddUint64(&c.pool.returnCnt, 1) + } + return c.pool.put(&pooledConn{ + Conn: c.Conn, + pool: c.pool, lastUse: time.Now(), unusable: false, }) } // MarkUnusable() marks the connection not usable any more, to let the pool close it instead of returning it to pool. -func (p *pooledConn) MarkUnusable() { - p.mu.Lock() - defer p.mu.Unlock() - if p.unusable { +func (c *pooledConn) MarkUnusable() { + c.mu.Lock() + defer c.mu.Unlock() + if c.unusable { return } - p.unusable = true - atomic.AddInt32(&p.c.total, -1) + c.unusable = true + atomic.AddInt32(&c.pool.total, -1) } diff --git a/x/pool_test.go b/x/pool_test.go index cb0112e..e561155 100644 --- a/x/pool_test.go +++ b/x/pool_test.go @@ -1,6 +1,7 @@ package x import ( + "fmt" "net" "testing" "time" @@ -8,69 +9,133 @@ import ( "github.com/stretchr/testify/assert" ) +func dumpStatistics(stat PoolStatistics) { + fmt.Println("Active/Idle/Capacity:", fmt.Sprintf("%d/%d/%d", stat.Active, stat.Idle, stat.Capacity)) + fmt.Println("Get Success:", fmt.Sprintf("%d times, %d ms", stat.GetSuccessCnt, stat.GetSuccessMillis)) + fmt.Println("Get Failure:", fmt.Sprintf("%d times, %d ms", stat.GetFailureCnt, stat.GetFailureMillis)) + fmt.Println("Return:", stat.ReturnCnt) + fmt.Println("Close:", stat.CloseCnt) +} + func TestPoolCapacity(t *testing.T) { echoServer := NewEchoServer("tcp", "127.0.0.1:12345") err := echoServer.Start() assert.Nil(t, err) cnt := 0 - p, err := NewBoundedPool(1, 3, time.Second, time.Second, func() (net.Conn, error) { + pool, err := NewBoundedPool(1, 3, time.Second, time.Second, func() (net.Conn, error) { cnt++ return net.Dial("tcp", "127.0.0.1:12345") }) - assert.Equal(t, 1, p.Len()) - assert.Equal(t, 1, p.Total()) + assert.Equal(t, 1, pool.Len()) + assert.Equal(t, 1, pool.Total()) assert.Equal(t, 1, cnt) - conn1, err := p.Get() - (p.(*boundedPool)).checkOnce() + // normal get + conn1, err := pool.Get() + (pool.(*boundedPool)).checkOnce() assert.Nil(t, err) - assert.Equal(t, 0, p.Len()) - assert.Equal(t, 1, p.Total()) + assert.Equal(t, 0, pool.Len()) + assert.Equal(t, 1, pool.Total()) assert.Equal(t, 1, cnt) - conn2, err := p.Get() - (p.(*boundedPool)).checkOnce() + // normal get + conn2, err := pool.Get() + (pool.(*boundedPool)).checkOnce() assert.Nil(t, err) - assert.Equal(t, 0, p.Len()) - assert.Equal(t, 2, p.Total()) + assert.Equal(t, 0, pool.Len()) + assert.Equal(t, 2, pool.Total()) assert.Equal(t, 2, cnt) - conn3, err := p.Get() - (p.(*boundedPool)).checkOnce() + // normal get + conn3, err := pool.Get() + (pool.(*boundedPool)).checkOnce() assert.Nil(t, err) - assert.Equal(t, 0, p.Len()) - assert.Equal(t, 3, p.Total()) + assert.Equal(t, 0, pool.Len()) + assert.Equal(t, 3, pool.Total()) assert.Equal(t, 3, cnt) - _, err = p.Get() - (p.(*boundedPool)).checkOnce() + // cannot get more, failure + 1 + _, err = pool.Get() + (pool.(*boundedPool)).checkOnce() assert.NotNil(t, err) - assert.Equal(t, 0, p.Len()) - assert.Equal(t, 3, p.Total()) + assert.Equal(t, 0, pool.Len()) + assert.Equal(t, 3, pool.Total()) assert.Equal(t, 3, cnt) + stat := pool.Statistics() + assert.Equal(t, 3, stat.Active) + assert.Equal(t, 3, stat.Capacity) + assert.Equal(t, uint64(3), stat.GetSuccessCnt) + assert.Equal(t, uint64(1), stat.GetFailureCnt) + assert.Equal(t, uint64(0), stat.ReturnCnt) + assert.Equal(t, uint64(0), stat.CloseCnt) + + // close 1 connection conn3.Close() - (p.(*boundedPool)).checkOnce() - assert.Equal(t, 1, p.Len()) - assert.Equal(t, 3, p.Total()) + (pool.(*boundedPool)).checkOnce() + assert.Equal(t, 1, pool.Len()) + assert.Equal(t, 3, pool.Total()) assert.Equal(t, 3, cnt) + // close remained 2 connections conn2.Close() conn1.Close() - (p.(*boundedPool)).checkOnce() - assert.Equal(t, 3, p.Len()) - assert.Equal(t, 3, p.Total()) + (pool.(*boundedPool)).checkOnce() + assert.Equal(t, 3, pool.Len()) + assert.Equal(t, 3, pool.Total()) assert.Equal(t, 3, cnt) time.Sleep(time.Second) // recycle idled connections - (p.(*boundedPool)).checkOnce() - (p.(*boundedPool)).checkOnce() - (p.(*boundedPool)).checkOnce() - (p.(*boundedPool)).checkOnce() - assert.Equal(t, 1, p.Len()) - assert.Equal(t, 1, p.Total()) - - p.Close() + (pool.(*boundedPool)).checkOnce() + (pool.(*boundedPool)).checkOnce() + (pool.(*boundedPool)).checkOnce() + (pool.(*boundedPool)).checkOnce() + assert.Equal(t, 1, pool.Len()) + assert.Equal(t, 1, pool.Total()) + + stat = pool.Statistics() + assert.Equal(t, 0, stat.Active) + assert.Equal(t, 3, stat.Capacity) + assert.Equal(t, uint64(3), stat.GetSuccessCnt) + assert.Equal(t, uint64(1), stat.GetFailureCnt) + assert.Equal(t, uint64(3), stat.ReturnCnt) + assert.Equal(t, uint64(2), stat.CloseCnt) + + pool.Close() + stat = pool.Statistics() + assert.Equal(t, uint64(3), stat.ReturnCnt) + assert.Equal(t, uint64(3), stat.CloseCnt) + echoServer.Close() +} + +func TestPoolCloseBeforeConnClose(t *testing.T) { + echoServer := NewEchoServer("tcp", "127.0.0.1:12345") + err := echoServer.Start() + assert.Nil(t, err) + cnt := 0 + pool, err := NewBoundedPool(1, 3, time.Second, time.Second, func() (net.Conn, error) { + cnt++ + return net.Dial("tcp", "127.0.0.1:12345") + }) + + // normal get + conn, err := pool.Get() + (pool.(*boundedPool)).checkOnce() + assert.Nil(t, err) + assert.Equal(t, 0, pool.Len()) + assert.Equal(t, 1, pool.Total()) + assert.Equal(t, 1, cnt) + + pool.Close() + stat := pool.Statistics() + assert.Equal(t, uint64(0), stat.ReturnCnt) + assert.Equal(t, uint64(0), stat.CloseCnt) + + conn.Close() + stat = pool.Statistics() + assert.Equal(t, uint64(1), stat.ReturnCnt) + assert.Equal(t, uint64(1), stat.CloseCnt) + echoServer.Close() } From 98627b955794f13b960f412d83c4f2d48d52f20c Mon Sep 17 00:00:00 2001 From: Jason Joo Date: Mon, 2 Nov 2020 19:19:03 +0800 Subject: [PATCH 36/46] Improve the precision of cost in creating connections Signed-off-by: Jason Joo --- x/pool.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/x/pool.go b/x/pool.go index b06e2ef..6bee255 100644 --- a/x/pool.go +++ b/x/pool.go @@ -63,8 +63,8 @@ type boundedPool struct { factory Factory // statistics - getSuccessCnt, getSuccessCost uint64 - getFailureCnt, getFailureCost uint64 + getSuccessCnt, getSuccessCost uint64 // in us + getFailureCnt, getFailureCost uint64 // in us returnCnt, closeCnt uint64 } @@ -163,9 +163,9 @@ func (pool *boundedPool) Get() (newConn PooledConn, errOccurred error) { } // statistics - begin := time.Now().Nanosecond() / 1e6 + begin := time.Now().Nanosecond() / 1e3 defer func() { - cost := uint64(time.Now().Nanosecond()/1e6 - begin) + cost := uint64(time.Now().Nanosecond()/1e3 - begin) if errOccurred == nil { atomic.AddUint64(&pool.getSuccessCnt, 1) atomic.AddUint64(&pool.getSuccessCost, cost) @@ -301,9 +301,9 @@ func (pool *boundedPool) Statistics() PoolStatistics { stat.Active = int(total) - stat.Idle stat.GetSuccessCnt = atomic.LoadUint64(&pool.getSuccessCnt) - stat.GetSuccessMillis = atomic.LoadUint64(&pool.getSuccessCost) + stat.GetSuccessMillis = atomic.LoadUint64(&pool.getSuccessCost) / 1e3 // us to ms stat.GetFailureCnt = atomic.LoadUint64(&pool.getFailureCnt) - stat.GetFailureMillis = atomic.LoadUint64(&pool.getFailureCost) + stat.GetFailureMillis = atomic.LoadUint64(&pool.getFailureCost) / 1e3 // us to ms stat.ReturnCnt = atomic.LoadUint64(&pool.returnCnt) stat.CloseCnt = atomic.LoadUint64(&pool.closeCnt) From e26ed1d457a4ef0aa686482a9d789bce2f1d05d6 Mon Sep 17 00:00:00 2001 From: Jason Joo Date: Mon, 2 Nov 2020 22:55:15 +0800 Subject: [PATCH 37/46] Fix possible overlap bug computing cost Signed-off-by: Jason Joo --- x/pool.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x/pool.go b/x/pool.go index 6bee255..8c40a3e 100644 --- a/x/pool.go +++ b/x/pool.go @@ -163,9 +163,9 @@ func (pool *boundedPool) Get() (newConn PooledConn, errOccurred error) { } // statistics - begin := time.Now().Nanosecond() / 1e3 + begin := time.Now() defer func() { - cost := uint64(time.Now().Nanosecond()/1e3 - begin) + cost := uint64(time.Now().Sub(begin).Microseconds()) if errOccurred == nil { atomic.AddUint64(&pool.getSuccessCnt, 1) atomic.AddUint64(&pool.getSuccessCost, cost) From f9927bffbf946f0c6a0b90e91ede4691bdb2146f Mon Sep 17 00:00:00 2001 From: Jason Joo Date: Wed, 4 Nov 2020 08:48:55 +0800 Subject: [PATCH 38/46] Ajust default configurations Signed-off-by: Jason Joo --- cmd/influxd/run/config.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cmd/influxd/run/config.go b/cmd/influxd/run/config.go index da4120d..b0a642f 100644 --- a/cmd/influxd/run/config.go +++ b/cmd/influxd/run/config.go @@ -77,12 +77,15 @@ func NewConfig() *Config { c.Data = tsdb.NewConfig() c.Data.Dir = "./data" c.Data.WALDir = "./wal" + c.Data.QueryLogEnabled = false c.Coordinator = coordinator.NewConfig() + c.Coordinator.LogQueriesAfter = itoml.Duration(500 * time.Millisecond) c.Precreator = precreator.NewConfig() c.Monitor = monitor.NewConfig() c.Subscriber = subscriber.NewConfig() c.HTTPD = httpd.NewConfig() + c.HTTPD.AccessLogPath = "./logs/access.log" c.Logging = logger.NewConfig() c.HintedHandoff = hh.NewConfig() From ac432ee30255503c200579117427f8450514899b Mon Sep 17 00:00:00 2001 From: Jason Joo Date: Wed, 4 Nov 2020 09:03:01 +0800 Subject: [PATCH 39/46] Optimze logging content when write shard failed Signed-off-by: Jason Joo --- coordinator/points_writer.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/coordinator/points_writer.go b/coordinator/points_writer.go index 81a5cbf..521c214 100644 --- a/coordinator/points_writer.go +++ b/coordinator/points_writer.go @@ -405,8 +405,11 @@ func (w *PointsWriter) writeToShard(shard *meta.ShardInfo, database, retentionPo atomic.AddInt64(&w.stats.PointWriteReqRemote, int64(len(points))) err := w.ShardWriter.WriteShard(shardID, owner.NodeID, points) - if err != nil && IsRetryable(err) { - w.Logger.Warn(fmt.Sprint("ShardWriter.WriteShard fail to ", owner.NodeID), zap.Error(err)) + if err != nil && canRetry(err) { + w.Logger.Warn(fmt.Sprintf( + "ShardWriter.WriteShard fail to %d and enqueue to hh", + owner.NodeID, + ), zap.Error(err)) // The remote write failed so queue it via hinted handoff atomic.AddInt64(&w.stats.WritePointReqHH, int64(len(points))) hherr := w.HintedHandoff.WriteShard(shardID, owner.NodeID, points) @@ -476,7 +479,7 @@ func (w *PointsWriter) writeToShard(shard *meta.ShardInfo, database, retentionPo return ErrWriteFailed } -func IsRetryable(err error) bool { +func canRetry(err error) bool { if err == nil { return true } From 1eddafbd615bb1b855ac78fc8ec791c9ea4ac18c Mon Sep 17 00:00:00 2001 From: Jason Joo Date: Wed, 11 Nov 2020 12:44:07 +0800 Subject: [PATCH 40/46] Refactor interator request processing Signed-off-by: Jason Joo --- coordinator/service.go | 36 ++++++++++++++++++------------------ go.mod | 1 + go.sum | 2 ++ 3 files changed, 21 insertions(+), 18 deletions(-) diff --git a/coordinator/service.go b/coordinator/service.go index 02285fb..97b6aaa 100644 --- a/coordinator/service.go +++ b/coordinator/service.go @@ -8,13 +8,13 @@ import ( "fmt" "io" "net" - "runtime/debug" "strings" "sync" "sync/atomic" "time" "github.com/angopher/chronus/x" + errs "github.com/go-errors/errors" "github.com/influxdata/influxdb" "github.com/influxdata/influxdb/models" "github.com/influxdata/influxdb/pkg/tracing" @@ -219,8 +219,7 @@ func (s *Service) serve() { defer func() { s.wg.Done() if err := recover(); err != nil { - buf := debug.Stack() - s.Logger.Error("recover from panic", zap.String("stack", string(buf))) + s.Logger.Error("recover from panic", zap.String("stack", errs.Wrap(err, 2).ErrorStack())) } }() @@ -253,8 +252,6 @@ func (s *Service) handle(conn net.Conn, typ byte, data []byte) (respType byte, r err = s.processExecuteStatementRequest(data) respType = writeShardResponseMessage resp = &WriteShardResponse{} - case createIteratorRequestMessage: - s.processCreateIteratorRequest(conn, data) case fieldDimensionsRequestMessage: respType = fieldDimensionsResponseMessage resp, err = s.processFieldDimensionsRequest(data) @@ -342,19 +339,27 @@ func (s *Service) handleConn(conn net.Conn) { conn.SetReadDeadline(time.Now().Add(500 * time.Millisecond)) typ, data, err := ReadTLV(conn) conn.SetDeadline(time.Time{}) - if err == io.EOF { - return - } - if err, ok := err.(net.Error); ok && err.Timeout() { - continue - } if err != nil { + if err == io.EOF || err.Error() == "use of closed network connection" { + break + } + if err, ok := err.(net.Error); ok && err.Timeout() { + continue + } s.Logger.Error("read error", zap.Error(err)) break } if typ == 0 { continue } + if typ == createIteratorRequestMessage { + // iterator is different from other requests + ioError := s.processCreateIteratorRequest(conn, data) + if ioError { + break + } + continue + } respType, resp, err := s.handle(conn, typ, data) if resp == nil { continue @@ -706,16 +711,10 @@ func (s *Service) processWriteShardRequest(buf []byte) error { return nil } -func (s *Service) processCreateIteratorRequest(conn net.Conn, buf []byte) { +func (s *Service) processCreateIteratorRequest(conn net.Conn, buf []byte) (ioError bool) { var itr query.Iterator var trace *tracing.Trace var span *tracing.Span - ioError := false - defer func() { - if ioError { - conn.Close() - } - }() respType := createIteratorResponseMessage if err := func() error { // Parse request. @@ -839,6 +838,7 @@ func (s *Service) processCreateIteratorRequest(conn net.Conn, buf []byte) { if _, err := conn.Write(itrTerminator); err != nil { ioError = true } + return } func (s *Service) processFieldDimensionsRequest(buf []byte) (*FieldDimensionsResponse, error) { diff --git a/go.mod b/go.mod index e88330c..186c881 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/dgraph-io/dgraph v1.2.7 github.com/dustin/go-humanize v1.0.0 github.com/fatih/color v1.9.0 + github.com/go-errors/errors v1.1.1 github.com/gogo/protobuf v1.3.1 github.com/golang/snappy v0.0.2 github.com/google/go-cmp v0.5.2 diff --git a/go.sum b/go.sum index 5c5f1d8..0453325 100644 --- a/go.sum +++ b/go.sum @@ -151,6 +151,8 @@ github.com/glycerine/go-unsnap-stream v0.0.0-20180323001048-9f0cb55181dd/go.mod github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31 h1:gclg6gY70GLy3PbkQ1AERPfmLMMagS60DKF78eWwLn8= github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24= github.com/go-chi/chi v3.3.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ= +github.com/go-errors/errors v1.1.1 h1:ljK/pL5ltg3qoN+OtN6yCv9HWSfMwxSx90GJCZQxYNg= +github.com/go-errors/errors v1.1.1/go.mod h1:psDX2osz5VnTOnFWbDeWwS7yejl+uV3FEWEp4lssFEs= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-ini/ini v1.39.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= From be74a69295b70c202de75905e471a1251517b13e Mon Sep 17 00:00:00 2001 From: Jason Joo Date: Fri, 13 Nov 2020 11:33:39 +0800 Subject: [PATCH 41/46] Fix empty iterator problem Signed-off-by: Jason Joo --- coordinator/cluster_planning.go | 2 +- coordinator/remote_executor.go | 23 ++++++++++++++++++++--- coordinator/service.go | 14 +++++++------- 3 files changed, 28 insertions(+), 11 deletions(-) diff --git a/coordinator/cluster_planning.go b/coordinator/cluster_planning.go index 8686a91..92e332e 100644 --- a/coordinator/cluster_planning.go +++ b/coordinator/cluster_planning.go @@ -176,7 +176,7 @@ func executePlanSingle(nodeId uint64, shards []meta.ShardInfo, fn QueryFn) (inte defer func() { r := recover() if r != nil { - fmt.Fprintln(os.Stderr, "panic recover:", r) + fmt.Fprintln(os.Stderr, "panic recover on node", nodeId, ":", r) } }() return fn(nodeId, shards) diff --git a/coordinator/remote_executor.go b/coordinator/remote_executor.go index b9e8d09..fe13b42 100644 --- a/coordinator/remote_executor.go +++ b/coordinator/remote_executor.go @@ -115,7 +115,7 @@ func (executor *remoteNodeExecutor) CreateIterator(nodeId uint64, ctx context.Co conn.MarkUnusable() return err } else if resp.Err != nil { - return err + return resp.Err } return nil @@ -544,16 +544,17 @@ func (r *iteratorReader) Read(p []byte) (n int, err error) { bytes := r.bytes r.bytes += n if n == 0 { - return n, err + return 0, err } dumped := r.judger.Dump(r.buf) r.judger.Write(p[:n]) - if r.bytes <= len(r.terminator) { + if r.bytes < len(r.terminator) { return 0, nil } shiftBuffer(p, r.buf[:x.Min(dumped, n)]) if r.judger.Compare(r.terminator) { r.terminated = true + err = io.EOF } if bytes < len(r.buf) { return n - (len(r.buf) - bytes), err @@ -561,6 +562,22 @@ func (r *iteratorReader) Read(p []byte) (n int, err error) { return n, err } +func (r *iteratorReader) consumeRest() (discarded int) { + if r.terminated { + return + } + buf := make([]byte, 32) + for { + n, err := r.Read(buf) + discarded += n + if err == io.EOF { + break + } + } + return +} + func (r *iteratorReader) Close() error { + r.consumeRest() return r.reader.Close() } diff --git a/coordinator/service.go b/coordinator/service.go index 97b6aaa..8decb6a 100644 --- a/coordinator/service.go +++ b/coordinator/service.go @@ -797,6 +797,7 @@ func (s *Service) processCreateIteratorRequest(conn net.Conn, buf []byte) (ioErr seriesN := 0 if itr != nil { + defer itr.Close() seriesN = itr.Stats().SeriesN } // Encode success response. @@ -807,6 +808,12 @@ func (s *Service) processCreateIteratorRequest(conn net.Conn, buf []byte) (ioErr ioError = true return } + defer func() { + // Write termination of iterator + if _, err := conn.Write(itrTerminator); err != nil { + ioError = true + } + }() // Exit if no iterator was produced. if itr == nil { @@ -822,8 +829,6 @@ func (s *Service) processCreateIteratorRequest(conn net.Conn, buf []byte) (ioErr return } - itr.Close() - if trace != nil { span.Finish() if err := encoder.EncodeTrace(trace); err != nil { @@ -833,11 +838,6 @@ func (s *Service) processCreateIteratorRequest(conn net.Conn, buf []byte) (ioErr return } } - - // Write termination of iterator - if _, err := conn.Write(itrTerminator); err != nil { - ioError = true - } return } From 002addc932ede68c787e7ed5a29a98996ceebd76 Mon Sep 17 00:00:00 2001 From: Jason Joo Date: Mon, 16 Nov 2020 13:15:51 +0800 Subject: [PATCH 42/46] Refactor service module making it more stable Signed-off-by: Jason Joo --- cmd/influxd-ctl/action/action.go | 7 +- coordinator/cluster_executor.go | 17 +- coordinator/internal/service_statistics.go | 104 ++++ coordinator/remote_executor.go | 58 +- coordinator/request/request.go | 93 +++ coordinator/request/request_test.go | 181 ++++++ coordinator/service.go | 691 ++------------------- coordinator/service_handlers.go | 513 +++++++++++++++ coordinator/shard_writer.go | 20 +- services/controller/service.go | 2 +- x/cyclic_buffer.go | 8 + 11 files changed, 1005 insertions(+), 689 deletions(-) create mode 100644 coordinator/internal/service_statistics.go create mode 100644 coordinator/request/request.go create mode 100644 coordinator/request/request_test.go create mode 100644 coordinator/service_handlers.go diff --git a/cmd/influxd-ctl/action/action.go b/cmd/influxd-ctl/action/action.go index 543bb25..7633429 100644 --- a/cmd/influxd-ctl/action/action.go +++ b/cmd/influxd-ctl/action/action.go @@ -4,7 +4,6 @@ import ( "encoding/json" "errors" "fmt" - "io" "net" "sort" "strconv" @@ -347,8 +346,8 @@ func RequestAndWaitResp(addr string, reqTyp, respTyp byte, req interface{}, resp return DecodeTLV(conn, respTyp, resp) } -func DecodeTLV(r io.Reader, expTyp byte, v interface{}) error { - typ, err := coordinator.ReadType(r) +func DecodeTLV(conn net.Conn, expTyp byte, v interface{}) error { + typ, err := coordinator.ReadType(conn) if err != nil { return err } @@ -356,7 +355,7 @@ func DecodeTLV(r io.Reader, expTyp byte, v interface{}) error { return fmt.Errorf("invalid type, exp: %d, got: %d", expTyp, typ) } - buf, err := coordinator.ReadLV(r) + buf, err := coordinator.ReadLV(conn, 4*time.Second) if err != nil { return err } diff --git a/coordinator/cluster_executor.go b/coordinator/cluster_executor.go index f6e8542..df553a8 100644 --- a/coordinator/cluster_executor.go +++ b/coordinator/cluster_executor.go @@ -406,6 +406,7 @@ func (me *ClusterExecutor) CreateIterator(ctx context.Context, m *influxql.Measu result, _ := n2s.ExecuteWithRetry(fn) seriesN := 0 + var errOccur error inputs := make([]query.Iterator, 0, len(result)) for _, t := range result { r, ok := t.(*Result) @@ -413,7 +414,8 @@ func (me *ClusterExecutor) CreateIterator(ctx context.Context, m *influxql.Measu continue } if r.err != nil { - return nil, r.err + errOccur = r.err + break } if r.iter != nil { stats := r.iter.Stats() @@ -421,6 +423,19 @@ func (me *ClusterExecutor) CreateIterator(ctx context.Context, m *influxql.Measu inputs = append(inputs, r.iter) } } + if errOccur != nil { + // close all iterators + for _, t := range result { + r, ok := t.(*Result) + if !ok { + continue + } + if r.iter != nil { + r.iter.Close() + } + } + return nil, errOccur + } if opt.MaxSeriesN > 0 && seriesN > opt.MaxSeriesN { query.Iterators(inputs).Close() diff --git a/coordinator/internal/service_statistics.go b/coordinator/internal/service_statistics.go new file mode 100644 index 0000000..f3fb3f6 --- /dev/null +++ b/coordinator/internal/service_statistics.go @@ -0,0 +1,104 @@ +package internal + +import ( + "sync/atomic" + + "github.com/influxdata/influxdb/models" +) + +// Statistics maintained by the cluster package +const ( + writeShardReq = "writeShardReq" + writeShardPointsReq = "writeShardPointsReq" + writeShardFail = "writeShardFail" + + createIteratorReq = "createIteratorReq" + createIteratorFail = "createIteratorFail" + + fieldDimensionsReq = "fieldDimensionsReq" + fieldDimensionsFail = "fieldDimensionsFail" + + tagKeysReq = "tagKeysReq" + tagKeysFail = "tagKeysFail" + + tagValuesReq = "tagValuesReq" + tagValuesFail = "tagValuesFail" + + measurementNamesReq = "measurementNamesReq" + measurementNamesFail = "measurementNamesFail" + + seriesCardinalityReq = "seriesCardinalityReq" + seriesCardinalityFail = "seriesCardinalityFail" + + iteratorCostReq = "iteratorCostReq" + iteratorCostFail = "iteratorCostFail" + + mapTypeReq = "mapTypeReq" + mapTypeFail = "mapTypeFail" +) + +type InternalServiceStatistics struct { + WriteShardReq int64 + WriteShardPointsReq int64 + WriteShardFail int64 + + CreateIteratorReq int64 + CreateIteratorFail int64 + + FieldDimensionsReq int64 + FieldDimensionsFail int64 + + TagKeysReq int64 + TagKeysFail int64 + + TagValuesReq int64 + TagValuesFail int64 + + MeasurementNamesReq int64 + MeasurementNamesFail int64 + + SeriesCardinalityReq int64 + SeriesCardinalityFail int64 + + IteratorCostReq int64 + IteratorCostFail int64 + + MapTypeReq int64 + MapTypeFail int64 +} + +func Statistics(stats *InternalServiceStatistics, tags map[string]string) []models.Statistic { + return []models.Statistic{{ + Name: "coordinator_service", + Tags: tags, + Values: map[string]interface{}{ + writeShardReq: atomic.LoadInt64(&stats.WriteShardReq), + writeShardPointsReq: atomic.LoadInt64(&stats.WriteShardPointsReq), + writeShardFail: atomic.LoadInt64(&stats.WriteShardFail), + + createIteratorReq: atomic.LoadInt64(&stats.CreateIteratorReq), + createIteratorFail: atomic.LoadInt64(&stats.CreateIteratorFail), + + fieldDimensionsReq: atomic.LoadInt64(&stats.FieldDimensionsReq), + fieldDimensionsFail: atomic.LoadInt64(&stats.FieldDimensionsFail), + + tagKeysReq: atomic.LoadInt64(&stats.TagKeysReq), + tagKeysFail: atomic.LoadInt64(&stats.TagKeysFail), + + tagValuesReq: atomic.LoadInt64(&stats.TagValuesReq), + tagValuesFail: atomic.LoadInt64(&stats.TagValuesFail), + + measurementNamesReq: atomic.LoadInt64(&stats.MeasurementNamesReq), + measurementNamesFail: atomic.LoadInt64(&stats.MeasurementNamesFail), + + seriesCardinalityReq: atomic.LoadInt64(&stats.SeriesCardinalityReq), + seriesCardinalityFail: atomic.LoadInt64(&stats.SeriesCardinalityFail), + + iteratorCostReq: atomic.LoadInt64(&stats.IteratorCostReq), + iteratorCostFail: atomic.LoadInt64(&stats.IteratorCostFail), + + mapTypeReq: atomic.LoadInt64(&stats.MapTypeReq), + mapTypeFail: atomic.LoadInt64(&stats.MapTypeFail), + }, + }} +} diff --git a/coordinator/remote_executor.go b/coordinator/remote_executor.go index fe13b42..5b849f4 100644 --- a/coordinator/remote_executor.go +++ b/coordinator/remote_executor.go @@ -2,7 +2,7 @@ package coordinator import ( "context" - "encoding/binary" + "encoding" "errors" "io" "net" @@ -49,12 +49,7 @@ func (executor *remoteNodeExecutor) WithLogger(log *zap.Logger) { func writeTestPacket(conn net.Conn) (err error) { conn.SetDeadline(time.Now().Add(500 * time.Millisecond)) defer conn.SetDeadline(time.Time{}) - typ := testRequestMessage - size := uint64(0) - if err = binary.Write(conn, binary.BigEndian, &typ); err != nil { - return - } - if err = binary.Write(conn, binary.BigEndian, &size); err != nil { + if _, err = conn.Write([]byte{testRequestMessage, 0, 0, 0, 0, 0, 0, 0, 0}); err != nil { return } return @@ -111,7 +106,7 @@ func (executor *remoteNodeExecutor) CreateIterator(nodeId uint64, ctx context.Co } // Read the response. - if _, err := DecodeTLV(conn, &resp); err != nil { + if _, err := decodeTLV(conn, &resp); err != nil { conn.MarkUnusable() return err } else if resp.Err != nil { @@ -153,7 +148,7 @@ func (executor *remoteNodeExecutor) MapType(nodeId uint64, m *influxql.Measureme return err } - if _, err := DecodeTLV(conn, &resp); err != nil { + if _, err := decodeTLV(conn, &resp); err != nil { conn.MarkUnusable() return err } else if resp.Err != "" { @@ -186,7 +181,7 @@ func (executor *remoteNodeExecutor) IteratorCost(nodeId uint64, m *influxql.Meas return err } - if _, err := DecodeTLV(conn, &resp); err != nil { + if _, err := decodeTLV(conn, &resp); err != nil { conn.MarkUnusable() return err } else if resp.Err != "" { @@ -218,7 +213,7 @@ func (executor *remoteNodeExecutor) FieldDimensions(nodeId uint64, m *influxql.M return err } - if _, err := DecodeTLV(conn, &resp); err != nil { + if _, err := decodeTLV(conn, &resp); err != nil { conn.MarkUnusable() return err } else if resp.Err != nil { @@ -250,7 +245,7 @@ func (executor *remoteNodeExecutor) TaskManagerStatement(nodeId uint64, stmt inf return err } - if _, err := DecodeTLV(conn, &resp); err != nil { + if _, err := decodeTLV(conn, &resp); err != nil { conn.MarkUnusable() return err } else if resp.Err != "" { @@ -284,7 +279,7 @@ func (executor *remoteNodeExecutor) SeriesCardinality(nodeId uint64, database st } var resp SeriesCardinalityResponse - if _, err := DecodeTLV(conn, &resp); err != nil { + if _, err := decodeTLV(conn, &resp); err != nil { conn.MarkUnusable() return err } else if resp.Err != "" { @@ -318,7 +313,7 @@ func (executor *remoteNodeExecutor) MeasurementNames(nodeId uint64, database str } var resp MeasurementNamesResponse - if _, err := DecodeTLV(conn, &resp); err != nil { + if _, err := decodeTLV(conn, &resp); err != nil { conn.MarkUnusable() return err } else if resp.Err != "" { @@ -354,7 +349,7 @@ func (executor *remoteNodeExecutor) TagValues(nodeId uint64, shardIDs []uint64, } var resp TagValuesResponse - if _, err := DecodeTLV(conn, &resp); err != nil { + if _, err := decodeTLV(conn, &resp); err != nil { conn.MarkUnusable() return err } else if resp.Err != "" { @@ -388,7 +383,7 @@ func (executor *remoteNodeExecutor) TagKeys(nodeId uint64, shardIDs []uint64, co } var resp TagKeysResponse - if _, err := DecodeTLV(conn, &resp); err != nil { + if _, err := decodeTLV(conn, &resp); err != nil { conn.MarkUnusable() return err } else if resp.Err != "" { @@ -421,7 +416,7 @@ func (executor *remoteNodeExecutor) DeleteMeasurement(nodeId uint64, database, n } var resp DeleteMeasurementResponse - if _, err := DecodeTLV(conn, &resp); err != nil { + if _, err := decodeTLV(conn, &resp); err != nil { conn.MarkUnusable() return err } else if resp.Err != "" { @@ -452,7 +447,7 @@ func (executor *remoteNodeExecutor) DeleteDatabase(nodeId uint64, database strin } var resp DeleteDatabaseResponse - if _, err := DecodeTLV(conn, &resp); err != nil { + if _, err := decodeTLV(conn, &resp); err != nil { conn.MarkUnusable() return err } else if resp.Err != "" { @@ -485,7 +480,7 @@ func (executor *remoteNodeExecutor) DeleteSeries(nodeId uint64, database string, } var resp DeleteSeriesResponse - if _, err := DecodeTLV(conn, &resp); err != nil { + if _, err := decodeTLV(conn, &resp); err != nil { conn.MarkUnusable() return err } else if resp.Err != "" { @@ -581,3 +576,28 @@ func (r *iteratorReader) Close() error { r.consumeRest() return r.reader.Close() } + +// decodeTLV reads the type-length-value record from r and unmarshal it into v. +func decodeTLV(conn net.Conn, v encoding.BinaryUnmarshaler) (typ byte, err error) { + typ, err = ReadType(conn) + if err != nil { + return 0, err + } + if err := decodeLV(conn, v); err != nil { + return 0, err + } + return typ, nil +} + +// decodeLV reads the length-value record from r and unmarshal it into v. +func decodeLV(conn net.Conn, v encoding.BinaryUnmarshaler) error { + buf, err := ReadLV(conn, 3*time.Second) + if err != nil { + return err + } + + if err := v.UnmarshalBinary(buf); err != nil { + return err + } + return nil +} diff --git a/coordinator/request/request.go b/coordinator/request/request.go new file mode 100644 index 0000000..4c337ab --- /dev/null +++ b/coordinator/request/request.go @@ -0,0 +1,93 @@ +package request + +import ( + "encoding/binary" + "fmt" + "io" + "sync" +) + +const ( + MAX_BODY = 1024 * 1024 * 1024 +) + +var ( + bufferPool = sync.Pool{ + New: func() interface{} { + return make([]byte, 64) + }, + } +) + +type ClusterMessage struct { + Type byte + Data []byte +} + +type ClusterMessageReader struct { + data []byte +} + +func (reader *ClusterMessageReader) fetch() *ClusterMessage { + sz := len(reader.data) + if sz < 9 { + return nil + } + l := binary.BigEndian.Uint64(reader.data[1:9]) + if l > MAX_BODY { + panic(fmt.Sprint("Error decoding cluster request, got a huge size of ", l)) + } + consumed := int(l + 8 + 1) + if sz < consumed { + // not enough + return nil + } + + data := make([]byte, l) + copy(data, reader.data[9:consumed]) + req := &ClusterMessage{ + Type: reader.data[0], + Data: data, + } + // shift the buffer + for i := 0; i < sz-consumed; i++ { + reader.data[i] = reader.data[i+consumed] + } + reader.data = reader.data[:sz-consumed] + return req +} + +func (reader *ClusterMessageReader) Read(r io.Reader) (*ClusterMessage, error) { + if req := reader.fetch(); req != nil { + return req, nil + } + buf := bufferPool.Get().([]byte) + defer bufferPool.Put(buf) + for { + n, err := r.Read(buf) + if n > 0 { + reader.data = append(reader.data, buf[:n]...) + if req := reader.fetch(); req != nil { + return req, nil + } + } + if err != nil { + return nil, err + } + } +} + +// Len returns the length of data in buffer +func (reader *ClusterMessageReader) Len() int { + return len(reader.data) +} + +// Data returns unprocessed data +func (reader *ClusterMessageReader) Data() []byte { + if len(reader.data) == 0 { + return []byte{} + } + data := make([]byte, len(reader.data)) + copy(data, reader.data) + return data +} diff --git a/coordinator/request/request_test.go b/coordinator/request/request_test.go new file mode 100644 index 0000000..11ccc0c --- /dev/null +++ b/coordinator/request/request_test.go @@ -0,0 +1,181 @@ +package request_test + +import ( + "errors" + "testing" + + "github.com/angopher/chronus/coordinator/request" + "github.com/stretchr/testify/assert" +) + +type readFn func(p []byte) (n int, err error) + +type mockReader struct { + reader readFn +} + +func (r *mockReader) Read(p []byte) (n int, err error) { + return r.reader(p) +} + +func TestRequestReaderTimeout(t *testing.T) { + reader := &request.ClusterMessageReader{} + req, err := reader.Read(&mockReader{ + reader: func(p []byte) (n int, err error) { + return 0, errors.New("timeout") + }, + }) + assert.Nil(t, req) + assert.NotNil(t, err) + assert.Contains(t, "timeout", err.Error()) +} + +func TestRequestReaderPartial(t *testing.T) { + step := 0 + reader := &request.ClusterMessageReader{} + req, err := reader.Read(&mockReader{ + reader: func(p []byte) (n int, err error) { + switch step { + case 0: + p[0] = 1 + p[1] = 0 + step++ + return 2, nil + case 1: + p[0] = 0 + p[1] = 0 + p[2] = 0 + p[3] = 0 + p[4] = 0 + step++ + return 5, nil + } + return 0, errors.New("timeout") + }, + }) + assert.Nil(t, req) + assert.NotNil(t, err) +} + +func TestRequestReaderNormal(t *testing.T) { + step := 0 + reader := &request.ClusterMessageReader{} + req, err := reader.Read(&mockReader{ + reader: func(p []byte) (n int, err error) { + switch step { + case 0: + p[0] = 1 + p[1] = 0 + step++ + return 2, nil + case 1: + p[0] = 0 + p[1] = 0 + p[2] = 0 + p[3] = 0 + p[4] = 0 + step++ + return 5, nil + case 2: + p[0] = 0 + p[1] = 2 + p[2] = 12 + step++ + return 3, nil + case 3: + p[0] = 34 + p[1] = 22 + step++ + return 2, errors.New("timeout") + } + return 0, errors.New("timeout") + }, + }) + assert.NotNil(t, req) + assert.Nil(t, err) + assert.Equal(t, byte(1), req.Type) + assert.Equal(t, []byte{12, 34}, req.Data) + assert.Equal(t, []byte{22}, reader.Data()) +} + +func TestRequestReaderMore(t *testing.T) { + step := 0 + reader := &request.ClusterMessageReader{} + r := &mockReader{ + reader: func(p []byte) (n int, err error) { + switch step { + case 0: + p[0] = 1 + p[1] = 0 + step++ + return 2, nil + case 1: + p[0] = 0 + p[1] = 0 + p[2] = 0 + p[3] = 0 + p[4] = 0 + step++ + return 5, nil + case 2: + p[0] = 0 + p[1] = 2 + p[2] = 12 + step++ + return 3, nil + case 3: + p[0] = 34 + p[1] = 22 + p[2] = 0 + step++ + return 3, errors.New("timeout") + case 4: + p[0] = 0 + p[1] = 0 + p[2] = 0 + p[3] = 0 + p[4] = 0 + p[5] = 0 + p[6] = 0 + step++ + return 7, nil + case 5: + n := copy(p, []byte{ + 1, 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, 0, + }) + step++ + return n, nil + } + return 0, errors.New("timeout") + }, + } + req, err := reader.Read(r) + assert.NotNil(t, req) + assert.Nil(t, err) + assert.Equal(t, byte(1), req.Type) + assert.Equal(t, []byte{12, 34}, req.Data) + + req, err = reader.Read(r) + assert.NotNil(t, req) + assert.Nil(t, err) + assert.Equal(t, byte(22), req.Type) + assert.Equal(t, 0, len(req.Data)) + + req, err = reader.Read(r) + assert.NotNil(t, req) + assert.Nil(t, err) + assert.Equal(t, byte(1), req.Type) + assert.Equal(t, 0, len(req.Data)) + req, err = reader.Read(r) + assert.NotNil(t, req) + assert.Nil(t, err) + assert.Equal(t, byte(1), req.Type) + assert.Equal(t, 0, len(req.Data)) + req, err = reader.Read(r) + assert.NotNil(t, req) + assert.Nil(t, err) + assert.Equal(t, byte(1), req.Type) + assert.Equal(t, 0, len(req.Data)) +} diff --git a/coordinator/service.go b/coordinator/service.go index 8decb6a..56fe08b 100644 --- a/coordinator/service.go +++ b/coordinator/service.go @@ -1,25 +1,21 @@ package coordinator import ( - "context" "encoding" "encoding/binary" - "errors" "fmt" "io" "net" "strings" "sync" - "sync/atomic" "time" - "github.com/angopher/chronus/x" + "github.com/angopher/chronus/coordinator/internal" + "github.com/angopher/chronus/coordinator/request" errs "github.com/go-errors/errors" "github.com/influxdata/influxdb" "github.com/influxdata/influxdb/models" - "github.com/influxdata/influxdb/pkg/tracing" "github.com/influxdata/influxdb/query" - "github.com/influxdata/influxdb/tsdb" "github.com/influxdata/influxql" "go.uber.org/zap" @@ -41,37 +37,6 @@ const MaxMessageSize = 1024 * 1024 * 1024 // 1GB // MuxHeader is the header byte used in the TCP mux. const MuxHeader = 2 -// Statistics maintained by the cluster package -const ( - writeShardReq = "writeShardReq" - writeShardPointsReq = "writeShardPointsReq" - writeShardFail = "writeShardFail" - - createIteratorReq = "createIteratorReq" - createIteratorFail = "createIteratorFail" - - fieldDimensionsReq = "fieldDimensionsReq" - fieldDimensionsFail = "fieldDimensionsFail" - - tagKeysReq = "tagKeysReq" - tagKeysFail = "tagKeysFail" - - tagValuesReq = "tagValuesReq" - tagValuesFail = "tagValuesFail" - - measurementNamesReq = "measurementNamesReq" - measurementNamesFail = "measurementNamesFail" - - seriesCardinalityReq = "seriesCardinalityReq" - seriesCardinalityFail = "seriesCardinalityFail" - - iteratorCostReq = "iteratorCostReq" - iteratorCostFail = "iteratorCostFail" - - mapTypeReq = "mapTypeReq" - mapTypeFail = "mapTypeFail" -) - type ServerResponse interface { SetCode(int) SetMessage(string) @@ -95,7 +60,7 @@ type Service struct { TaskManager *query.TaskManager Logger *zap.Logger - stats *InternalServiceStatistics + stats *internal.InternalServiceStatistics } // NewService returns a new instance of Service. @@ -103,7 +68,7 @@ func NewService(c Config) *Service { return &Service{ closing: make(chan struct{}), //Logger: log.New(os.Stderr, "[cluster] ", log.LstdFlags), - stats: &InternalServiceStatistics{}, + stats: &internal.InternalServiceStatistics{}, Logger: zap.NewNop(), } } @@ -112,7 +77,7 @@ func NewService(c Config) *Service { func (s *Service) Open() error { s.Logger.Info("Starting cluster service") - // Begin serving conections. + // Begin serving connections. s.wg.Add(1) go s.serve() @@ -124,72 +89,6 @@ func (s *Service) WithLogger(log *zap.Logger) { s.Logger = log.With(zap.String("service", "cluster")) } -type InternalServiceStatistics struct { - WriteShardReq int64 - WriteShardPointsReq int64 - WriteShardFail int64 - - CreateIteratorReq int64 - CreateIteratorFail int64 - - FieldDimensionsReq int64 - FieldDimensionsFail int64 - - TagKeysReq int64 - TagKeysFail int64 - - TagValuesReq int64 - TagValuesFail int64 - - MeasurementNamesReq int64 - MeasurementNamesFail int64 - - SeriesCardinalityReq int64 - SeriesCardinalityFail int64 - - IteratorCostReq int64 - IteratorCostFail int64 - - MapTypeReq int64 - MapTypeFail int64 -} - -func (w *Service) Statistics(tags map[string]string) []models.Statistic { - return []models.Statistic{{ - Name: "coordinator_service", - Tags: tags, - Values: map[string]interface{}{ - writeShardReq: atomic.LoadInt64(&w.stats.WriteShardReq), - writeShardPointsReq: atomic.LoadInt64(&w.stats.WriteShardPointsReq), - writeShardFail: atomic.LoadInt64(&w.stats.WriteShardFail), - - createIteratorReq: atomic.LoadInt64(&w.stats.CreateIteratorReq), - createIteratorFail: atomic.LoadInt64(&w.stats.CreateIteratorFail), - - fieldDimensionsReq: atomic.LoadInt64(&w.stats.FieldDimensionsReq), - fieldDimensionsFail: atomic.LoadInt64(&w.stats.FieldDimensionsFail), - - tagKeysReq: atomic.LoadInt64(&w.stats.TagKeysReq), - tagKeysFail: atomic.LoadInt64(&w.stats.TagKeysFail), - - tagValuesReq: atomic.LoadInt64(&w.stats.TagValuesReq), - tagValuesFail: atomic.LoadInt64(&w.stats.TagValuesFail), - - measurementNamesReq: atomic.LoadInt64(&w.stats.MeasurementNamesReq), - measurementNamesFail: atomic.LoadInt64(&w.stats.MeasurementNamesFail), - - seriesCardinalityReq: atomic.LoadInt64(&w.stats.SeriesCardinalityReq), - seriesCardinalityFail: atomic.LoadInt64(&w.stats.SeriesCardinalityFail), - - iteratorCostReq: atomic.LoadInt64(&w.stats.IteratorCostReq), - iteratorCostFail: atomic.LoadInt64(&w.stats.IteratorCostFail), - - mapTypeReq: atomic.LoadInt64(&w.stats.MapTypeReq), - mapTypeFail: atomic.LoadInt64(&w.stats.MapTypeFail), - }, - }} -} - // serve accepts connections from the listener and handles them. func (s *Service) serve() { defer s.wg.Done() @@ -241,7 +140,7 @@ func (s *Service) Close() error { return nil } -func (s *Service) handle(conn net.Conn, typ byte, data []byte) (respType byte, resp encoding.BinaryMarshaler, err error) { +func (s *Service) handleRequest(typ byte, data []byte) (respType byte, resp encoding.BinaryMarshaler, err error) { // Delegate message processing by type. switch typ { case writeShardRequestMessage: @@ -307,7 +206,7 @@ func (s *Service) handle(conn net.Conn, typ byte, data []byte) (respType byte, r return } -func (s *Service) writeClusterResponse(conn net.Conn, respType byte, resp encoding.BinaryMarshaler) (err error) { +func (s *Service) writeClusterResponse(conn io.Writer, respType byte, resp encoding.BinaryMarshaler) (err error) { // Marshal response to binary. buf, err := resp.MarshalBinary() if err != nil { @@ -322,6 +221,16 @@ func (s *Service) writeClusterResponse(conn net.Conn, respType byte, resp encodi return } +func canContinue(err error) bool { + if err == io.EOF || err.Error() == "use of closed network connection" { + return false + } + if err, ok := err.(net.Error); ok && err.Timeout() { + return true + } + return false +} + // handleConn services an individual TCP connection. func (s *Service) handleConn(conn net.Conn) { // Ensure connection is closed when service is closed. @@ -330,6 +239,7 @@ func (s *Service) handleConn(conn net.Conn) { s.Logger.Info(fmt.Sprintf("close remote connection from %+v", conn.RemoteAddr())) }() s.Logger.Info(fmt.Sprintf("accept remote connection from %+v", conn.RemoteAddr())) + requestReader := &request.ClusterMessageReader{} for { select { case <-s.closing: @@ -337,30 +247,29 @@ func (s *Service) handleConn(conn net.Conn) { default: } conn.SetReadDeadline(time.Now().Add(500 * time.Millisecond)) - typ, data, err := ReadTLV(conn) - conn.SetDeadline(time.Time{}) + req, err := requestReader.Read(conn) + conn.SetReadDeadline(time.Time{}) if err != nil { - if err == io.EOF || err.Error() == "use of closed network connection" { - break - } - if err, ok := err.(net.Error); ok && err.Timeout() { + if canContinue(err) { continue } - s.Logger.Error("read error", zap.Error(err)) + if err != io.EOF { + s.Logger.Error("read error", zap.Error(err)) + } break } - if typ == 0 { + if req == nil { continue } - if typ == createIteratorRequestMessage { + if req.Type == createIteratorRequestMessage { // iterator is different from other requests - ioError := s.processCreateIteratorRequest(conn, data) + ioError := s.processCreateIteratorRequest(conn, req.Data) if ioError { break } continue } - respType, resp, err := s.handle(conn, typ, data) + respType, resp, err := s.handleRequest(req.Type, req.Data) if resp == nil { continue } @@ -372,283 +281,6 @@ func (s *Service) handleConn(conn net.Conn) { } } -func (s *Service) processTaskManagerRequest(buf []byte) (*TaskManagerStatementResponse, error) { - var ( - resp TaskManagerStatementResponse - err error - ) - if err = func() error { - var req TaskManagerStatementRequest - if err := req.UnmarshalBinary(buf); err != nil { - return err - } - - stmt, err := influxql.ParseStatement(req.Statement()) - if err != nil { - return err - } - - recvCtx := &query.ExecutionContext{ - Context: context.Background(), - Results: make(chan *query.Result, 1), - } - err = s.TaskManager.ExecuteStatement(stmt, recvCtx) - if err != nil { - return err - } - resp.Result = *(<-recvCtx.Results) - return nil - }(); err != nil { - resp.Err = err.Error() - } - return &resp, err -} - -func (s *Service) processMapTypeRequest(buf []byte) (*MapTypeResponse, error) { - var ( - resp MapTypeResponse - err error - ) - atomic.AddInt64(&s.stats.MapTypeReq, 1) - if err = func() error { - var req MapTypeRequest - if err := req.UnmarshalBinary(buf); err != nil { - return err - } - - if len(req.Sources) == 0 { - return errors.New(fmt.Sprintf("bad request %+v: no sources", req)) - } - m := req.Sources[0].(*influxql.Measurement) - - sg := s.TSDBStore.ShardGroup(req.ShardIDs) - - var names []string - if m.Regex != nil && m.Name != "_series" && m.Name != "_fieldKeys" && m.Name != "_tagKeys" { - names = sg.MeasurementsByRegex(m.Regex.Val) - } else { - names = []string{m.Name} - } - - typ := influxql.Unknown - for _, name := range names { - if m.SystemIterator != "" { - name = m.SystemIterator - } - t := sg.MapType(name, req.Field) - if typ.LessThan(t) { - typ = t - } - } - resp.DataType = typ - return nil - }(); err != nil { - atomic.AddInt64(&s.stats.MapTypeFail, 1) - resp.Err = err.Error() - } - return &resp, err -} - -func (s *Service) processIteratorCostRequest(buf []byte) (*IteratorCostResponse, error) { - var ( - resp IteratorCostResponse - err error - ) - atomic.AddInt64(&s.stats.IteratorCostReq, 1) - if err = func() error { - var req IteratorCostRequest - if err := req.UnmarshalBinary(buf); err != nil { - return err - } - - if len(req.Sources) == 0 { - return errors.New(fmt.Sprintf("bad request %+v: no sources", req)) - } - m := req.Sources[0].(*influxql.Measurement) - opt := req.Opt - - sg := s.TSDBStore.ShardGroup(req.ShardIDs) - if m.Regex != nil { - resp.Cost, err = func() (query.IteratorCost, error) { - var costs query.IteratorCost - measurements := sg.MeasurementsByRegex(m.Regex.Val) - for _, measurement := range measurements { - c, err := sg.IteratorCost(measurement, opt) - if err != nil { - return c, err - } - costs = costs.Combine(c) - } - return costs, nil - }() - } else { - resp.Cost, err = sg.IteratorCost(m.Name, opt) - } - return err - }(); err != nil { - atomic.AddInt64(&s.stats.IteratorCostFail, 1) - resp.Err = err.Error() - } - return &resp, err -} - -func (s *Service) processDeleteMeasurementRequest(buf []byte) (*DeleteMeasurementResponse, error) { - var ( - resp DeleteMeasurementResponse - err error - ) - if err = func() error { - var req DeleteMeasurementRequest - if err := req.UnmarshalBinary(buf); err != nil { - return err - } - return s.TSDBStore.DeleteMeasurement(req.Database, req.Name) - }(); err != nil { - resp.Err = err.Error() - } - return &resp, err -} - -func (s *Service) processDeleteDatabaseRequest(buf []byte) (*DeleteDatabaseResponse, error) { - var ( - resp DeleteDatabaseResponse - err error - ) - if err = func() error { - var req DeleteDatabaseRequest - if err := req.UnmarshalBinary(buf); err != nil { - return err - } - return s.TSDBStore.DeleteDatabase(req.Database) - }(); err != nil { - resp.Err = err.Error() - } - return &resp, err -} - -func (s *Service) processDeleteSeriesRequest(buf []byte) (*DeleteSeriesResponse, error) { - var ( - resp DeleteSeriesResponse - err error - ) - if err = func() error { - var req DeleteSeriesRequest - if err := req.UnmarshalBinary(buf); err != nil { - return err - } - - cond := influxql.Reduce(req.Cond, &influxql.NowValuer{Now: time.Now().UTC()}) - err := s.TSDBStore.DeleteSeries(req.Database, req.Sources, cond) - return err - }(); err != nil { - resp.Err = err.Error() - } - return &resp, err -} - -func (s *Service) processSeriesCardinalityRequest(buf []byte) (*SeriesCardinalityResponse, error) { - var ( - resp SeriesCardinalityResponse - err error - ) - atomic.AddInt64(&s.stats.SeriesCardinalityReq, 1) - if err = func() error { - var req SeriesCardinalityRequest - if err := req.UnmarshalBinary(buf); err != nil { - return err - } - - n, err := s.TSDBStore.SeriesCardinality(req.Database) - resp.N = n - return err - }(); err != nil { - atomic.AddInt64(&s.stats.SeriesCardinalityFail, 1) - resp.Err = err.Error() - } - return &resp, err -} - -func (s *Service) processMeasurementNamesRequest(buf []byte) (*MeasurementNamesResponse, error) { - var ( - resp MeasurementNamesResponse - err error - ) - if err = func() error { - var req MeasurementNamesRequest - if err := req.UnmarshalBinary(buf); err != nil { - return err - } - - var err error - resp.Names, err = s.TSDBStore.MeasurementNames(nil, req.Database, req.Cond) - return err - }(); err != nil { - atomic.AddInt64(&s.stats.MeasurementNamesFail, 1) - resp.Err = err.Error() - } - return &resp, err -} - -func (s *Service) processTagKeysRequest(buf []byte) (*TagKeysResponse, error) { - var ( - resp TagKeysResponse - err error - ) - atomic.AddInt64(&s.stats.TagKeysReq, 1) - if err = func() error { - var req TagKeysRequest - if err := req.UnmarshalBinary(buf); err != nil { - return err - } - - var err error - resp.TagKeys, err = s.TSDBStore.TagKeys(nil, req.ShardIDs, req.Cond) - return err - }(); err != nil { - atomic.AddInt64(&s.stats.TagKeysFail, 1) - resp.Err = err.Error() - } - return &resp, err -} - -func (s *Service) processTagValuesRequest(buf []byte) (*TagValuesResponse, error) { - var ( - resp TagValuesResponse - err error - ) - atomic.AddInt64(&s.stats.TagValuesReq, 1) - if err = func() error { - var req TagValuesRequest - if err := req.UnmarshalBinary(buf); err != nil { - return err - } - - var err error - resp.TagValues, err = s.TSDBStore.TagValues(nil, req.ShardIDs, req.Cond) - return err - }(); err != nil { - atomic.AddInt64(&s.stats.TagValuesFail, 1) - resp.Err = err.Error() - } - return &resp, err -} - -func (s *Service) processExecuteStatementRequest(buf []byte) error { - // Unmarshal the request. - var req ExecuteStatementRequest - if err := req.UnmarshalBinary(buf); err != nil { - return err - } - - // Parse the InfluxQL statement. - stmt, err := influxql.ParseStatement(req.Statement()) - if err != nil { - return err - } - - return s.executeStatement(stmt, req.Database()) -} - func (s *Service) executeStatement(stmt influxql.Statement, database string) error { switch t := stmt.(type) { case *influxql.DropDatabaseStatement: @@ -664,237 +296,8 @@ func (s *Service) executeStatement(stmt influxql.Statement, database string) err } } -func (s *Service) processWriteShardRequest(buf []byte) error { - // Build request - var req WriteShardRequest - if err := req.UnmarshalBinary(buf); err != nil { - return err - } - - points := req.Points() - // stats - atomic.AddInt64(&s.stats.WriteShardReq, 1) - atomic.AddInt64(&s.stats.WriteShardPointsReq, int64(len(points))) - err := s.TSDBStore.WriteToShard(req.ShardID(), points) - - // We may have received a write for a shard that we don't have locally because the - // sending node may have just created the shard (via the metastore) and the write - // arrived before the local store could create the shard. In this case, we need - // to check the metastore to determine what database and retention policy this - // shard should reside within. - if err == tsdb.ErrShardNotFound { - db, rp := req.Database(), req.RetentionPolicy() - if db == "" || rp == "" { - s.Logger.Error("drop write request: no database or rentention policy received\n", - zap.Uint64("shard", req.ShardID())) - return nil - } - - err = s.TSDBStore.CreateShard(req.Database(), req.RetentionPolicy(), req.ShardID(), true) //enable what mean? - if err != nil { - atomic.AddInt64(&s.stats.WriteShardFail, 1) - return fmt.Errorf("create shard %d: %s", req.ShardID(), err) - } - - err = s.TSDBStore.WriteToShard(req.ShardID(), points) - if err != nil { - atomic.AddInt64(&s.stats.WriteShardFail, 1) - return fmt.Errorf("write shard %d: %s", req.ShardID(), err) - } - } - - if err != nil { - atomic.AddInt64(&s.stats.WriteShardFail, 1) - return fmt.Errorf("write shard %d: %s", req.ShardID(), err) - } - - return nil -} - -func (s *Service) processCreateIteratorRequest(conn net.Conn, buf []byte) (ioError bool) { - var itr query.Iterator - var trace *tracing.Trace - var span *tracing.Span - respType := createIteratorResponseMessage - if err := func() error { - // Parse request. - var req CreateIteratorRequest - if err := req.UnmarshalBinary(buf); err != nil { - return err - } - - ctx := context.Background() - if req.SpanContex != nil { - trace, span = tracing.NewTraceFromSpan(fmt.Sprintf("remote_node_id: %d", s.Node.ID), *req.SpanContex) - ctx = tracing.NewContextWithTrace(ctx, trace) - ctx = tracing.NewContextWithSpan(ctx, span) - //var aux query.Iterators - //ctx = query.NewContextWithIterators(ctx, &aux) - } - - var err error - - //TODO:求证是否直接使用&req.Measurement会不会导致内存错误 - m := new(influxql.Measurement) - *m = req.Measurement - - sg := s.TSDBStore.ShardGroup(req.ShardIDs) - if m.Regex != nil { - measurements := sg.MeasurementsByRegex(m.Regex.Val) - inputs := make([]query.Iterator, 0, len(measurements)) - if err := func() error { - for _, measurement := range measurements { - mm := m.Clone() - mm.Name = measurement - input, err := sg.CreateIterator(ctx, mm, req.Opt) - if err != nil { - return err - } - inputs = append(inputs, input) - } - return nil - }(); err != nil { - query.Iterators(inputs).Close() - return err - } - - itr, err = query.Iterators(inputs).Merge(req.Opt) - } else { - itr, err = sg.CreateIterator(ctx, m, req.Opt) - } - - if err != nil { - return err - } - // Generate a single iterator from all shards. - //i, err := influxql.IteratorCreators(ics).CreateIterator(req.Opt) - - return nil - }(); err != nil { - atomic.AddInt64(&s.stats.CreateIteratorFail, 1) - if itr != nil { - itr.Close() - } - s.Logger.Error("error reading CreateIterator request fail", zap.Error(err)) - if err = EncodeTLV(conn, respType, &CreateIteratorResponse{Err: err}); err != nil { - s.Logger.Error("CreateIteratorRequest EncodeTLV fail", zap.Error(err)) - ioError = true - } - return - } - - dataType := influxql.Unknown - switch itr.(type) { - case query.FloatIterator: - dataType = influxql.Float - case query.IntegerIterator: - dataType = influxql.Integer - case query.StringIterator: - dataType = influxql.String - case query.BooleanIterator: - dataType = influxql.Boolean - } - - seriesN := 0 - if itr != nil { - defer itr.Close() - seriesN = itr.Stats().SeriesN - } - // Encode success response. - itrTerminator := x.RandBytes(8) - if err := EncodeTLV(conn, respType, &CreateIteratorResponse{DataType: dataType, SeriesN: seriesN, Termination: itrTerminator}); err != nil { - s.Logger.Error("error writing CreateIterator response, EncodeTLV fail", zap.Error(err)) - atomic.AddInt64(&s.stats.CreateIteratorFail, 1) - ioError = true - return - } - defer func() { - // Write termination of iterator - if _, err := conn.Write(itrTerminator); err != nil { - ioError = true - } - }() - - // Exit if no iterator was produced. - if itr == nil { - return - } - - // Stream iterator to connection. - encoder := query.NewIteratorEncoder(conn) - if err := encoder.EncodeIterator(itr); err != nil { - s.Logger.Error("encoding CreateIterator iterator fail", zap.Error(err)) - atomic.AddInt64(&s.stats.CreateIteratorFail, 1) - ioError = true - return - } - - if trace != nil { - span.Finish() - if err := encoder.EncodeTrace(trace); err != nil { - s.Logger.Error("EncodeTrace fail", zap.Error(err)) - atomic.AddInt64(&s.stats.CreateIteratorFail, 1) - ioError = true - return - } - } - return -} - -func (s *Service) processFieldDimensionsRequest(buf []byte) (*FieldDimensionsResponse, error) { - var ( - err error - ) - var fields map[string]influxql.DataType - var dimensions map[string]struct{} - atomic.AddInt64(&s.stats.FieldDimensionsReq, 1) - if err = func() error { - // Parse request. - var req FieldDimensionsRequest - if err := req.UnmarshalBinary(buf); err != nil { - return err - } - - // Generate a single iterator from all shards. - measurements := make([]string, 0) - ms := req.Sources.Measurements() - for _, m := range ms { - if m.Regex != nil { - measurements = s.TSDBStore.ShardGroup(req.ShardIDs).MeasurementsByRegex(m.Regex.Val) - } else { - measurements = append(measurements, m.Name) - } - } - - f, d, err := s.TSDBStore.ShardGroup(req.ShardIDs).FieldDimensions(measurements) - if err != nil { - return err - } - fields, dimensions = f, d - - return nil - }(); err != nil { - atomic.AddInt64(&s.stats.FieldDimensionsFail, 1) - return &FieldDimensionsResponse{Err: err}, err - } - return &FieldDimensionsResponse{ - Fields: fields, - Dimensions: dimensions, - }, err -} - -// ReadTLV reads a type-length-value record from r. -func ReadTLV(r io.Reader) (byte, []byte, error) { - typ, err := ReadType(r) - if err != nil { - return 0, nil, err - } - - buf, err := ReadLV(r) - if err != nil { - return 0, nil, err - } - return typ, buf, err +func (s *Service) Statistics(tags map[string]string) []models.Statistic { + return internal.Statistics(s.stats, tags) } // ReadType reads the type from a TLV record. @@ -907,10 +310,12 @@ func ReadType(r io.Reader) (byte, error) { } // ReadLV reads the length-value from a TLV record. -func ReadLV(r io.Reader) ([]byte, error) { +func ReadLV(conn net.Conn, timeout time.Duration) ([]byte, error) { // Read the size of the message. var sz int64 - if err := binary.Read(r, binary.BigEndian, &sz); err != nil { + conn.SetReadDeadline(time.Now().Add(timeout)) + defer conn.SetReadDeadline(time.Time{}) + if err := binary.Read(conn, binary.BigEndian, &sz); err != nil { return nil, err } @@ -925,7 +330,8 @@ func ReadLV(r io.Reader) ([]byte, error) { // Read the value. buf := make([]byte, sz) - if _, err := io.ReadFull(r, buf); err != nil { + conn.SetReadDeadline(time.Now().Add(timeout)) + if _, err := io.ReadFull(conn, buf); err != nil { return nil, err } @@ -988,28 +394,3 @@ func EncodeLV(w io.Writer, v encoding.BinaryMarshaler) error { } return nil } - -// DecodeTLV reads the type-length-value record from r and unmarshals it into v. -func DecodeTLV(r io.Reader, v encoding.BinaryUnmarshaler) (typ byte, err error) { - typ, err = ReadType(r) - if err != nil { - return 0, err - } - if err := DecodeLV(r, v); err != nil { - return 0, err - } - return typ, nil -} - -// DecodeLV reads the length-value record from r and unmarshals it into v. -func DecodeLV(r io.Reader, v encoding.BinaryUnmarshaler) error { - buf, err := ReadLV(r) - if err != nil { - return err - } - - if err := v.UnmarshalBinary(buf); err != nil { - return err - } - return nil -} diff --git a/coordinator/service_handlers.go b/coordinator/service_handlers.go new file mode 100644 index 0000000..3175240 --- /dev/null +++ b/coordinator/service_handlers.go @@ -0,0 +1,513 @@ +package coordinator + +import ( + "context" + "errors" + "fmt" + "io" + "sync/atomic" + "time" + + "github.com/angopher/chronus/x" + "github.com/influxdata/influxdb/pkg/tracing" + "github.com/influxdata/influxdb/query" + "github.com/influxdata/influxdb/tsdb" + "github.com/influxdata/influxql" + "go.uber.org/zap" +) + +func (s *Service) processTaskManagerRequest(buf []byte) (*TaskManagerStatementResponse, error) { + var ( + resp TaskManagerStatementResponse + err error + ) + if err = func() error { + var req TaskManagerStatementRequest + if err := req.UnmarshalBinary(buf); err != nil { + return err + } + + stmt, err := influxql.ParseStatement(req.Statement()) + if err != nil { + return err + } + + recvCtx := &query.ExecutionContext{ + Context: context.Background(), + Results: make(chan *query.Result, 1), + } + err = s.TaskManager.ExecuteStatement(stmt, recvCtx) + if err != nil { + return err + } + resp.Result = *(<-recvCtx.Results) + return nil + }(); err != nil { + resp.Err = err.Error() + } + return &resp, err +} + +func (s *Service) processMapTypeRequest(buf []byte) (*MapTypeResponse, error) { + var ( + resp MapTypeResponse + err error + ) + atomic.AddInt64(&s.stats.MapTypeReq, 1) + if err = func() error { + var req MapTypeRequest + if err := req.UnmarshalBinary(buf); err != nil { + return err + } + + if len(req.Sources) == 0 { + return errors.New(fmt.Sprintf("bad request %+v: no sources", req)) + } + m := req.Sources[0].(*influxql.Measurement) + + sg := s.TSDBStore.ShardGroup(req.ShardIDs) + + var names []string + if m.Regex != nil && m.Name != "_series" && m.Name != "_fieldKeys" && m.Name != "_tagKeys" { + names = sg.MeasurementsByRegex(m.Regex.Val) + } else { + names = []string{m.Name} + } + + typ := influxql.Unknown + for _, name := range names { + if m.SystemIterator != "" { + name = m.SystemIterator + } + t := sg.MapType(name, req.Field) + if typ.LessThan(t) { + typ = t + } + } + resp.DataType = typ + return nil + }(); err != nil { + atomic.AddInt64(&s.stats.MapTypeFail, 1) + resp.Err = err.Error() + } + return &resp, err +} + +func (s *Service) processIteratorCostRequest(buf []byte) (*IteratorCostResponse, error) { + var ( + resp IteratorCostResponse + err error + ) + atomic.AddInt64(&s.stats.IteratorCostReq, 1) + if err = func() error { + var req IteratorCostRequest + if err := req.UnmarshalBinary(buf); err != nil { + return err + } + + if len(req.Sources) == 0 { + return errors.New(fmt.Sprintf("bad request %+v: no sources", req)) + } + m := req.Sources[0].(*influxql.Measurement) + opt := req.Opt + + sg := s.TSDBStore.ShardGroup(req.ShardIDs) + if m.Regex != nil { + resp.Cost, err = func() (query.IteratorCost, error) { + var costs query.IteratorCost + measurements := sg.MeasurementsByRegex(m.Regex.Val) + for _, measurement := range measurements { + c, err := sg.IteratorCost(measurement, opt) + if err != nil { + return c, err + } + costs = costs.Combine(c) + } + return costs, nil + }() + } else { + resp.Cost, err = sg.IteratorCost(m.Name, opt) + } + return err + }(); err != nil { + atomic.AddInt64(&s.stats.IteratorCostFail, 1) + resp.Err = err.Error() + } + return &resp, err +} + +func (s *Service) processDeleteMeasurementRequest(buf []byte) (*DeleteMeasurementResponse, error) { + var ( + resp DeleteMeasurementResponse + err error + ) + if err = func() error { + var req DeleteMeasurementRequest + if err := req.UnmarshalBinary(buf); err != nil { + return err + } + return s.TSDBStore.DeleteMeasurement(req.Database, req.Name) + }(); err != nil { + resp.Err = err.Error() + } + return &resp, err +} + +func (s *Service) processDeleteDatabaseRequest(buf []byte) (*DeleteDatabaseResponse, error) { + var ( + resp DeleteDatabaseResponse + err error + ) + if err = func() error { + var req DeleteDatabaseRequest + if err := req.UnmarshalBinary(buf); err != nil { + return err + } + return s.TSDBStore.DeleteDatabase(req.Database) + }(); err != nil { + resp.Err = err.Error() + } + return &resp, err +} + +func (s *Service) processDeleteSeriesRequest(buf []byte) (*DeleteSeriesResponse, error) { + var ( + resp DeleteSeriesResponse + err error + ) + if err = func() error { + var req DeleteSeriesRequest + if err := req.UnmarshalBinary(buf); err != nil { + return err + } + + cond := influxql.Reduce(req.Cond, &influxql.NowValuer{Now: time.Now().UTC()}) + err := s.TSDBStore.DeleteSeries(req.Database, req.Sources, cond) + return err + }(); err != nil { + resp.Err = err.Error() + } + return &resp, err +} + +func (s *Service) processSeriesCardinalityRequest(buf []byte) (*SeriesCardinalityResponse, error) { + var ( + resp SeriesCardinalityResponse + err error + ) + atomic.AddInt64(&s.stats.SeriesCardinalityReq, 1) + if err = func() error { + var req SeriesCardinalityRequest + if err := req.UnmarshalBinary(buf); err != nil { + return err + } + + n, err := s.TSDBStore.SeriesCardinality(req.Database) + resp.N = n + return err + }(); err != nil { + atomic.AddInt64(&s.stats.SeriesCardinalityFail, 1) + resp.Err = err.Error() + } + return &resp, err +} + +func (s *Service) processMeasurementNamesRequest(buf []byte) (*MeasurementNamesResponse, error) { + var ( + resp MeasurementNamesResponse + err error + ) + if err = func() error { + var req MeasurementNamesRequest + if err := req.UnmarshalBinary(buf); err != nil { + return err + } + + var err error + resp.Names, err = s.TSDBStore.MeasurementNames(nil, req.Database, req.Cond) + return err + }(); err != nil { + atomic.AddInt64(&s.stats.MeasurementNamesFail, 1) + resp.Err = err.Error() + } + return &resp, err +} + +func (s *Service) processTagKeysRequest(buf []byte) (*TagKeysResponse, error) { + var ( + resp TagKeysResponse + err error + ) + atomic.AddInt64(&s.stats.TagKeysReq, 1) + if err = func() error { + var req TagKeysRequest + if err := req.UnmarshalBinary(buf); err != nil { + return err + } + + var err error + resp.TagKeys, err = s.TSDBStore.TagKeys(nil, req.ShardIDs, req.Cond) + return err + }(); err != nil { + atomic.AddInt64(&s.stats.TagKeysFail, 1) + resp.Err = err.Error() + } + return &resp, err +} + +func (s *Service) processTagValuesRequest(buf []byte) (*TagValuesResponse, error) { + var ( + resp TagValuesResponse + err error + ) + atomic.AddInt64(&s.stats.TagValuesReq, 1) + if err = func() error { + var req TagValuesRequest + if err := req.UnmarshalBinary(buf); err != nil { + return err + } + + var err error + resp.TagValues, err = s.TSDBStore.TagValues(nil, req.ShardIDs, req.Cond) + return err + }(); err != nil { + atomic.AddInt64(&s.stats.TagValuesFail, 1) + resp.Err = err.Error() + } + return &resp, err +} + +func (s *Service) processExecuteStatementRequest(buf []byte) error { + // Unmarshal the request. + var req ExecuteStatementRequest + if err := req.UnmarshalBinary(buf); err != nil { + return err + } + + // Parse the InfluxQL statement. + stmt, err := influxql.ParseStatement(req.Statement()) + if err != nil { + return err + } + + return s.executeStatement(stmt, req.Database()) +} + +func (s *Service) processWriteShardRequest(buf []byte) error { + // Build request + var req WriteShardRequest + if err := req.UnmarshalBinary(buf); err != nil { + return err + } + + points := req.Points() + // stats + atomic.AddInt64(&s.stats.WriteShardReq, 1) + atomic.AddInt64(&s.stats.WriteShardPointsReq, int64(len(points))) + err := s.TSDBStore.WriteToShard(req.ShardID(), points) + + // We may have received a write for a shard that we don't have locally because the + // sending node may have just created the shard (via the metastore) and the write + // arrived before the local store could create the shard. In this case, we need + // to check the metastore to determine what database and retention policy this + // shard should reside within. + if err == tsdb.ErrShardNotFound { + db, rp := req.Database(), req.RetentionPolicy() + if db == "" || rp == "" { + s.Logger.Error("drop write request: no database or retention policy received\n", + zap.Uint64("shard", req.ShardID())) + return nil + } + + err = s.TSDBStore.CreateShard(req.Database(), req.RetentionPolicy(), req.ShardID(), true) //enable what mean? + if err != nil { + atomic.AddInt64(&s.stats.WriteShardFail, 1) + return fmt.Errorf("create shard %d: %s", req.ShardID(), err) + } + + err = s.TSDBStore.WriteToShard(req.ShardID(), points) + if err != nil { + atomic.AddInt64(&s.stats.WriteShardFail, 1) + return fmt.Errorf("write shard %d: %s", req.ShardID(), err) + } + } + + if err != nil { + atomic.AddInt64(&s.stats.WriteShardFail, 1) + return fmt.Errorf("write shard %d: %s", req.ShardID(), err) + } + + return nil +} + +func (s *Service) processCreateIteratorRequest(conn io.ReadWriter, buf []byte) (ioError bool) { + var itr query.Iterator + var trace *tracing.Trace + var span *tracing.Span + respType := createIteratorResponseMessage + if err := func() error { + // Parse request. + var req CreateIteratorRequest + if err := req.UnmarshalBinary(buf); err != nil { + return err + } + + ctx := context.Background() + if req.SpanContex != nil { + trace, span = tracing.NewTraceFromSpan(fmt.Sprintf("remote_node_id: %d", s.Node.ID), *req.SpanContex) + ctx = tracing.NewContextWithTrace(ctx, trace) + ctx = tracing.NewContextWithSpan(ctx, span) + //var aux query.Iterators + //ctx = query.NewContextWithIterators(ctx, &aux) + } + + var err error + + //TODO:求证是否直接使用&req.Measurement会不会导致内存错误 + m := new(influxql.Measurement) + *m = req.Measurement + + sg := s.TSDBStore.ShardGroup(req.ShardIDs) + if m.Regex != nil { + measurements := sg.MeasurementsByRegex(m.Regex.Val) + inputs := make([]query.Iterator, 0, len(measurements)) + if err := func() error { + for _, measurement := range measurements { + mm := m.Clone() + mm.Name = measurement + input, err := sg.CreateIterator(ctx, mm, req.Opt) + if err != nil { + return err + } + inputs = append(inputs, input) + } + return nil + }(); err != nil { + query.Iterators(inputs).Close() + return err + } + + itr, err = query.Iterators(inputs).Merge(req.Opt) + } else { + itr, err = sg.CreateIterator(ctx, m, req.Opt) + } + + if err != nil { + return err + } + // Generate a single iterator from all shards. + //i, err := influxql.IteratorCreators(ics).CreateIterator(req.Opt) + + return nil + }(); err != nil { + atomic.AddInt64(&s.stats.CreateIteratorFail, 1) + if itr != nil { + itr.Close() + } + s.Logger.Error("error reading CreateIterator request fail", zap.Error(err)) + if err = EncodeTLV(conn, respType, &CreateIteratorResponse{Err: err}); err != nil { + s.Logger.Error("CreateIteratorRequest EncodeTLV fail", zap.Error(err)) + ioError = true + } + return + } + + dataType := influxql.Unknown + switch itr.(type) { + case query.FloatIterator: + dataType = influxql.Float + case query.IntegerIterator: + dataType = influxql.Integer + case query.StringIterator: + dataType = influxql.String + case query.BooleanIterator: + dataType = influxql.Boolean + } + + seriesN := 0 + if itr != nil { + defer itr.Close() + seriesN = itr.Stats().SeriesN + } + // Encode success response. + itrTerminator := x.RandBytes(8) + if err := EncodeTLV(conn, respType, &CreateIteratorResponse{DataType: dataType, SeriesN: seriesN, Termination: itrTerminator}); err != nil { + s.Logger.Error("error writing CreateIterator response, EncodeTLV fail", zap.Error(err)) + atomic.AddInt64(&s.stats.CreateIteratorFail, 1) + ioError = true + return + } + defer func() { + // Write termination of iterator + if _, err := conn.Write(itrTerminator); err != nil { + ioError = true + } + }() + + // Exit if no iterator was produced. + if itr == nil { + return + } + + // Stream iterator to connection. + encoder := query.NewIteratorEncoder(conn) + if err := encoder.EncodeIterator(itr); err != nil { + s.Logger.Error("encoding CreateIterator iterator fail", zap.Error(err)) + atomic.AddInt64(&s.stats.CreateIteratorFail, 1) + ioError = true + return + } + + if trace != nil { + span.Finish() + if err := encoder.EncodeTrace(trace); err != nil { + s.Logger.Error("EncodeTrace fail", zap.Error(err)) + atomic.AddInt64(&s.stats.CreateIteratorFail, 1) + ioError = true + return + } + } + return +} + +func (s *Service) processFieldDimensionsRequest(buf []byte) (*FieldDimensionsResponse, error) { + var ( + err error + ) + var fields map[string]influxql.DataType + var dimensions map[string]struct{} + atomic.AddInt64(&s.stats.FieldDimensionsReq, 1) + if err = func() error { + // Parse request. + var req FieldDimensionsRequest + if err := req.UnmarshalBinary(buf); err != nil { + return err + } + + // Generate a single iterator from all shards. + measurements := make([]string, 0) + ms := req.Sources.Measurements() + for _, m := range ms { + if m.Regex != nil { + measurements = s.TSDBStore.ShardGroup(req.ShardIDs).MeasurementsByRegex(m.Regex.Val) + } else { + measurements = append(measurements, m.Name) + } + } + + f, d, err := s.TSDBStore.ShardGroup(req.ShardIDs).FieldDimensions(measurements) + if err != nil { + return err + } + fields, dimensions = f, d + + return nil + }(); err != nil { + atomic.AddInt64(&s.stats.FieldDimensionsFail, 1) + return &FieldDimensionsResponse{Err: err}, err + } + return &FieldDimensionsResponse{ + Fields: fields, + Dimensions: dimensions, + }, err +} diff --git a/coordinator/shard_writer.go b/coordinator/shard_writer.go index 66e15fc..636948b 100644 --- a/coordinator/shard_writer.go +++ b/coordinator/shard_writer.go @@ -4,6 +4,7 @@ import ( "fmt" "time" + "github.com/angopher/chronus/coordinator/request" "github.com/influxdata/influxdb/models" "github.com/influxdata/influxdb/services/meta" "go.uber.org/zap" @@ -100,15 +101,15 @@ func (w *ShardWriter) WriteShard(shardID, ownerID uint64, points []models.Point) return nil } - // Build write request. - var request WriteShardRequest - request.SetShardID(shardID) - request.SetDatabase(db) - request.SetRetentionPolicy(rp) - request.AddPoints(points) + // Build write writeReq. + var writeReq WriteShardRequest + writeReq.SetShardID(shardID) + writeReq.SetDatabase(db) + writeReq.SetRetentionPolicy(rp) + writeReq.AddPoints(points) // Marshal into protocol buffers. - buf, err := request.MarshalBinary() + buf, err := writeReq.MarshalBinary() if err != nil { return err } @@ -121,8 +122,9 @@ func (w *ShardWriter) WriteShard(shardID, ownerID uint64, points []models.Point) } // Read the response. + requestReader := &request.ClusterMessageReader{} conn.SetReadDeadline(time.Now().Add(w.timeout)) - _, buf, err = ReadTLV(conn) + resp, err := requestReader.Read(conn) if err != nil { conn.MarkUnusable() return err @@ -131,7 +133,7 @@ func (w *ShardWriter) WriteShard(shardID, ownerID uint64, points []models.Point) // Unmarshal response. var response WriteShardResponse - if err := response.UnmarshalBinary(buf); err != nil { + if err := response.UnmarshalBinary(resp.Data); err != nil { return err } diff --git a/services/controller/service.go b/services/controller/service.go index dd52540..d9bdcc1 100644 --- a/services/controller/service.go +++ b/services/controller/service.go @@ -268,7 +268,7 @@ func (s *Service) writeResponse(w io.Writer, t ResponseType, obj interface{}) { } func (s *Service) readRequest(conn net.Conn, obj interface{}) error { - buf, err := coordinator.ReadLV(conn) + buf, err := coordinator.ReadLV(conn, 10*time.Second) if err != nil { s.Logger.Error("unable to read length-value", zap.Error(err)) return err diff --git a/x/cyclic_buffer.go b/x/cyclic_buffer.go index 1f34525..f397fca 100644 --- a/x/cyclic_buffer.go +++ b/x/cyclic_buffer.go @@ -30,6 +30,14 @@ func (c *CyclicBuffer) write(data []byte, offset int) { } } +func (c *CyclicBuffer) Cap() int { + return len(c.buf) +} + +func (c *CyclicBuffer) Len() int { + return c.cnt +} + // Dump dumps internal data into given slice and returns write byte count // ATTENTION: the given buffer should not smaller than buffer capacity func (c *CyclicBuffer) Dump(data []byte) int { From f8a3e267cba01c952f43dc606f655d1ad20e7885 Mon Sep 17 00:00:00 2001 From: Jason Joo Date: Mon, 16 Nov 2020 13:18:44 +0800 Subject: [PATCH 43/46] Add ignore Signed-off-by: Jason Joo --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 479919d..9b5001b 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ cmd/influxd-ctl/influxd-ctl cmd/influxd/influxd cmd/metad/metad +sync_simulation From 1375a2a599b0d6e1d599b8bdd3c0ca8089069e69 Mon Sep 17 00:00:00 2001 From: Jason Joo Date: Mon, 16 Nov 2020 15:17:40 +0800 Subject: [PATCH 44/46] Polish cluster executor logic Signed-off-by: Jason Joo --- coordinator/cluster_executor.go | 72 ++++++++++++++++++--------------- 1 file changed, 39 insertions(+), 33 deletions(-) diff --git a/coordinator/cluster_executor.go b/coordinator/cluster_executor.go index df553a8..b1198ad 100644 --- a/coordinator/cluster_executor.go +++ b/coordinator/cluster_executor.go @@ -350,6 +350,44 @@ func (me *ClusterExecutor) TagValues(auth query.Authorizer, ids []uint64, cond i return tagValues, nil } +func (me *ClusterExecutor) createLocalIteratorfunc( + localCtx context.Context, + m *influxql.Measurement, + shardIDs []uint64, + opt query.IteratorOptions, +) (itr query.Iterator, err error) { + span := tracing.SpanFromContext(localCtx) + if span != nil { + span = span.StartSpan(fmt.Sprintf("local_node_id: %d", me.Node.ID)) + defer span.Finish() + + localCtx = tracing.NewContextWithSpan(localCtx, span) + } + sg := me.TSDBStore.ShardGroup(shardIDs) + if m.Regex != nil { + measurements := sg.MeasurementsByRegex(m.Regex.Val) + inputs := make([]query.Iterator, 0, len(measurements)) + if err := func() error { + for _, measurement := range measurements { + mm := m.Clone() + mm.Name = measurement + input, err := sg.CreateIterator(localCtx, mm, opt) + if err != nil { + return err + } + inputs = append(inputs, input) + } + return nil + }(); err != nil { + query.Iterators(inputs).Close() + return nil, err + } + + return query.Iterators(inputs).Merge(opt) + } + return sg.CreateIterator(localCtx, m, opt) +} + func (me *ClusterExecutor) CreateIterator(ctx context.Context, m *influxql.Measurement, opt query.IteratorOptions, shards []meta.ShardInfo) (query.Iterator, error) { type Result struct { iter query.Iterator @@ -363,39 +401,7 @@ func (me *ClusterExecutor) CreateIterator(ctx context.Context, m *influxql.Measu shardIDs := toShardIDs(shards) if nodeId == me.Node.ID { //localCtx only use for local node - localCtx := ctx - iter, err = func() (query.Iterator, error) { - span := tracing.SpanFromContext(localCtx) - if span != nil { - span = span.StartSpan(fmt.Sprintf("local_node_id: %d", me.Node.ID)) - defer span.Finish() - - localCtx = tracing.NewContextWithSpan(localCtx, span) - } - sg := me.TSDBStore.ShardGroup(shardIDs) - if m.Regex != nil { - measurements := sg.MeasurementsByRegex(m.Regex.Val) - inputs := make([]query.Iterator, 0, len(measurements)) - if err := func() error { - for _, measurement := range measurements { - mm := m.Clone() - mm.Name = measurement - input, err := sg.CreateIterator(localCtx, mm, opt) - if err != nil { - return err - } - inputs = append(inputs, input) - } - return nil - }(); err != nil { - query.Iterators(inputs).Close() - return nil, err - } - - return query.Iterators(inputs).Merge(opt) - } - return sg.CreateIterator(localCtx, m, opt) - }() + iter, err = me.createLocalIteratorfunc(ctx, m, shardIDs, opt) } else { iter, err = me.RemoteNodeExecutor.CreateIterator(nodeId, ctx, m, opt, shardIDs) } From 4f77af742d7046a5d96d5b50926ac3ef00462d68 Mon Sep 17 00:00:00 2001 From: Jason Joo Date: Sun, 13 Dec 2020 18:52:25 +0800 Subject: [PATCH 45/46] Make DeletedAt of shard group consistent Signed-off-by: Jason Joo --- coordinator/cluster_meta_client.go | 5 +++-- raftmeta/apply.go | 18 ++++++++++++++++-- raftmeta/meta_service.go | 20 +++++++++++++++----- raftmeta/meta_store.go | 4 ++-- services/meta/data.go | 21 +++++++++++++++++++++ services/meta/store.go | 7 +++---- services/meta/store_test.go | 2 +- 7 files changed, 61 insertions(+), 16 deletions(-) diff --git a/coordinator/cluster_meta_client.go b/coordinator/cluster_meta_client.go index e2e3088..0f298da 100644 --- a/coordinator/cluster_meta_client.go +++ b/coordinator/cluster_meta_client.go @@ -307,14 +307,15 @@ func (me *ClusterMetaClient) DeleteShardGroup(database, policy string, id uint64 if err := me.metaCli.DeleteShardGroup(database, policy, id); err != nil { return err } - return me.cache.DeleteShardGroup(database, policy, id) + // To cache data DeletedAt is not important + return me.cache.DeleteShardGroup(database, policy, id, time.Now()) } func (me *ClusterMetaClient) PruneShardGroups() error { if err := me.metaCli.PruneShardGroups(); err != nil { return err } - return me.cache.PruneShardGroups() + return nil } func (me *ClusterMetaClient) PrecreateShardGroups(from, to time.Time) error { diff --git a/raftmeta/apply.go b/raftmeta/apply.go index a4b672f..773d0ad 100644 --- a/raftmeta/apply.go +++ b/raftmeta/apply.go @@ -7,6 +7,7 @@ import ( "time" "github.com/angopher/chronus/raftmeta/internal" + imeta "github.com/angopher/chronus/services/meta" "github.com/angopher/chronus/x" "github.com/influxdata/influxdb/services/meta" "go.uber.org/zap" @@ -241,14 +242,27 @@ func (s *RaftNode) applyCommitted(proposal *internal.Proposal, index uint64) err return s.MetaStore.TruncateShardGroups(req.Time) case internal.PruneShardGroups: - return s.MetaStore.PruneShardGroups() + var req PruneShardGroupsReq + if len(proposal.Data) > 1 { + err := json.Unmarshal(proposal.Data, &req) + x.Check(err) + s.SugaredLogger.Debugf("req %+v", req) + } + if req.Expiration.IsZero() { + // fallback + req.Expiration = time.Now().Add(imeta.SHARDGROUP_INFO_EVICTION) + } + return s.MetaStore.PruneShardGroups(req.Expiration) case internal.DeleteShardGroup: var req DeleteShardGroupReq err := json.Unmarshal(proposal.Data, &req) x.Check(err) + if req.Now.IsZero() { + req.Now = time.Now() + } s.SugaredLogger.Debugf("req %+v", req) - return s.MetaStore.DeleteShardGroup(req.Database, req.Policy, req.Id) + return s.MetaStore.DeleteShardGroup(req.Database, req.Policy, req.Id, req.Now) case internal.PrecreateShardGroups: var req PrecreateShardGroupsReq diff --git a/raftmeta/meta_service.go b/raftmeta/meta_service.go index 490900a..bf7415a 100644 --- a/raftmeta/meta_service.go +++ b/raftmeta/meta_service.go @@ -1009,6 +1009,10 @@ func (s *MetaService) TruncateShardGroups(w http.ResponseWriter, r *http.Request s.Logger.Info("TruncateShardGroups ok", zap.Time("Time", req.Time)) } +type PruneShardGroupsReq struct { + Expiration time.Time +} + type PruneShardGroupsResp struct { CommonResp } @@ -1019,7 +1023,11 @@ func (s *MetaService) PruneShardGroups(w http.ResponseWriter, r *http.Request) { resp.RetMsg = "fail" defer WriteResp(w, &resp) - err := s.ProposeAndWait(internal.PruneShardGroups, []byte{}, nil) + req := &PruneShardGroupsReq{ + Expiration: time.Now().Add(imeta.SHARDGROUP_INFO_EVICTION), + } + data, _ := json.Marshal(req) + err := s.ProposeAndWait(internal.PruneShardGroups, data, nil) if err != nil { resp.RetMsg = err.Error() s.Logger.Error("PruneShardGroups fail", zap.Error(err)) @@ -1036,6 +1044,7 @@ type DeleteShardGroupReq struct { Database string Policy string Id uint64 + Now time.Time } type DeleteShardGroupResp struct { CommonResp @@ -1050,21 +1059,22 @@ func (s *MetaService) DeleteShardGroup(w http.ResponseWriter, r *http.Request) { data, err := ioutil.ReadAll(r.Body) if err != nil { resp.RetMsg = err.Error() - s.Logger.Error("TruncateShardGroups fail", zap.Error(err)) + s.Logger.Error("DeleteShardGroup fail", zap.Error(err)) return } var req DeleteShardGroupReq if err := json.Unmarshal(data, &req); err != nil { resp.RetMsg = err.Error() - s.Logger.Error("TruncateShardGroups fail", zap.Error(err)) + s.Logger.Error("DeleteShardGroup fail", zap.Error(err)) return } + req.Now = time.Now() err = s.ProposeAndWait(internal.DeleteShardGroup, data, nil) if err != nil { resp.RetMsg = err.Error() - s.Logger.Error("TruncateShardGroups fail", + s.Logger.Error("DeleteShardGroup fail", zap.String("Database", req.Database), zap.String("Policy", req.Policy), zap.Uint64("Id", req.Id), @@ -1074,7 +1084,7 @@ func (s *MetaService) DeleteShardGroup(w http.ResponseWriter, r *http.Request) { resp.RetCode = 0 resp.RetMsg = "ok" - s.Logger.Info("TruncateShardGroups ok", + s.Logger.Info("DeleteShardGroup ok", zap.String("Database", req.Database), zap.String("Policy", req.Policy), zap.Uint64("Id", req.Id)) diff --git a/raftmeta/meta_store.go b/raftmeta/meta_store.go index d50d9c5..343e40a 100644 --- a/raftmeta/meta_store.go +++ b/raftmeta/meta_store.go @@ -25,8 +25,8 @@ type MetaStore interface { FreezeDataNode(id uint64) error UnfreezeDataNode(id uint64) error Authenticate(username, password string) (meta.User, error) - PruneShardGroups() error - DeleteShardGroup(database, policy string, id uint64) error + PruneShardGroups(expiration time.Time) error + DeleteShardGroup(database, policy string, id uint64, t time.Time) error PrecreateShardGroups(from, to time.Time) error AddShardOwner(shardID, nodeID uint64) error diff --git a/services/meta/data.go b/services/meta/data.go index 0a47eae..98e39bd 100644 --- a/services/meta/data.go +++ b/services/meta/data.go @@ -536,3 +536,24 @@ func (data *Data) RemoveShardOwner(id, nodeID uint64) { } } } + +// DeleteShardGroup removes a shard group from a database and retention policy by id. +func (data *Data) DeleteShardGroup(database, policy string, id uint64, t time.Time) error { + // Find retention policy. + rpi, err := data.RetentionPolicy(database, policy) + if err != nil { + return err + } else if rpi == nil { + return influxdb.ErrRetentionPolicyNotFound(policy) + } + + // Find shard group by ID and set its deletion timestamp. + for i := range rpi.ShardGroups { + if rpi.ShardGroups[i].ID == id { + rpi.ShardGroups[i].DeletedAt = t.UTC() + return nil + } + } + + return meta.ErrShardGroupNotFound +} diff --git a/services/meta/store.go b/services/meta/store.go index e8659d8..7378ad4 100644 --- a/services/meta/store.go +++ b/services/meta/store.go @@ -766,9 +766,8 @@ func (c *Client) TruncateShardGroups(t time.Time) error { } // PruneShardGroups remove deleted shard groups from the data store. -func (c *Client) PruneShardGroups() error { +func (c *Client) PruneShardGroups(expiration time.Time) error { var changed bool - expiration := time.Now().Add(SHARDGROUP_INFO_EVICTION) c.mu.Lock() defer c.mu.Unlock() data := c.cacheData.Clone() @@ -898,13 +897,13 @@ func (c *Client) UnfreezeDataNode(id uint64) error { } // DeleteShardGroup removes a shard group from a database and retention policy by id. -func (c *Client) DeleteShardGroup(database, policy string, id uint64) error { +func (c *Client) DeleteShardGroup(database, policy string, id uint64, t time.Time) error { c.mu.Lock() defer c.mu.Unlock() data := c.cacheData.Clone() - if err := data.DeleteShardGroup(database, policy, id); err != nil { + if err := data.DeleteShardGroup(database, policy, id, t); err != nil { return err } diff --git a/services/meta/store_test.go b/services/meta/store_test.go index 09fb5dd..4a9072d 100644 --- a/services/meta/store_test.go +++ b/services/meta/store_test.go @@ -1089,7 +1089,7 @@ func TestMetaClient_PruneShardGroups(t *testing.T) { t.Fatal(err) } - if err := c.PruneShardGroups(); err != nil { + if err := c.PruneShardGroups(time.Now().Add(imeta.SHARDGROUP_INFO_EVICTION)); err != nil { t.Fatal(err) } From 0c7b3e3ceb718238a804bf122931303c22047e4c Mon Sep 17 00:00:00 2001 From: Jason Joo Date: Mon, 29 Mar 2021 00:56:01 +0800 Subject: [PATCH 46/46] Make it possible to reload meta peers from meta server when pings failed Signed-off-by: Jason Joo --- coordinator/cluster_meta_client.go | 79 ++++++++++++++++++++++-------- coordinator/meta_client_impl.go | 16 +++++- raftmeta/meta_service.go | 9 ++++ 3 files changed, 83 insertions(+), 21 deletions(-) diff --git a/coordinator/cluster_meta_client.go b/coordinator/cluster_meta_client.go index 0f298da..37b2a44 100644 --- a/coordinator/cluster_meta_client.go +++ b/coordinator/cluster_meta_client.go @@ -24,15 +24,18 @@ func NewMetaClient(mc *meta.Config, cc Config, nodeID uint64) *ClusterMetaClient return &ClusterMetaClient{ NodeID: nodeID, metaCli: &MetaClientImpl{ - Addrs: cc.MetaServices, + Addrs: cc.MetaServices, + Logger: zap.NewNop(), }, pingIntervalMs: cc.PingMetaServiceIntervalMs, cache: imeta.NewClient(mc), + Logger: zap.NewNop(), } } func (me *ClusterMetaClient) WithLogger(log *zap.Logger) { me.Logger = log.With(zap.String("coordinator", "ClusterMetaClient")) + me.metaCli.Logger = log.With(zap.String("coordinator", "MetaClientImpl")) } func (me *ClusterMetaClient) Open() error { @@ -60,40 +63,60 @@ func (me *ClusterMetaClient) syncData() error { me.Logger.Info("start sync data") data, err := me.metaCli.Data() if err != nil { - fmt.Println("start sync fail ", err) + me.Logger.Error(fmt.Sprintf("start sync fail: %v", err)) return err } me.Logger.Info("start sync done") + if len(data.MetaNodes) > 0 { + addrs := make([]string, len(data.MetaNodes)) + for i, metaNode := range data.MetaNodes { + addrs[i] = metaNode.Host + } + me.metaCli.UpdateAddrs(addrs) + } return me.cache.ReplaceData(data) } func (me *ClusterMetaClient) syncLoop() { ticker := time.NewTicker(time.Duration(me.pingIntervalMs) * time.Millisecond) printLimiter := rate.NewLimiter(0.1, 1) + pingFailed := 0 for { select { case <-ticker.C: + needSync := false index, err := me.metaCli.Ping() if err != nil { me.Logger.Warn("Ping fail", zap.Error(err)) - continue + pingFailed++ + if pingFailed > 30 { + needSync = true + pingFailed = 0 + } else { + continue + } + } else { + switch { + case index > me.cache.DataIndex(): + needSync = true + case index < me.cache.DataIndex(): + me.Logger.Warn(fmt.Sprintf("index:%d < local index:%d", index, me.cache.DataIndex())) + default: + // normal + if printLimiter.Allow() { + // one log in 10 seconds + me.Logger.Debug(fmt.Sprintf("index=%d local_index=%d", index, me.cache.Data().Index)) + } + } } - if index > me.cache.DataIndex() { + if needSync { if err := me.syncData(); err != nil { me.Logger.Warn("syncData fail", zap.Error(err)) } else { me.Logger.Info("syncData success") } - } else if index < me.cache.DataIndex() { - me.Logger.Warn(fmt.Sprintf("index:%d < local index:%d", index, me.cache.DataIndex())) - } else { - // normal - if printLimiter.Allow() { - // one log in 10 seconds - me.Logger.Debug(fmt.Sprintf("index=%d local_index=%d", index, me.cache.Data().Index)) - } } } } @@ -101,14 +124,30 @@ func (me *ClusterMetaClient) syncLoop() { func (me *ClusterMetaClient) Start() { // sync first synchronously - wait := me.WaitForDataChanged() - go me.syncData() - //wait sync meta data from meta server - select { - case <-time.After(5 * time.Second): - //TODO: - panic("sync meta data failed") - case <-wait: + retryTimes := 0 +LOOP: + for { + retryTimes++ + errC := make(chan error, 1) + go func(c chan error) { + c <- me.syncData() + close(c) + }(errC) + + //wait sync meta data from meta server + err := <-errC + if err != nil { + // retry + if retryTimes < 10 { + me.Logger.Warn("Load data on startup failed, retry") + continue LOOP + } else { + panic("sync meta data failed") + } + } + + break LOOP + } go me.syncLoop() } diff --git a/coordinator/meta_client_impl.go b/coordinator/meta_client_impl.go index 8b3c16d..3266847 100644 --- a/coordinator/meta_client_impl.go +++ b/coordinator/meta_client_impl.go @@ -10,10 +10,12 @@ import ( "net" "net/http" "os" + "sync" "time" "github.com/influxdata/influxdb/services/meta" "github.com/influxdata/influxql" + "go.uber.org/zap" "github.com/angopher/chronus/raftmeta" imeta "github.com/angopher/chronus/services/meta" @@ -38,12 +40,24 @@ var ( ) type MetaClientImpl struct { - Addrs []string + Addrs []string + Logger *zap.Logger + + mu sync.Mutex } // return &MetaClientImpl{MetaServiceHost: "127.0.0.1:1234"} +func (me *MetaClientImpl) UpdateAddrs(addrs []string) { + me.mu.Lock() + defer me.mu.Unlock() + me.Addrs = addrs + me.Logger.Warn(fmt.Sprintf("The addresses of the meta servers have been updated: %v", addrs)) +} + func (me *MetaClientImpl) Url(path string) string { + me.mu.Lock() + defer me.mu.Unlock() return fmt.Sprintf("http://%s%s", me.Addrs[rand.Intn(len(me.Addrs))], path) } diff --git a/raftmeta/meta_service.go b/raftmeta/meta_service.go index bf7415a..ccde66e 100644 --- a/raftmeta/meta_service.go +++ b/raftmeta/meta_service.go @@ -1420,6 +1420,15 @@ func (s *MetaService) Data(w http.ResponseWriter, r *http.Request) { } data := s.cli.Data() + // set metad peers + peers := s.Node.Transport.ClonePeers() + data.MetaNodes = nil + for id, addr := range peers { + data.MetaNodes = append(data.MetaNodes, meta.NodeInfo{ + ID: id, + Host: addr, + }) + } resp.Data, err = data.MarshalBinary() if err != nil { resp.RetMsg = err.Error()