-
Notifications
You must be signed in to change notification settings - Fork 0
/
connector.go
119 lines (103 loc) · 3.12 KB
/
connector.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
package cdc
import (
"context"
cdc "github.com/Trendyol/go-pq-cdc"
"github.com/Trendyol/go-pq-cdc-elasticsearch/config"
"github.com/Trendyol/go-pq-cdc-elasticsearch/elasticsearch"
"github.com/Trendyol/go-pq-cdc-elasticsearch/elasticsearch/bulk"
"github.com/Trendyol/go-pq-cdc-elasticsearch/elasticsearch/client"
"github.com/Trendyol/go-pq-cdc-elasticsearch/internal/slices"
"github.com/Trendyol/go-pq-cdc/logger"
"github.com/Trendyol/go-pq-cdc/pq/message/format"
"github.com/Trendyol/go-pq-cdc/pq/replication"
es "github.com/elastic/go-elasticsearch/v8"
"github.com/go-playground/errors"
"github.com/prometheus/client_golang/prometheus"
)
type Connector interface {
Start(ctx context.Context)
Close()
}
type connector struct {
handler Handler
responseHandler elasticsearch.ResponseHandler
cfg *config.Config
cdc cdc.Connector
esClient *es.Client
bulk bulk.Indexer
metrics []prometheus.Collector
}
func NewConnector(ctx context.Context, cfg config.Config, handler Handler, options ...Option) (Connector, error) {
cfg.SetDefault()
esConnector := &connector{
cfg: &cfg,
handler: handler,
}
Options(options).Apply(esConnector)
pqCDC, err := cdc.NewConnector(ctx, esConnector.cfg.CDC, esConnector.listener)
if err != nil {
return nil, err
}
esConnector.cdc = pqCDC
esConnector.cfg.CDC = *pqCDC.GetConfig()
esClient, err := client.NewClient(esConnector.cfg)
if err != nil {
return nil, errors.Wrap(err, "elasticsearch new client")
}
esConnector.esClient = esClient
esConnector.bulk, err = bulk.NewBulk(
esConnector.cfg,
esClient,
bulk.WithResponseHandler(esConnector.responseHandler))
if err != nil {
return nil, errors.Wrap(err, "elasticsearch new bulk")
}
pqCDC.SetMetricCollectors(esConnector.bulk.GetMetric().PrometheusCollectors()...)
pqCDC.SetMetricCollectors(esConnector.metrics...)
return esConnector, nil
}
func (c *connector) Start(ctx context.Context) {
go func() {
logger.Info("waiting for connector start...")
if err := c.cdc.WaitUntilReady(ctx); err != nil {
panic(err)
}
logger.Info("bulk process started")
c.bulk.StartBulk()
}()
c.cdc.Start(ctx)
}
func (c *connector) Close() {
c.cdc.Close()
c.bulk.Close()
}
func (c *connector) listener(ctx *replication.ListenerContext) {
var msg Message
switch m := ctx.Message.(type) {
case *format.Insert:
msg = NewInsertMessage(c.esClient, m)
case *format.Update:
msg = NewUpdateMessage(c.esClient, m)
case *format.Delete:
msg = NewDeleteMessage(c.esClient, m)
default:
return
}
actions := c.handler(msg)
if len(actions) == 0 {
if err := ctx.Ack(); err != nil {
logger.Error("ack", "error", err)
}
return
}
batchSizeLimit := c.cfg.Elasticsearch.BatchSizeLimit
if len(actions) > batchSizeLimit {
chunks := slices.ChunkWithSize[elasticsearch.Action](actions, batchSizeLimit)
lastChunkIndex := len(chunks) - 1
for idx, chunk := range chunks {
c.bulk.AddActions(ctx, msg.EventTime, chunk, msg.TableNamespace, msg.TableName, idx == lastChunkIndex)
}
} else {
c.bulk.AddActions(ctx, msg.EventTime, actions, msg.TableNamespace, msg.TableName, true)
}
}