Roles and Functions

Deploying canal-server

For detailed steps on deploying canal-server in a Kubernetes cluster, refer to official documentation. A new instance must be created to listen to the source database; its recommended to configure it via the canal-admin web interface.

Deploying canal-adapter

Deploy canal-adapter using Docker with the official image:

docker pull slpcat/canal-adapter:v1.1.5-jdk8

Two configuration files must be mounted when starting the container:

1. Main configuration file: /opt/canal-adapter/conf/application.yml
2. Adapter-specific configuration (for writing to target databases): /opt/canal-adapter/conf/rdb/client.yml (multiple [client].yml files can be placed in the rdb directory if writing to multiple targets).

Detailed Configuration

canal-server

Instance Configuration

#################################################
## mysql serverId , v1.0.26+ will autoGen
# canal.instance.mysql.slaveId=0

# enable gtid use true/false
canal.instance.gtidon=false

# position info
# MySQL connection details
canal.instance.master.address=192.168.1.10:3306
canal.instance.master.journal.name=
canal.instance.master.position=
canal.instance.master.timestamp=
canal.instance.master.gtid=

# rds oss binlog
# rds oss authorization (no longer supported by Alibaba Cloud RDS)
# canal.instance.rds.accesskey=LT123456789
# canal.instance.rds.secretkey=zZ3s123456789
# canal.instance.rds.instanceId=rm-uf123456789

# table meta tsdb info
canal.instance.tsdb.enable=true
#canal.instance.tsdb.url=jdbc:mysql://127.0.0.1:3306/canal_tsdb
#canal.instance.tsdb.dbUsername=canal
#canal.instance.tsdb.dbPassword=canal

# username/password for source database
# Requires a high-privilege account
canal.instance.dbUsername=root
canal.instance.dbPassword=abc^123456789
canal.instance.connectionCharset = utf8mb4
# enable druid Decrypt database password
canal.instance.enableDruid=false
#canal.instance.pwdPublicKey=MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBALK4BUxdDltRRE5/zXpVEVPUgunvscYFtEip3pmLlhrWpacX7y7GCMo2/JM6LeHmiiNdH1FWgGCpUfircSwlWKUCAwEAAQ==

# table regex (whitelist)
# 1. Capture all tables in all databases
# canal.instance.filter.regex=.*\..*
# 2. Capture all tables in a specific database
# canal.instance.filter.regex=databases1\..*
# canal.instance.filter.regex=databases1\..*,databases2\..*
# 3. Capture specific tables in a database
# canal.instance.filter.regex=databases1.table1
# canal.instance.filter.regex=databases1.table1,databases1.table2
canal.instance.filter.regex=databases1.table1

# table black regex (blacklist)
canal.instance.filter.black.regex=

# mq config (if using MQ mode)
canal.mq.topic=example
# dynamic topic route by schema or table regex
#canal.mq.dynamicTopic=mytest1.user,mytest2\..*,.*\..*
canal.mq.partition=0
# hash partition config
#canal.mq.partitionsNum=3
#canal.mq.partitionHash=test.table:id^name,.*\..*
#################################################

canal-adapter

application.yml

server:
  port: 8081
spring:
  jackson:
    date-format: yyyy-MM-dd HH:mm:ss
    time-zone: GMT+8
    default-property-inclusion: non_null

canal.conf:
  mode: tcp # tcp / kafka / rocketMQ / rabbitMQ
  flatMessage: true
  zookeeperHosts:
  syncBatchSize: 1000
  retries: 0
  timeout:
  accessKey:
  secretKey:
  consumerProperties:
    # TCP consumer (for canal-server via TCP)
    canal.tcp.server.host: 192.168.1.30:11111
    canal.tcp.zookeeper.hosts:
    canal.tcp.batch.size: 500
    canal.tcp.username: admin
    canal.tcp.password: 123456
    # Kafka consumer (example)
    #kafka.bootstrap.servers: 127.0.0.1:9092
    #kafka.enable.auto.commit: false
    # ...
  # Source data sources (optional, for adapter-side querying)
  # srcDataSources:
  #   testDS:
  #     url: jdbc:mysql://10.10.101.40:3306/mytest2?useUnicode=true
  #     username: root
  #     password: password
  canalAdapters:
    - instance: instance-cmoon   # instance name from canal-server
      groups:
        - groupId: g-cmoon
          outerAdapters:
            - name: rdb
              key: mysql-cmoon
              properties:
                jdbc.driverClassName: com.mysql.jdbc.Driver
                jdbc.url: jdbc:mysql://192.168.2.43:3306/databases1?useUnicode=true&characterEncoding=utf-8
                jdbc.username: root
                jdbc.password: CMOON^123456789
            # Example for other databases:
            # - name: rdb
            #   key: oracle1
            #   properties:
            #     jdbc.driverClassName: oracle.jdbc.OracleDriver
            #     jdbc.url: jdbc:oracle:thin:@localhost:49161:XE
            #     ...

client.yml

This file defines how data from a specific instance is mapped to a target database.

destination: instance-cmoon                # Must match the instance name in canal-server
groupId: g-cmoon                          # Corresponds to groupId in application.yml
outerAdapterKey: mysql-cmoon              # Corresponds to the key in outerAdapters
concurrent: true                           # Enable parallel sync (requires that primary keys never change and are not foreign keys)
dbMapping:
  database: databases1                     # Source schema/database name
  # table: table_name                      # Optional: specific table name; if omitted, all tables from the database are mapped when mirrorDb is true
  # targetTable: target_db.target_table   # Optional: custom target table name
  # targetPk:                              # Optional: custom primary key mapping (compound keys supported)
  #   id: id
  mapAll: true                             # Map all columns with identical names; overridden by targetColumns if specified
  mirrorDb: true                           # Enable database mirroring (scheme: mirror source DB structure to target DB)
  # targetColumns:                         # Optional: explicit column mapping (target: source)
  #   new_field: old_field