Mysync doesn't start when incorrect zk DNS is specified #144
Description
If we specify an incorrect zk DNS in the Mysync configuration or delete a zk host along with its DNS record, Mysync will not start. For an example, I delete zoo3 from docker-compose.yaml and get an error that server zoo3 is misbehaving. This happens because we immediately throw an error from the function if we cannot get the host's IP using DNS in the function.
for _, server := range srvs {
ips, err := addrsByHostname(server)
if err != nil {
return nil, err
}
for _, ip := range ips {
addrToHostname[ip] = server
}
}Is it a bug or a feature? :)
I believe this is not a common occurrence, and in most cases, it can help prevent users from specifying invalid DNS records. However, in the rare cases when a user deletes a host along with its DNS and restarts Mysync, they will encounter this error. I applied Mysync to my project using Kubernetes+Helm and encountered this issue when scaling down the Zookeeper cluster.
Configurations
version: '2.3'
services:
zoo1:
build:
context: ./zookeeper
args:
- VERSION=$VERSION
hostname: zoo1
ports:
- 2181
- 2281
- 2888
- 3888
privileged: true
environment:
ZK_SERVERS: server.1=zoo1:2888:3888, server.2=zoo2:2888:3888, server.3=zoo3:2888:3888
ZK_MYID: 1
healthcheck:
test: "nc -z localhost 2181"
start_period: 30s
timeout: 5s
interval: 10s
retries: 3
networks:
mysync_net:
ipv4_address: 192.168.233.10
zoo2:
build:
context: ./zookeeper
args:
- VERSION=$VERSION
hostname: zoo2
ports:
- 2181
- 2281
- 2888
- 3888
privileged: true
environment:
ZK_SERVERS: server.1=zoo1:2888:3888, server.2=zoo2:2888:3888, server.3=zoo3:2888:3888
ZK_MYID: 2
networks:
mysync_net:
ipv4_address: 192.168.233.11
#zoo3:
# build:
# context: ./zookeeper
# args:
# - VERSION=$VERSION
# hostname: zoo3
# ports:
# - 2181
# - 2281
# - 2888
# - 3888
# privileged: true
# environment:
# ZK_SERVERS: server.1=zoo1:2888:3888, server.2=zoo2:2888:3888, server.3=zoo3:2888:3888
# ZK_MYID: 3
# networks:
# mysync_net:
# ipv4_address: 192.168.233.12
mysql1:
build:
context: ./mysql
args:
- VERSION=$VERSION
hostname: mysql1
ports:
- 3306
privileged: true
environment:
MYSQL_SERVER_ID: 1
MYSQL_ADMIN_USER: admin
MYSQL_ADMIN_PASSWORD: admin_pwd
MYSQL_MASTER: ''
MYSYNC_FAILOVER:
MYSYNC_FAILOVER_DELAY:
MYSYNC_FAILOVER_COOLDOWN:
MYSYNC_INACTIVATION_DELAY:
ZK_SERVERS: zoo1:2281, zoo2:2281, zoo3:2281
MYSQL_PORT: 3306
MYSYNC_DISABLE_REPLICATION_ON_MAINT: "true"
MYSYNC_SEMISYNC:
MYSYNC_ASYNC:
ASYNC_ALLOWED_LAG:
MYSYNC_CRITICAL_DISK_USAGE:
MYSYNC_KEEP_SUPER_WRITABLE_ON_CRITICAL_DISK_USAGE:
MYSYNC_WAIT_FOR_SLAVE_COUNT:
MYSYNC_STREAM_FROM_REASONABLE_LAG:
MYSYNC_RESETUP_CRASHED_HOSTS:
MYSYNC_REPLICATION_REPAIR_AGGRESSIVE_MODE:
MYSYNC_SET_RO_TIMEOUT:
MYSYNC_REPLICATION_LAG_QUERY:
REPL_MON:
healthcheck:
test: "mysql --user=admin --password=admin_pwd -e 'SELECT 1'"
start_period: 30s
timeout: 5s
interval: 10s
retries: 3
depends_on:
zoo1:
condition: service_healthy
networks:
mysync_net:
ipv4_address: 192.168.233.13
mysql2:
build:
context: ./mysql
args:
- VERSION=$VERSION
hostname: mysql2
ports:
- 3306
privileged: true
environment:
MYSQL_SERVER_ID: 2
MYSQL_ADMIN_USER: admin
MYSQL_ADMIN_PASSWORD: admin_pwd
MYSQL_MASTER: mysql1
MYSYNC_FAILOVER:
MYSYNC_FAILOVER_DELAY:
MYSYNC_FAILOVER_COOLDOWN:
MYSYNC_INACTIVATION_DELAY:
ZK_SERVERS: zoo1:2281, zoo2:2281, zoo3:2281
MYSQL_PORT: 3306
MYSYNC_DISABLE_REPLICATION_ON_MAINT: "true"
MYSYNC_SEMISYNC:
MYSYNC_ASYNC:
ASYNC_ALLOWED_LAG:
MYSYNC_CRITICAL_DISK_USAGE:
MYSYNC_KEEP_SUPER_WRITABLE_ON_CRITICAL_DISK_USAGE:
MYSYNC_WAIT_FOR_SLAVE_COUNT:
MYSYNC_STREAM_FROM_REASONABLE_LAG:
MYSYNC_RESETUP_CRASHED_HOSTS:
MYSYNC_REPLICATION_REPAIR_AGGRESSIVE_MODE:
MYSYNC_SET_RO_TIMEOUT:
MYSYNC_REPLICATION_LAG_QUERY:
REPL_MON:
depends_on:
mysql1:
condition: service_healthy
networks:
mysync_net:
ipv4_address: 192.168.233.14
mysql3:
build:
context: ./mysql
args:
- VERSION=$VERSION
hostname: mysql3
ports:
- 3306
privileged: true
environment:
MYSQL_SERVER_ID: 3
MYSQL_ADMIN_USER: admin
MYSQL_ADMIN_PASSWORD: admin_pwd
MYSQL_MASTER: mysql1
MYSYNC_FAILOVER:
MYSYNC_FAILOVER_DELAY:
MYSYNC_FAILOVER_COOLDOWN:
MYSYNC_INACTIVATION_DELAY:
ZK_SERVERS: zoo1:2281, zoo2:2281, zoo3:2281
MYSQL_PORT: 3306
MYSYNC_DISABLE_REPLICATION_ON_MAINT: "true"
MYSYNC_SEMISYNC:
MYSYNC_ASYNC:
ASYNC_ALLOWED_LAG:
MYSYNC_CRITICAL_DISK_USAGE:
MYSYNC_KEEP_SUPER_WRITABLE_ON_CRITICAL_DISK_USAGE:
MYSYNC_WAIT_FOR_SLAVE_COUNT:
MYSYNC_STREAM_FROM_REASONABLE_LAG:
MYSYNC_RESETUP_CRASHED_HOSTS:
MYSYNC_REPLICATION_REPAIR_AGGRESSIVE_MODE:
MYSYNC_SET_RO_TIMEOUT:
MYSYNC_REPLICATION_LAG_QUERY:
REPL_MON:
depends_on:
mysql1:
condition: service_healthy
networks:
mysync_net:
ipv4_address: 192.168.233.15
networks:
mysync_net:
driver: bridge
ipam:
driver: default
config:
- subnet: 192.168.233.0/24
gateway: 192.168.233.1And specify in mysync.yaml:
zookeeper:
session_timeout: 3s
namespace: /test
hosts: [ zoo1:2281, zoo2:2281, zoo3:2281 ]
auth: true
username: testuser
password: testpassword123
use_ssl: true
keyfile: /etc/zk-ssl/server.key
certfile: /etc/zk-ssl/server.crt
ca_cert: /etc/zk-ssl/ca.cert.pem
verify_certs: trueLogs
2024-10-17T05:50:10Z INFO: MYSYNC START
2024-10-17T05:50:10Z INFO: config failover: false semisync: true
2024-10-17T05:50:10Z ERROR: failed to connect to zkDCS: lookup zoo3 on 127.0.0.11:53: server misbehaving
2024-10-17T05:50:31Z INFO: external replication is enabled
2024-10-17T05:50:31Z INFO: MYSYNC START
2024-10-17T05:50:31Z INFO: config failover: false semisync: true
2024-10-17T05:50:31Z ERROR: failed to connect to zkDCS: lookup zoo3 on 127.0.0.11:53: server misbehaving
2024-10-17T05:50:53Z INFO: external replication is enabled
2024-10-17T05:50:53Z INFO: MYSYNC START
2024-10-17T05:50:53Z INFO: config failover: false semisync: true
2024-10-17T05:50:54Z ERROR: failed to connect to zkDCS: lookup zoo3 on 127.0.0.11:53: server misbehaving
2024-10-17T05:51:17Z INFO: external replication is enabled