1 nixops mariadb galera
Izorkin edited this page 2 years ago

В предыдущей статье мы настроили управление одной удалённой система с помощью утилиты nixops. Давайте теперь настроим более сложный вариант - поднимем ещё 3 удалённые системы и настроим на ней кластер MariaDB Galera. Устанавливаем на 3 удалённых системах OC NixOS с такой конфигурацией (используется конфигурация, как в прошлой статье):

{ config, pkgs, ... }:
{
  imports = [
    ./hardware-configuration.nix
  ];

  boot.loader.grub.enable = true;
  boot.loader.grub.version = 2;
  boot.loader.grub.device = "/dev/vda";

  networking.hostName = "basic-nixops";

  environment.systemPackages = with pkgs; [
    wget vim mkpasswd
  ];

  services.openssh = {
    enable = true;
    passwordAuthentication = false;
    permitRootLogin = "yes";
  };

  users = {
    mutableUsers = false;
    users.root = {
      openssh.authorizedKeys.keys = [
        "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAII5rqPqf/eFAyAEPZhwX/Hg7sNLZj4LWEexxsnv6izMW rebrain@NixOS-example"
      ];
      hashedPassword = null;
    };
    users.rebrain = {
      isNormalUser = true;
      uid = 51011;
      group = "rebrain";
      extraGroups = [ "wheel" "users" ];
      hashedPassword = "$6$zejLDpVGQfekr$no6c35WweI7j59W8diZ2pZwA2xadi5NxJNacLkBBoOmyt/4Lqt/pDX2pO3vUw157eb59XUZ71GZcMXQs2FqKL/";
    };
    groups.rebrain = {
     gid = 51011;
    };
  };

  system.stateVersion = "20.09";
}

Значения hashedPassword и openssh.authorizedKeys.keys у вас будут отличаться.

Подготовим структуру для деплоя:

mkdir -p ~/works/nixops/deploy/labs-galera/{galera1,galera2,galera3}

Создадим конфигурационные файлы:

nano ~/works/nixops/deploy/labs-galera.nix
{
  network.description = "labs galera servers";
  network.enableRollback = true;

  galera1 = { config, lib, pkgs, ... }: {
    require = [ ./labs-galera/galera1/configuration.nix ];

    deployment.targetEnv = "none";
    deployment.targetHost = "192.168.0.231";
    deployment.targetPort = 22;
  };

  galera2 = { config, lib, pkgs, ... }: {
    require = [ ./labs-galera/galera2/configuration.nix ];

    deployment.targetEnv = "none";
    deployment.targetHost = "192.168.0.232";
    deployment.targetPort = 22;
  };

  galera3 = { config, lib, pkgs, ... }: {
    require = [ ./labs-galera/galera3/configuration.nix ];

    deployment.targetEnv = "none";
    deployment.targetHost = "192.168.0.233";
    deployment.targetPort = 22;
  };
}

Где deployment.targetHost = "192.168.0.231"; - указываем IP адрес, который назначен нашей установленной удаленной системе. В вашем случае IP адрес будет отличаться.

nano ~/works/nixops/deploy/labs-galera/galera1/configuration.nix
{
  imports =[
    ./../../../config-defs/core.nix
    ./hardware-configuration.nix
    ./users.nix
  ];

  boot = {
    loader.grub.device = "/dev/vda";
    kernelParams = [ "lockdown=confidentiality" ];
  };

  networking.hostName = "vm11-galera1";
}
nano ~/works/nixops/deploy/labs-galera/galera2/configuration.nix
{
  imports =[
    ./../../../config-defs/core.nix
    ./hardware-configuration.nix
    ./users.nix
  ];

  boot = {
    loader.grub.device = "/dev/vda";
    kernelParams = [ "lockdown=confidentiality" ];
  };

  networking.hostName = "vm12-galera2";
}
nano ~/works/nixops/deploy/labs-galera/galera3/configuration.nix
{
  imports =[
    ./../../../config-defs/core.nix
    ./hardware-configuration.nix
    ./users.nix
  ];

  boot = {
    loader.grub.device = "/dev/vda";
    kernelParams = [ "lockdown=confidentiality" ];
  };

  networking.hostName = "vm13-galera3";
}

Не забываем скопировать с удаленных систем конфигурацию hardware-configuration.nix:

scp root@192.168.0.231:/etc/nixos/hardware-configuration.nix ~/works/nixops/deploy/labs-galera/galera1/hardware-configuration.nix
scp root@192.168.0.232:/etc/nixos/hardware-configuration.nix ~/works/nixops/deploy/labs-galera/galera2/hardware-configuration.nix
scp root@192.168.0.233:/etc/nixos/hardware-configuration.nix ~/works/nixops/deploy/labs-galera/galera3/hardware-configuration.nix
The authenticity of host '192.168.0.231 (192.168.0.231)' can't be established.
ED25519 key fingerprint is SHA256:qSFWEYnxgK/9mSbRDvE1PfBOJC/zSO6ySzHk038Tfj8.
Are you sure you want to continue connecting (yes/no/[fingerprint])? yes
Warning: Permanently added '192.168.0.231' (ED25519) to the list of known hosts.
Enter passphrase for key '/home/rebrain/.ssh/id_ed25519':
hardware-configuration.nix                                                                                                    100%  735     1.3MB/s   00:00
The authenticity of host '192.168.0.232 (192.168.0.232)' can't be established.
ED25519 key fingerprint is SHA256:8eljNfpuyksXiFialZfQV31wuLv/RvLGmI5t26aPKsw.
Are you sure you want to continue connecting (yes/no/[fingerprint])? yes
Warning: Permanently added '192.168.0.232' (ED25519) to the list of known hosts.
Enter passphrase for key '/home/rebrain/.ssh/id_ed25519':
hardware-configuration.nix                                                                                                    100%  735     1.2MB/s   00:00
The authenticity of host '192.168.0.233 (192.168.0.233)' can't be established.
ED25519 key fingerprint is SHA256:U2tQCYIr2tNT+DdGkZCAQ6XSu/ZzGk/D6c8QQqp96Lw.
Are you sure you want to continue connecting (yes/no/[fingerprint])? yes
Warning: Permanently added '192.168.0.233' (ED25519) to the list of known hosts.
Enter passphrase for key '/home/rebrain/.ssh/id_ed25519':
hardware-configuration.nix                                                                                                    100%  735     1.2MB/s   00:00

В файле users.nix в разделе users.root.openssh.authorizedKeys.keys прописываем ssh ключ, который сгенерировали выше. В разделе users.rebrain.openssh.authorizedKeys.keys уже прописан ключ, который мы сгенерировали при первом знакомстве с NixOS.

nano ~/works/nixops/deploy/labs-galera/galera1/users.nix
{ config, ... }:
let
  ssh-keys = import ./../../../config-defs/generic/security/ssh-keys.nix;

in {
  users = with ssh-keys; {
    users.root = {
      openssh.authorizedKeys.keys = [
        "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAII5rqPqf/eFAyAEPZhwX/Hg7sNLZj4LWEexxsnv6izMW rebrain@NixOS-example"
      ];
      hashedPassword = null;
    };
    users.rebrain = {
      isNormalUser = true;
      uid = config.uid-gid.rebrain;
      group = "rebrain";
      extraGroups = [ "wheel" "users" ];
      openssh.authorizedKeys.keys = [
        work.rebrain_example
      ];
      hashedPassword = "$6$zejLDpVGQfekr$no6c35WweI7j59W8diZ2pZwA2xadi5NxJNacLkBBoOmyt/4Lqt/pDX2pO3vUw157eb59XUZ71GZcMXQs2FqKL/";
    };
    groups.rebrain = {
      gid = config.uid-gid.rebrain;
    };
    groups.ssh-users = {
      members = [ "root" "rebrain" ];
    };
  };
}

Копируем файл users.nix на остальные удалённые системы:

cp ~/works/nixops/deploy/labs-galera/galera1/users.nix ~/works/nixops/deploy/labs-galera/galera2/users.nix
cp ~/works/nixops/deploy/labs-galera/galera1/users.nix ~/works/nixops/deploy/labs-galera/galera3/users.nix

Создаём деплой labs-galera:

nixops create -d labs-galera ~/works/nixops/deploy/labs-galera.nix
created deployment ‘b37dab82-e16e-11ea-be56-525400c283bd’
b37dab82-e16e-11ea-be56-525400c283bd

Просмотрим информацию о нашем деплое:

nixops info -d labs-galera
Network name: labs-galera
Network UUID: b37dab82-e16e-11ea-be56-525400c283bd
Network description: labs galera servers
Nix expressions: /home/rebrain/works/nixops/deploy/labs-galera.nix
Nix profile: /nix/var/nix/profiles/per-user/rebrain/nixops/b37dab82-e16e-11ea-be56-525400c283bd

+---------+---------+------+-------------+------------+
| Name    |  Status | Type | Resource Id | IP address |
+---------+---------+------+-------------+------------+
| galera1 | Missing | none |             |            |
| galera2 | Missing | none |             |            |
| galera3 | Missing | none |             |            |
+---------+---------+------+-------------+------------+

Если мы создали в прошлой статье SSH ключ с паролем, то, чтобы каждый раз не вводить пароль, предварительно загрузим его командой ssh-add:

ssh-add ~/.ssh/id_ed25519
Enter passphrase for /home/rebrain/.ssh/id_ed25519:
Identity added: /home/rebrain/.ssh/id_ed25519 (rebrain@NixOS-example)

Разворачиваем нашу конфигурацию:

nixops deploy -d labs-galera
building all machine configurations...
unpacking 'https://github.com/nix-community/NUR/archive/master.tar.gz'...
building '/nix/store/i6lzdz8qjisj7ja9jdmj1ps86s645khn-source.drv'...

trying https://github.com/Izorkin/nur-packages/archive/62e1c57f85af85425009ce1c9a2e355736f4869f.zip
  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current
                                 Dload  Upload   Total   Spent    Left  Speed
100   159  100   159    0     0    436      0 --:--:-- --:--:-- --:--:--   436
100  144k    0  144k    0     0   118k      0 --:--:--  0:00:01 --:--:--  396k
unpacking source archive /build/62e1c57f85af85425009ce1c9a2e355736f4869f.zip
these derivations will be built:
  /nix/store/00g78nym1gi6hpp86ghjbxmd1hnahpvs-extra-hosts.drv
  /nix/store/1yz7i6pjz7vmmrj4r41bbadgz00cc0gc-root-authorized_keys.drv
  /nix/store/plyy5mzbv9f6b0hnsf5dsqf7ks63ncj8-initrd-fsinfo.drv
  /nix/store/2avzn6g7y9ks5y9d4rzmm861m79jm87d-stage-1-init.sh.drv
  /nix/store/amc644rci6jm96nm2dvx9f79vlv2sxnm-string-hosts.drv
  /nix/store/3bdd4xykarg6djym9b86wy3v8mpwn7wm-hosts.drv
  /nix/store/fy4hyq6r4zbhp1cyvydd4vi2kgs2bf7s-extra-hosts.drv
  /nix/store/mrv3svj7xbsfp526iqpg4w3waz1n24f4-string-hosts.drv
  /nix/store/pid2aah3y9kh5drqgc8476c7da2lph0q-hosts.drv
  /nix/store/3x1nl3n38r1mg7ba3b0c1i2k6dlbi243-unit-nscd.service.drv
  /nix/store/49ppnlyihiqkh2szwwnmbmlpwrsd3k80-root-authorized_keys.drv
...
building '/nix/store/lxaq0vz5z5ax9kpwbcsm04xl8zgxs8g5-nixos-system-vm11-galera1-20.09pre238361.33548111764.drv'...
building '/nix/store/7ndmd468hv4zfbi5akg8nvc5q12x5dil-nixops-machines.drv'...
galera1> copying closure...
galera3> copying closure...
galera2> copying closure...
galera2> copying path '/nix/store/gqp6v7n851dv5masw3qnlcjbzcsw9842-acl-2.2.53-doc' from 'https://cache.nixos.org'...
galera2> copying path '/nix/store/43na49ygxsqwp5z367pi07h76rg42f91-acl-2.2.53-man' from 'https://cache.nixos.org'...
galera3> copying path '/nix/store/gqp6v7n851dv5masw3qnlcjbzcsw9842-acl-2.2.53-doc' from 'https://cache.nixos.org'...
galera3> copying path '/nix/store/43na49ygxsqwp5z367pi07h76rg42f91-acl-2.2.53-man' from 'https://cache.nixos.org'...
galera1> copying path '/nix/store/gqp6v7n851dv5masw3qnlcjbzcsw9842-acl-2.2.53-doc' from 'https://cache.nixos.org'...
galera1> copying path '/nix/store/43na49ygxsqwp5z367pi07h76rg42f91-acl-2.2.53-man' from 'https://cache.nixos.org'...
galera3> copying path '/nix/store/g4snl6w2bd6x5aaaabwvhza82ph6cfx2-apparmor-profiles-2.13.4' from 'https://cache.nixos.org'...
...
galera2> the following new units were started: chronyd.service, disable-kernel-module-loading.service, encrypted-links.target, logrotate.timer, nix-optimise.timer, qemu-guest-agent.service, sshd.socket, syslog-ng.service, unbound.service
galera2> warning: the following units failed: apparmor.service
galera2>
galera2> ● apparmor.service
galera2>      Loaded: loaded (/nix/store/5db40ssdzmzhwb26ii0247v08hl2l2qn-unit-apparmor.service/apparmor.service; enabled; vendor preset: enabled)
galera2>      Active: failed (Result: exit-code) since Tue 2020-08-18 19:32:20 MSK; 447ms ago
galera2>     Process: 2166 ExecStart=/nix/store/mlwjiwjw3x8pvq7gijy3wr28im7n5nx7-apparmor-parser-2.13.4/bin/apparmor_parser -rKv -I /nix/store/g4snl6w2bd6x5aaaabwvhza82ph6cfx2-apparmor-profiles-2.13.4/etc/apparmor.d /nix/store/h5jpz7vxiikfwy3in2fbi7gipzi2k49c-ping (code=exited, status=1/FAILURE)
galera2>    Main PID: 2166 (code=exited, status=1/FAILURE)
galera2>          IP: 0B in, 0B out
galera2>         CPU: 3ms
galera2>
galera2> Aug 18 19:32:20 vm12-galera2 systemd[1]: Starting apparmor.service...
galera2> Aug 18 19:32:20 vm12-galera2 apparmor_parser[2166]: Warning from stdin (line 1): config file '/etc/apparmor/parser.conf' not found
galera2> Aug 18 19:32:20 vm12-galera2 apparmor_parser[2166]: Cache read/write disabled: interface file missing. (Kernel needs AppArmor 2.4 compatibility patch.)
galera2> Aug 18 19:32:20 vm12-galera2 apparmor_parser[2166]: Warning: unable to find a suitable fs in /proc/mounts, is it mounted?
galera2> Aug 18 19:32:20 vm12-galera2 apparmor_parser[2166]: Use --subdomainfs to override.
galera2> Aug 18 19:32:20 vm12-galera2 systemd[1]: apparmor.service: Main process exited, code=exited, status=1/FAILURE
galera2> Aug 18 19:32:20 vm12-galera2 systemd[1]: apparmor.service: Failed with result 'exit-code'.
galera2> Aug 18 19:32:20 vm12-galera2 systemd[1]: Failed to start apparmor.service.
galera2> error: Traceback (most recent call last):
  File "/nix/store/ihb5kkmbds0f5zvxl7mb51mj4sgkcys8-nixops-1.7/lib/python2.7/site-packages/nixops/deployment.py", line 743, in worker
    raise Exception("unable to activate new configuration (exit code {})".format(res))
Exception: unable to activate new configuration (exit code 4)

Traceback (most recent call last):
  File "/nix/store/ihb5kkmbds0f5zvxl7mb51mj4sgkcys8-nixops-1.7/bin/..nixops-wrapped-wrapped", line 991, in <module>
    args.op()
  File "/nix/store/ihb5kkmbds0f5zvxl7mb51mj4sgkcys8-nixops-1.7/bin/..nixops-wrapped-wrapped", line 412, in op_deploy
    max_concurrent_activate=args.max_concurrent_activate)
  File "/nix/store/ihb5kkmbds0f5zvxl7mb51mj4sgkcys8-nixops-1.7/lib/python2.7/site-packages/nixops/deployment.py", line 1063, in deploy
    self.run_with_notify('deploy', lambda: self._deploy(**kwargs))
  File "/nix/store/ihb5kkmbds0f5zvxl7mb51mj4sgkcys8-nixops-1.7/lib/python2.7/site-packages/nixops/deployment.py", line 1052, in run_with_notify
    f()
  File "/nix/store/ihb5kkmbds0f5zvxl7mb51mj4sgkcys8-nixops-1.7/lib/python2.7/site-packages/nixops/deployment.py", line 1063, in <lambda>
    self.run_with_notify('deploy', lambda: self._deploy(**kwargs))
  File "/nix/store/ihb5kkmbds0f5zvxl7mb51mj4sgkcys8-nixops-1.7/lib/python2.7/site-packages/nixops/deployment.py", line 1019, in _deploy
    dry_activate=dry_activate, max_concurrent_activate=max_concurrent_activate)
  File "/nix/store/ihb5kkmbds0f5zvxl7mb51mj4sgkcys8-nixops-1.7/lib/python2.7/site-packages/nixops/deployment.py", line 775, in activate_configs
    .format(len(failed), len(res), ", ".join(["‘{0}’".format(x) for x in failed])))
Exception: activation of 3 of 3 machines failed (namely on ‘galera1’, ‘galera3’, ‘galera2’)
Time: 0h:03m:51s

Просмотрим ещё раз информацию о нашем деплое:

nixops info -d labs-galera

Network name: labs-galera Network UUID: b37dab82-e16e-11ea-be56-525400c283bd Network description: labs galera servers Nix expressions: /home/rebrain/works/nixops/deploy/labs-galera.nix Nix profile: /nix/var/nix/profiles/per-user/rebrain/nixops/b37dab82-e16e-11ea-be56-525400c283bd

+---------+---------------+------+-----------------------------------------------------+------------+
| Name    |     Status    | Type | Resource Id                                         | IP address |
+---------+---------------+------+-----------------------------------------------------+------------+
| galera1 | Up / Outdated | none | nixops-b37dab82-e16e-11ea-be56-525400c283bd-galera1 |            |
| galera2 | Up / Outdated | none | nixops-b37dab82-e16e-11ea-be56-525400c283bd-galera2 |            |
| galera3 | Up / Outdated | none | nixops-b37dab82-e16e-11ea-be56-525400c283bd-galera3 |            |
+---------+---------------+------+-----------------------------------------------------+------------+
Time: 0h:00m:03s

Начальная настройка закончена, сохраним коммит в git:

cd ~/works/nixops/deploy
git add -A
git commit -m "add labs-galera"
[master 98392fd] add labs-galera
 10 files changed, 221 insertions(+)
 create mode 100644 labs-galera.nix
 create mode 100644 labs-galera/galera1/configuration.nix
 create mode 100644 labs-galera/galera1/hardware-configuration.nix
 create mode 100644 labs-galera/galera1/users.nix
 create mode 100644 labs-galera/galera2/configuration.nix
 create mode 100644 labs-galera/galera2/hardware-configuration.nix
 create mode 100644 labs-galera/galera2/users.nix
 create mode 100644 labs-galera/galera3/configuration.nix
 create mode 100644 labs-galera/galera3/hardware-configuration.nix
 create mode 100644 labs-galera/galera3/users.nix

Сейчас мы сделали начальную настройку удалённых систем. Сперва добавим основные службы - файерволл, fail2ban и мониторинг netdata:

cp -r /etc/nixos/nix-config/servers/example/services ~/works/nixops/deploy/labs-galera/galera1
cp -r /etc/nixos/nix-config/servers/example/services ~/works/nixops/deploy/labs-galera/galera2
cp -r /etc/nixos/nix-config/servers/example/services ~/works/nixops/deploy/labs-galera/galera3

В файлы configuration.nix добавляем новые сервисы:

nano ~/works/nixops/deploy/labs-galera/galera1/configuration.nix
nano ~/works/nixops/deploy/labs-galera/galera2/configuration.nix
nano ~/works/nixops/deploy/labs-galera/galera3/configuration.nix
...
  imports =[
    ./../../../config-defs/core.nix
    ./services/fail2ban.nix
    ./services/firefall-nft.nix
    ./services/netdata.nix
    ./hardware-configuration.nix
...

Должно получиться так:

git diff
diff --git a/labs-galera/galera1/configuration.nix b/labs-galera/galera1/configuration.nix
index 7ae5cbd..058e56c 100644
--- a/labs-galera/galera1/configuration.nix
+++ b/labs-galera/galera1/configuration.nix
@@ -1,6 +1,9 @@
 {
   imports =[
     ./../../../config-defs/core.nix
+    ./services/fail2ban.nix
+    ./services/firefall-nft.nix
+    ./services/netdata.nix
     ./hardware-configuration.nix
     ./users.nix
   ];
diff --git a/labs-galera/galera2/configuration.nix b/labs-galera/galera2/configuration.nix
index 360ed9b..27987e7 100644
--- a/labs-galera/galera2/configuration.nix
+++ b/labs-galera/galera2/configuration.nix
@@ -1,6 +1,9 @@
 {
   imports =[
     ./../../../config-defs/core.nix
+    ./services/fail2ban.nix
+    ./services/firefall-nft.nix
+    ./services/netdata.nix
     ./hardware-configuration.nix
     ./users.nix
   ];
diff --git a/labs-galera/galera3/configuration.nix b/labs-galera/galera3/configuration.nix
index 4290170..93abbcd 100644
--- a/labs-galera/galera3/configuration.nix
+++ b/labs-galera/galera3/configuration.nix
@@ -1,6 +1,9 @@
 {
   imports =[
     ./../../../config-defs/core.nix
+    ./services/fail2ban.nix
+    ./services/firefall-nft.nix
+    ./services/netdata.nix
     ./hardware-configuration.nix
     ./users.nix
   ];

Добавляем изменения в git:

git add -A
git commit -m "labs-galera: add default services"
[master 42ee147] labs-galera: add default services
 12 files changed, 153 insertions(+)
 create mode 100644 labs-galera/galera1/services/fail2ban.nix
 create mode 100644 labs-galera/galera1/services/firefall-nft.nix
 create mode 100644 labs-galera/galera1/services/netdata.nix
 create mode 100644 labs-galera/galera2/services/fail2ban.nix
 create mode 100644 labs-galera/galera2/services/firefall-nft.nix
 create mode 100644 labs-galera/galera2/services/netdata.nix
 create mode 100644 labs-galera/galera3/services/fail2ban.nix
 create mode 100644 labs-galera/galera3/services/firefall-nft.nix
 create mode 100644 labs-galera/galera3/services/netdata.nix

Обновляем удалённые системы:

nixops deploy -d labs-galera
building all machine configurations...
these derivations will be built:
  /nix/store/pb8cclhhmxnfl23js0k8gj4idbg799y4-nixos.conf.drv
  /nix/store/9irh6j2amz8ypcrsx565mxxbx6z5v1iz-unit-systemd-modules-load.service.drv
  /nix/store/fq81023s7sdqq7ahhljbhvj8cgd1xal7-system-units.drv
  /nix/store/czr08w51hpqsmdb2hb7sxih7dyn8369x-etc.drv
  /nix/store/1f9wj8f7x4aaidwnsnafi5zrgr8hamzc-nixos-system-vm11-galera1-20.09pre238361.33548111764.drv
  /nix/store/rqxp9g028f7pvzlngvc2z5r5xlvnhw78-system-units.drv
  /nix/store/pxbdrfjadsqdigbj41mgayq3sl4m1vpf-etc.drv
  /nix/store/hspfps3hxhdjb2f4gc1sb2y0vada2mnj-nixos-system-vm13-galera3-20.09pre238361.33548111764.drv
  /nix/store/gasjnbwxgzjn7ra0i17387l67z06pnnd-system-units.drv
  /nix/store/qyfn3d90yvjd617qidij7nm0s2hfjxx5-etc.drv
  /nix/store/knm13m5ivncwbl513r1wkyl12ff5scq4-nixos-system-vm12-galera2-20.09pre238361.33548111764.drv
  /nix/store/7z2vh22p62df17cwyjr6dhmng0p66haj-nixops-machines.drv
building '/nix/store/pb8cclhhmxnfl23js0k8gj4idbg799y4-nixos.conf.drv'...
building '/nix/store/9irh6j2amz8ypcrsx565mxxbx6z5v1iz-unit-systemd-modules-load.service.drv'...
building '/nix/store/fq81023s7sdqq7ahhljbhvj8cgd1xal7-system-units.drv'...
building '/nix/store/gasjnbwxgzjn7ra0i17387l67z06pnnd-system-units.drv'...
building '/nix/store/rqxp9g028f7pvzlngvc2z5r5xlvnhw78-system-units.drv'...
building '/nix/store/czr08w51hpqsmdb2hb7sxih7dyn8369x-etc.drv'...
building '/nix/store/qyfn3d90yvjd617qidij7nm0s2hfjxx5-etc.drv'...
building '/nix/store/pxbdrfjadsqdigbj41mgayq3sl4m1vpf-etc.drv'...
building '/nix/store/1f9wj8f7x4aaidwnsnafi5zrgr8hamzc-nixos-system-vm11-galera1-20.09pre238361.33548111764.drv'...
building '/nix/store/knm13m5ivncwbl513r1wkyl12ff5scq4-nixos-system-vm12-galera2-20.09pre238361.33548111764.drv'...
building '/nix/store/hspfps3hxhdjb2f4gc1sb2y0vada2mnj-nixos-system-vm13-galera3-20.09pre238361.33548111764.drv'...
building '/nix/store/7z2vh22p62df17cwyjr6dhmng0p66haj-nixops-machines.drv'...
galera1> copying closure...
galera3> copying closure...
galera2> copying closure...
galera2> copying path '/nix/store/i65m49njwrdjqscpbrglva67993l9vx7-wrapped-plugins' from 'https://cache.nixos.org'...
galera2> copying path '/nix/store/01113j97n0pymlrlzwmrnsrrl9mm9fdy-freeipmi-1.6.5' from 'https://cache.nixos.org'...
galera3> copying path '/nix/store/i65m49njwrdjqscpbrglva67993l9vx7-wrapped-plugins' from 'https://cache.nixos.org'...
galera3> copying path '/nix/store/01113j97n0pymlrlzwmrnsrrl9mm9fdy-freeipmi-1.6.5' from 'https://cache.nixos.org'...
galera3> copying path '/nix/store/i0zbc4g3h2qvyvc02h0d0gm0q6nv4dzg-jansson-2.13.1' from 'https://cache.nixos.org'...
...
galera1> setting up tmpfiles
galera1> reloading the following units: dbus.service
galera1> starting the following units: systemd-modules-load.service
galera1> the following new units were started: fail2ban.service, logrotate.service, netdata.service
galera1> warning: the following units failed: nftables.service
galera1>
galera1> ● nftables.service - nftables firewall
galera1>      Loaded: loaded (/nix/store/9ir93n0mzn57c9x8yrh1m28lbl6szs2a-unit-nftables.service/nftables.service; enabled; vendor preset: enabled)
galera1>      Active: failed (Result: exit-code) since Tue 2020-08-18 19:56:21 MSK; 40ms ago
galera1>     Process: 1122 ExecStart=/nix/store/0pns2jkqpv6pddrbimpxi6kr47xwsnm4-nftables-check (code=exited, status=1/FAILURE)
galera1>    Main PID: 1122 (code=exited, status=1/FAILURE)
galera1>          IP: 0B in, 0B out
galera1>         CPU: 24ms
galera1>
galera1> авг 18 19:56:20 vm11-galera1 systemd[1]: Starting nftables firewall...
galera1> авг 18 19:56:21 vm11-galera1 0pns2jkqpv6pddrbimpxi6kr47xwsnm4-nftables-check[1122]: Unload ip_tables before using nftables!
galera1> авг 18 19:56:21 vm11-galera1 systemd[1]: nftables.service: Main process exited, code=exited, status=1/FAILURE
galera1> авг 18 19:56:21 vm11-galera1 systemd[1]: nftables.service: Failed with result 'exit-code'.
galera1> авг 18 19:56:21 vm11-galera1 systemd[1]: Failed to start nftables firewall.
galera1> error: Traceback (most recent call last):
  File "/nix/store/ihb5kkmbds0f5zvxl7mb51mj4sgkcys8-nixops-1.7/lib/python2.7/site-packages/nixops/deployment.py", line 743, in worker
    raise Exception("unable to activate new configuration (exit code {})".format(res))
Exception: unable to activate new configuration (exit code 4)

Traceback (most recent call last):
  File "/nix/store/ihb5kkmbds0f5zvxl7mb51mj4sgkcys8-nixops-1.7/bin/..nixops-wrapped-wrapped", line 991, in <module>
    args.op()
  File "/nix/store/ihb5kkmbds0f5zvxl7mb51mj4sgkcys8-nixops-1.7/bin/..nixops-wrapped-wrapped", line 412, in op_deploy
    max_concurrent_activate=args.max_concurrent_activate)
  File "/nix/store/ihb5kkmbds0f5zvxl7mb51mj4sgkcys8-nixops-1.7/lib/python2.7/site-packages/nixops/deployment.py", line 1063, in deploy
    self.run_with_notify('deploy', lambda: self._deploy(**kwargs))
  File "/nix/store/ihb5kkmbds0f5zvxl7mb51mj4sgkcys8-nixops-1.7/lib/python2.7/site-packages/nixops/deployment.py", line 1052, in run_with_notify
    f()
  File "/nix/store/ihb5kkmbds0f5zvxl7mb51mj4sgkcys8-nixops-1.7/lib/python2.7/site-packages/nixops/deployment.py", line 1063, in <lambda>
    self.run_with_notify('deploy', lambda: self._deploy(**kwargs))
  File "/nix/store/ihb5kkmbds0f5zvxl7mb51mj4sgkcys8-nixops-1.7/lib/python2.7/site-packages/nixops/deployment.py", line 1019, in _deploy
    dry_activate=dry_activate, max_concurrent_activate=max_concurrent_activate)
  File "/nix/store/ihb5kkmbds0f5zvxl7mb51mj4sgkcys8-nixops-1.7/lib/python2.7/site-packages/nixops/deployment.py", line 775, in activate_configs
    .format(len(failed), len(res), ", ".join(["‘{0}’".format(x) for x in failed])))
Exception: activation of 3 of 3 machines failed (namely on ‘galera2’, ‘galera3’, ‘galera1’)
Time: 0h:00m:59s

Перезагружаем наши удалённые системы:

nixops reboot -d labs-galera
galera1> rebooting...
galera3> rebooting...
galera2> rebooting...
Connection to 192.168.0.232 closed by remote host.
Connection to 192.168.0.233 closed by remote host.
Connection to 192.168.0.231 closed by remote host.
galera2> waiting for the machine to finish rebooting...
galera3> waiting for the machine to finish rebooting...
galera1> waiting for the machine to finish rebooting...
galera3> [down]
galera2> [down]
galera1> [down]
galera3> .
...
galera3> .
galera2> [up]
galera1> .
galera3> .
galera1> [up]
galera3> [up]
Time: 0h:00m:23s

Теперь приступим к настройке MariaDB Galera. Минимальная конфигурация кластера выглядит так:

nano ~/works/nixops/deploy/labs-galera/galera1/services/mysql.nix
{ pkgs, ... }:
{
  networking = {
    extraHosts = ''
      192.168.0.231 galera_01
      192.168.0.232 galera_02
      192.168.0.233 galera_03
    '';
  };
  systemd.services.mysql = with pkgs; {
    path = [ bash gawk gnutar inetutils which gzip iproute netcat procps pv socat ];
  };
  services.mysql = {
    enable = true;
    package = pkgs.mariadb;
    initialScript = pkgs.writeText "mariadb-init.sql" ''
      GRANT RELOAD, PROCESS, LOCK TABLES, REPLICATION CLIENT ON *.* TO 'repl_user'@'localhost' IDENTIFIED BY 'repl_pass';
      GRANT USAGE ON *.* TO `netdata`@`localhost` IDENTIFIED BY 'netdata_pass';
      FLUSH PRIVILEGES;
    '';
    settings = {
      mysqld = {
        bind_address = "0.0.0.0";
      };
      galera = {
        wsrep_on = "ON";
        wsrep_debug = "NONE";
        wsrep_retry_autocommit = "3";
        wsrep_provider = "${pkgs.mariadb-galera}/lib/galera/libgalera_smm.so";
        wsrep_cluster_address = "gcomm://";
        wsrep_cluster_name = "galera";
        wsrep_node_address = "galera_01";
        wsrep_node_name = "galera_01";
        wsrep_sst_method = "mariabackup";
        wsrep_sst_auth = "repl_user:repl_pass";
        binlog_format = "ROW";
        enforce_storage_engine = "InnoDB";
        innodb_autoinc_lock_mode = "2";
      };
    };
  };
}

Где:

  • networking = { extraHosts - прописывает имена хостов для наших удалённых серверов, которые будут использоваться сервисом MariaDB Galera.
  • services.mysql.package = pkgs.mariadb; - указываем использовать пакет MariaDB.
  • initialScript = pkgs.writeText "mariadb-init.sql" - SQL скрипт, который выполняется при первоначальной инициализации базы данных MariaDB. Здесь добавляем 2 пользователей: repl_user для репликации баз данных и netdata для мониторинга.
  • services.mysql.settings.mysqld.bind_address = 0.0.0.0"; - указываем слушать все интерфейсы.
  • services.mysql.settings.mysqld.galera.wsrep_on = "ON"; - активируем MariaDB Galera кластер.
  • services.mysql.settings.mysqld.galera.wsrep_debug = "NONE"; - отключаем DEBUG режим.
  • services.mysql.settings.mysqld.galera.wsrep_retry_autocommit = "3"; - указываем количество повторных попыток выполненных запросов из-за конфликтов внутри кластера перед возвратом ошибки клиенту.
  • services.mysql.settings.mysqld.galera.wsrep_provider = "${pkgs.mariadb-galera}/lib/galera/libgalera_smm.so"; - указываем путь к библиотеке MariaDB Galera.
  • services.mysql.settings.mysqld.galera.wsrep_cluster_address = "gcomm://"; - здесь перечисляем все ноды MariaDB Galera. Обратите внимание, при первом запуске кластера на первой ноде указываем пустое значение gcomm://.
  • services.mysql.settings.mysqld.galera.wsrep_cluster_name = "galera"; - указываем название кластера.
  • services.mysql.settings.mysqld.galera.wsrep_node_address = "galera_01"; - указываем адрес ноды, совпадает с именем в хост-файле.
  • services.mysql.settings.mysqld.galera.wsrep_node_name = "galera_01"; - указываем имя ноды.
  • services.mysql.settings.mysqld.galera.wsrep_sst_method = "mariabackup"; - указываем метод SST репликации.
  • services.mysql.settings.mysqld.galera.wsrep_sst_auth = "repl_user:repl_pass"; - указываем логин и пароль mysql пользователя для репликации.
  • services.mysql.settings.mysqld.galera.binlog_format = "ROW"; - указываем использовать ROW формат бин-логов.
  • services.mysql.settings.mysqld.galera.enforce_storage_engine = "InnoDB"; - запрещаем использовать все форматы базы данных, кроме InnoDB.
  • services.mysql.settings.mysqld.galera.innodb_autoinc_lock_mode = "2"; - указываем использовать interleaved режим блокировки.
nano ~/works/nixops/deploy/labs-galera/galera2/services/mysql.nix
{ pkgs, ... }:
{
  networking = {
    extraHosts = ''
      192.168.0.231 galera_01
      192.168.0.232 galera_02
      192.168.0.233 galera_03
    '';
  };
  systemd.services.mysql = with pkgs; {
    path = [ bash gawk gnutar inetutils which gzip iproute netcat procps pv socat ];
  };
  services.mysql = {
    enable = true;
    package = pkgs.mariadb;
    initialScript = pkgs.writeText "mariadb-init.sql" ''
      GRANT RELOAD, PROCESS, LOCK TABLES, REPLICATION CLIENT ON *.* TO 'repl_user'@'localhost' IDENTIFIED BY 'repl_pass';
      GRANT USAGE ON *.* TO `netdata`@`localhost` IDENTIFIED BY 'netdata_pass';
      FLUSH PRIVILEGES;
    '';
    settings = {
      mysqld = {
        bind_address = "0.0.0.0";
      };
      galera = {
        wsrep_on = "ON";
        wsrep_debug = "NONE";
        wsrep_retry_autocommit = "3";
        wsrep_provider = "${pkgs.mariadb-galera}/lib/galera/libgalera_smm.so";
        wsrep_cluster_address = "gcomm://galera_01,galera_02,galera_03";
        wsrep_cluster_name = "galera";
        wsrep_node_address = "galera_02";
        wsrep_node_name = "galera_02";
        wsrep_sst_method = "mariabackup";
        wsrep_sst_auth = "repl_user:repl_pass";
        binlog_format = "ROW";
        enforce_storage_engine = "InnoDB";
        innodb_autoinc_lock_mode = "2";
      };
    };
  };
}
nano ~/works/nixops/deploy/labs-galera/galera3/services/mysql.nix
{ pkgs, ... }:
{
  networking = {
    extraHosts = ''
      192.168.0.231 galera_01
      192.168.0.232 galera_02
      192.168.0.233 galera_03
    '';
  };
  systemd.services.mysql = with pkgs; {
    path = [ bash gawk gnutar inetutils which gzip iproute netcat procps pv socat ];
  };
  services.mysql = {
    enable = true;
    package = pkgs.mariadb;
    initialScript = pkgs.writeText "mariadb-init.sql" ''
      GRANT RELOAD, PROCESS, LOCK TABLES, REPLICATION CLIENT ON *.* TO 'repl_user'@'localhost' IDENTIFIED BY 'repl_pass';
      GRANT USAGE ON *.* TO `netdata`@`localhost` IDENTIFIED BY 'netdata_pass';
      FLUSH PRIVILEGES;
    '';
    settings = {
      mysqld = {
        bind_address = "0.0.0.0";
      };
      galera = {
        wsrep_on = "ON";
        wsrep_debug = "NONE";
        wsrep_retry_autocommit = "3";
        wsrep_provider = "${pkgs.mariadb-galera}/lib/galera/libgalera_smm.so";
        wsrep_cluster_address = "gcomm://galera_01,galera_02,galera_03";
        wsrep_cluster_name = "galera";
        wsrep_node_address = "galera_03";
        wsrep_node_name = "galera_03";
        wsrep_sst_method = "mariabackup";
        wsrep_sst_auth = "repl_user:repl_pass";
        binlog_format = "ROW";
        enforce_storage_engine = "InnoDB";
        innodb_autoinc_lock_mode = "2";
      };
    };
  };
}

Добавляем файл mysql.nix в конфигурацию configuration.nix:

nano ~/works/nixops/deploy/labs-galera/galera1/configuration.nix
nano ~/works/nixops/deploy/labs-galera/galera2/configuration.nix
nano ~/works/nixops/deploy/labs-galera/galera3/configuration.nix
...
    ./services/mysql.nix
    ./services/netdata.nix
    ./hardware-configuration.nix
...
git diff
diff --git a/labs-galera/galera1/configuration.nix b/labs-galera/galera1/configuration.nix
index 058e56c..f585a71 100644
--- a/labs-galera/galera1/configuration.nix
+++ b/labs-galera/galera1/configuration.nix
@@ -3,6 +3,7 @@
     ./../../../config-defs/core.nix
     ./services/fail2ban.nix
     ./services/firefall-nft.nix
+    ./services/mysql.nix
     ./services/netdata.nix
     ./hardware-configuration.nix
     ./users.nix
diff --git a/labs-galera/galera2/configuration.nix b/labs-galera/galera2/configuration.nix
index 27987e7..89827f3 100644
--- a/labs-galera/galera2/configuration.nix
+++ b/labs-galera/galera2/configuration.nix
@@ -3,6 +3,7 @@
     ./../../../config-defs/core.nix
     ./services/fail2ban.nix
     ./services/firefall-nft.nix
+    ./services/mysql.nix
     ./services/netdata.nix
     ./hardware-configuration.nix
     ./users.nix
diff --git a/labs-galera/galera3/configuration.nix b/labs-galera/galera3/configuration.nix
index 93abbcd..7b3a248 100644
--- a/labs-galera/galera3/configuration.nix
+++ b/labs-galera/galera3/configuration.nix
@@ -3,6 +3,7 @@
     ./../../../config-defs/core.nix
     ./services/fail2ban.nix
     ./services/firefall-nft.nix
+    ./services/mysql.nix
     ./services/netdata.nix
     ./hardware-configuration.nix
     ./users.nix

Добавляем изменения в git, но пока не сохраняем:

git add -A

Так как у нас установлен файерволл nftables необходимо открыть порты для корректной работы кластера:

nano ~/works/nixops/deploy/labs-galera/galera1/services/firefall-nft.nix
nano ~/works/nixops/deploy/labs-galera/galera2/services/firefall-nft.nix
nano ~/works/nixops/deploy/labs-galera/galera3/services/firefall-nft.nix
...
          ct state invalid drop
          ct state {established, related} accept
        }
        set allow-mysql {
          type ipv4_addr
          flags interval
          elements = { 192.168.0.0/24 }
        }
        set mariadb-galera {
          type ipv4_addr
          elements = { galera_01, galera_02, galera_03 }
        }
...
          tcp dport { 22 } ct state new accept
          tcp dport { 19999 } ct state new accept
          tcp dport { 3306  } ip saddr @allow-mysql ct state new accept
          tcp dport { 3306  } ip saddr @mariadb-galera ct state new accept
          tcp dport { 4444, 4567, 4568 } ip saddr @mariadb-galera ct state new accept
          udp dport { 4567 } ip saddr @mariadb-galera ct state new accept
        }

Где:

  • set allow-mysql { - указываем с каких IP адресов разрешён доступ к mysql серверу.
  • set mariadb-galera - здесь перечислены хосты нашего кластера.
  • tcp dport { 4444, 4567, 4568 } и udp dport { 4567 } - отркываем порты, которые используются для репликации кластером MariaDB Galera.

Текущие изменения в файерволле:

git diff
diff --git a/labs-galera/galera1/services/firefall-nft.nix b/labs-galera/galera1/services/firefall-nft.nix
index f6c2648..9bfd485 100644
--- a/labs-galera/galera1/services/firefall-nft.nix
+++ b/labs-galera/galera1/services/firefall-nft.nix
@@ -10,6 +10,15 @@
           ct state invalid drop
           ct state {established, related} accept
         }
+        set allow-mysql {
+          type ipv4_addr
+          flags interval
+          elements = { 192.168.0.0/24 }
+        }
+        set mariadb-galera {
+          type ipv4_addr
+          elements = { galera_01, galera_02, galera_03 }
+        }
         chain input {
           type filter hook input priority 0; policy drop;
           jump checks-base
@@ -17,6 +26,10 @@
           ip protocol icmp icmp type { echo-request} ct state new accept
           tcp dport { 22 } ct state new accept
           tcp dport { 19999 } ct state new accept
+          tcp dport { 3306  } ip saddr @allow-mysql ct state new accept
+          tcp dport { 3306  } ip saddr @mariadb-galera ct state new accept
+          tcp dport { 4444, 4567, 4568 } ip saddr @mariadb-galera ct state new accept
+          udp dport { 4567 } ip saddr @mariadb-galera ct state new accept
         }
         chain output {
           type filter hook output priority 0; policy accept;
diff --git a/labs-galera/galera2/services/firefall-nft.nix b/labs-galera/galera2/services/firefall-nft.nix
index f6c2648..9bfd485 100644
--- a/labs-galera/galera2/services/firefall-nft.nix
+++ b/labs-galera/galera2/services/firefall-nft.nix
@@ -10,6 +10,15 @@
           ct state invalid drop
           ct state {established, related} accept
         }
+        set allow-mysql {
+          type ipv4_addr
+          flags interval
+          elements = { 192.168.0.0/24 }
+        }
+        set mariadb-galera {
+          type ipv4_addr
+          elements = { galera_01, galera_02, galera_03 }
+        }
         chain input {
           type filter hook input priority 0; policy drop;
           jump checks-base
@@ -17,6 +26,10 @@
           ip protocol icmp icmp type { echo-request} ct state new accept
           tcp dport { 22 } ct state new accept
           tcp dport { 19999 } ct state new accept
+          tcp dport { 3306  } ip saddr @allow-mysql ct state new accept
+          tcp dport { 3306  } ip saddr @mariadb-galera ct state new accept
+          tcp dport { 4444, 4567, 4568 } ip saddr @mariadb-galera ct state new accept
+          udp dport { 4567 } ip saddr @mariadb-galera ct state new accept
         }
         chain output {
           type filter hook output priority 0; policy accept;
diff --git a/labs-galera/galera3/services/firefall-nft.nix b/labs-galera/galera3/services/firefall-nft.nix
index f6c2648..9bfd485 100644
--- a/labs-galera/galera3/services/firefall-nft.nix
+++ b/labs-galera/galera3/services/firefall-nft.nix
@@ -10,6 +10,15 @@
           ct state invalid drop
           ct state {established, related} accept
         }
+        set allow-mysql {
+          type ipv4_addr
+          flags interval
+          elements = { 192.168.0.0/24 }
+        }
+        set mariadb-galera {
+          type ipv4_addr
+          elements = { galera_01, galera_02, galera_03 }
+        }
         chain input {
           type filter hook input priority 0; policy drop;
           jump checks-base
@@ -17,6 +26,10 @@
           ip protocol icmp icmp type { echo-request} ct state new accept
           tcp dport { 22 } ct state new accept
           tcp dport { 19999 } ct state new accept
+          tcp dport { 3306  } ip saddr @allow-mysql ct state new accept
+          tcp dport { 3306  } ip saddr @mariadb-galera ct state new accept
+          tcp dport { 4444, 4567, 4568 } ip saddr @mariadb-galera ct state new accept
+          udp dport { 4567 } ip saddr @mariadb-galera ct state new accept
         }
         chain output {
           type filter hook output priority 0; policy accept;

Снова добавляем изменения в git, но не сохраняем:

git add -A

Теперь применим изменения на наших удалённых системах. Для того чтобы во время обновления не развалить кластер MariaDB Galera во время перезапуска службы mysql, рекомендуется обновлять удалённые системы по одной за раз. Для этого в утилите 'nixops' есть опция --include, которая позволяет указать с какой удаленной системой нам работать:

nixops deploy -d labs-galera --include galera1
building all machine configurations...
unpacking 'https://github.com/nix-community/NUR/archive/master.tar.gz'...
these derivations will be built:
  /nix/store/30dj3jh5i4y1pj2lrl0fvg3kwqwfp4v4-unit-script-mysql-pre-start.drv
  /nix/store/ny0pm6p1x5vimilmf7z6m9kp8369abja-extra-hosts.drv
  /nix/store/73mcyq5yxa2c1vwxm3yvpyavfi7xa17w-hosts.drv
  /nix/store/75sfizzpb58k0hq96zh7qnq537h8mpyk-users-groups.json.drv
...
building '/nix/store/gvifdl0gli22fajm2xb4khp6x71qb796-nixos-system-vm11-galera1-20.09pre238361.33548111764.drv'...
building '/nix/store/xrixzd6b7app56hxybss42wxdxa2m1zk-nixops-machines.drv'...
galera1> copying closure...
galera1> copying path '/nix/store/ppip01szl98s4cvqvy13pg2i4x0ba9wv-mariadb-server-10.4.13-man' from 'https://cache.nixos.org'...
galera1> copying path '/nix/store/al4qgq2gv0nl98ln7wdn79hmlxbk0drb-perl5.30.3-Capture-Tiny-0.48' from 'https://cache.nixos.org'...
galera1> copying path '/nix/store/g518g1sja3mr3zya4ngr2bh25kmkvdgm-check-0.15.0' from 'https://cache.nixos.org'...
...
galera1> reloading user units for rebrain...
galera1>           ::::.    ':::::     ::::'           rebrain@vm11-galera1
galera1>           ':::::    ':::::.  ::::'            OS: NixOS 20.09pre238361.33548111764 (Nightingale)
galera1>             :::::     '::::.:::::             Kernel: x86_64 Linux 5.7.15-hardened
galera1>       .......:::::..... ::::::::              Uptime: 58m
galera1>      ::::::::::::::::::. ::::::    ::::.      Packages: 1214
galera1>     ::::::::::::::::::::: :::::.  .::::'      Shell: sh
galera1>            .....           ::::' :::::'       Disk: 3,1G / 16G (21%)
galera1>           :::::            '::' :::::'        CPU: Intel Core (Haswell, no TSX, IBRS) @ 2x 2.195GHz
galera1>  ........:::::               ' :::::::::::.   GPU: Red Hat, Inc. QXL paravirtual graphic card (rev 05)
galera1> :::::::::::::                 :::::::::::::   RAM: 441MiB / 1993MiB
galera1>  ::::::::::: ..              :::::
galera1>      .::::: .:::            :::::
galera1>     .:::::  :::::          '''''    .....
galera1>     :::::   ':::::.  ......:::::::::::::'
galera1>      :::     ::::::. ':::::::::::::::::'
galera1>             .:::::::: '::::::::::
galera1>            .::::''::::.     '::::.
galera1>           .::::'   ::::.     '::::.
galera1>          .::::      ::::      '::::.
galera1> setting up tmpfiles
galera1> reloading the following units: dbus.service, nftables.service
galera1> starting the following units: nscd.service
galera1> the following new units were started: mysql.service
galera1> activation finished successfully
labs-galera> deployment finished successfully
Time: 0h:00m:58s

Первая удалённая система готова, настраиваем остальные по порядку:

nixops deploy -d labs-galera --include galera2
...
galera2> setting up tmpfiles
galera2> reloading the following units: dbus.service, nftables.service
galera2> starting the following units: nscd.service
galera2> the following new units were started: mysql.service
galera2> activation finished successfully
labs-galera> deployment finished successfully
Time: 0h:00m:55s
nixops deploy -d labs-galera --include galera3
galera3> setting up tmpfiles
galera3> reloading the following units: dbus.service, nftables.service
galera3> starting the following units: nscd.service
galera3> the following new units were started: mysql.service
galera3> activation finished successfully
labs-galera> deployment finished successfully
Time: 0h:00m:56s

Проверим состояние кластера MariaDB Galera:

nixops ssh-for-each -d labs-galera --include galera1 -- "mysql -N -e \"show status where variable_name in ('wsrep_cluster_size','wsrep_local_state_comment');\""
galera1> wsrep_local_state_comment      Synced
galera1> wsrep_cluster_size     3

Как мы видим, у нас кластер поднялся и синхронизировался. Теперь на первом хосте прописываем полный адрес кластера gcomm://galera_01,galera_02,galera_03:

nano ~/works/nixops/deploy/labs-galera/galera1/services/mysql.nix
git diff
index 1ea886f..2f613fc 100644
--- a/labs-galera/galera1/services/mysql.nix
+++ b/labs-galera/galera1/services/mysql.nix
@@ -27,7 +27,7 @@
         wsrep_debug = "NONE";
         wsrep_retry_autocommit = "3";
         wsrep_provider = "${pkgs.mariadb-galera}/lib/galera/libgalera_smm.so";
-        wsrep_cluster_address = "gcomm://";
+        wsrep_cluster_address = "gcomm://galera_01,galera_02,galera_03";
         wsrep_cluster_name = "galera";
         wsrep_node_address = "galera_01";
         wsrep_node_name = "galera_01";

Применяем изменение на удаленной машине:

nixops deploy -d labs-galera --include galera1

Добавляем изменения в git и делаем коммит:

git add -A
git commit -m "labs-galera: init mariadb galera cluster"
[master 3106ccd] labs-galera: init mariadb galera cluster
 9 files changed, 168 insertions(+)
 create mode 100644 labs-galera/galera1/services/mysql.nix
 create mode 100644 labs-galera/galera2/services/mysql.nix
 create mode 100644 labs-galera/galera3/services/mysql.nix

Ещё мы можем добавить мониторинг нашего кластера с помощью утилиты netdata. Для этого приводим содержимое файла netdata.nix к такому виду:

nano ~/works/nixops/deploy/labs-galera/galera1/services/netdata.nix
{
  services.netdata = {
    enable = true;
    python = {
      enable = true;
      extraPackages = ps: [
        ps.mysqlclient
      ];
    };
  };

  environment.etc."netdata/python.d/mysql.conf".text = ''
    update_every: 1
    local:
      user    : 'netdata'
      pass    : 'netdata_pass'
      socket  : '/run/mysqld/mysqld.sock'
  '';
}

Где:

  • services.netdata.python.enable - активирует плагины на основе python.
  • services.netdata.python.extraPackages = ps: [ ps.mysqlclient ]; - добавляет утилиту ps.mysqlclient, которая выполняет мониторинг mysql.
  • environment.etc."netdata/python.d/mysql.conf".text - создаём файл /etc/netdata/python.d/mysql.conf с конфигурацией mysql плагина netdata.

Копируем изменённый файл на другие системы:

cp -f ~/works/nixops/deploy/labs-galera/galera1/services/netdata.nix ~/works/nixops/deploy/labs-galera/galera2/services/netdata.nix
cp -f ~/works/nixops/deploy/labs-galera/galera1/services/netdata.nix ~/works/nixops/deploy/labs-galera/galera3/services/netdata.nix

Применим изменения на удаленных системах поочерёдно:

nixops deploy -d labs-galera --include galera1
...
galera1> setting up tmpfiles
galera1> starting the following units: netdata.service
galera1> the following new units were started: logrotate.service
galera1> activation finished successfully
labs-galera> deployment finished successfully
Time: 0h:00m:22s
nixops deploy -d labs-galera --include galera2
...
galera2> setting up tmpfiles
galera2> starting the following units: netdata.service
galera2> the following new units were started: logrotate.service
galera2> activation finished successfully
labs-galera> deployment finished successfully
Time: 0h:00m:21s
nixops deploy -d labs-galera --include galera3
galera3> setting up tmpfiles
galera3> starting the following units: netdata.service
galera3> the following new units were started: logrotate.service
galera3> activation finished successfully
labs-galera> deployment finished successfully
Time: 0h:00m:21s

Добавляем изменения в git и делаем коммит:

git add -A
git commit -m "netdata: add mysql monitoring"
[master 61fa857] netdata: add mysql monitoring
 3 files changed, 42 insertions(+)

В итоге мы поэтапно настроили кластер MariaDB Galera с помощью утилиты nixops.