Knowledge base PIX - Master - Installing PIX Master cluster

Меню

Installing PIX Master cluster

Redis-node
For every redis-node-n:


sudo nano /etc/apt/sources.list

Add repositories:


deb https://dl.astralinux.ru/astra/stable/2.12_x86-64/repository orel/main InRelease
deb https://dl.astralinux.ru/astra/stable/2.12_x86-64/repository orel/contrib InRelease
deb https://dl.astralinux.ru/astra/stable/2.12_x86-64/repository orel/non-free InRelease


sudo apt update
sudo apt install redis redis-sentinel

Check status of redis-server


redis-cli ping

Configure redis


sudo nano /etc/redis/redis.conf

bind 127.0.0.1 {current_redis_node_ip}
protected-mode no
port 6379
tcp-backlog 511
timeout 0
tcp-keepalive 300
daemonize yes
supervised no
pidfile /var/run/redis/redis-server.pid
loglevel notice
logfile /var/log/redis/redis-server.log
databases 16
always-show-logo yes
save 900 1
save 300 10
save 60 10000
stop-writes-on-bgsave-error yes
rdbcompression yes
rdbchecksum yes
dbfilename dump.rdb
dir /var/lib/redis
replica-serve-stale-data yes
replica-read-only yes
repl-diskless-sync no
repl-diskless-sync-delay 5
repl-disable-tcp-nodelay no
replica-priority 100
lazyfree-lazy-eviction no
lazyfree-lazy-expire no
lazyfree-lazy-server-del no
replica-lazy-flush no
appendonly no
appendfilename "appendonly.aof"
appendfsync everysec
no-appendfsync-on-rewrite no
auto-aof-rewrite-percentage 100
auto-aof-rewrite-min-size 64mb
aof-load-truncated yes
aof-use-rdb-preamble yes
lua-time-limit 5000
slowlog-log-slower-than 10000
slowlog-max-len 128
latency-monitor-threshold 0
notify-keyspace-events ""
hash-max-ziplist-entries 512
hash-max-ziplist-value 64
list-max-ziplist-size -2
list-compress-depth 0
set-max-intset-entries 512
zset-max-ziplist-entries 128
zset-max-ziplist-value 64
hll-sparse-max-bytes 3000
stream-node-max-bytes 4096
stream-node-max-entries 100
activerehashing yes
client-output-buffer-limit normal 0 0 0
client-output-buffer-limit replica 256mb 64mb 60
client-output-buffer-limit pubsub 32mb 8mb 60
hz 10
dynamic-hz yes
aof-rewrite-incremental-fsync yes
rdb-save-incremental-fsync yes
maxmemory 1gb

On dependent nodes, add a line:


replicaof {redis_master_node_ip}

Press Сtrl+Х, then Y, then Enter, to confirm changes.

Restart redis


sudo systemctl restart redis

Install redis-sentinel


sudo apt-get install redis-sentinel

Delete the default configuration file and create a new one


sudo rm /etc/redis/sentinel.conf

sudo nano /etc/redis/sentinel.conf


bind 127.0.0.1 {current_node_ip}
port 26379
daemonize yes
pidfile "/var/run/sentinel/redis-sentinel.pid"
logfile "/var/log/redis/redis-sentinel.log"
dir "/var/lib/redis"
sentinel myid 46fe7fe668cf7fb97f5fde3741e2c862f69a5ef7
sentinel deny-scripts-reconfig yes
sentinel monitor mymaster {redis_master_node_ip} 6379 2
sentinel config-epoch mymaster 0
sentinel leader-epoch mymaster 0
protected-mode no
sentinel current-epoch 0
sentinel down-after-milliseconds mymaster 6001
sentinel failover-timeout 10000

monitor - address of master node which we will monitor, 2 - number of Sentinel instances which will make decisions.
down-after-milliseconds - time after which the master will be considered offline.
failover-timeout - timeout after changing worker role to master in case the master fails.
auth-pass password for authorization on the master.
auth-user user we use to log in to the master.

F5 for Redis
Monitor configuration for the REDIS server can be done by configuring two separate TCP monitors with different send strings.
In Properties you need to specify:

Connection created by the Studio activity Create a database connection;
Database Schema name;
Name of the table where the data should be placed;
Table with data to be placed in the database;
Clear table - when true, the original table in the database will be deleted;
Variable name where changed records will be written - the result of saving DataTable.

Note: The send and receive strings must be configured as:
Send string: PING
Receive string: PONG

Configure the monitor using the Configuration Utility:

Go to Local Traffic ›› Monitors ›› and click Create;
Select Type - TCP;
Enter the name "tcp_monitor_ping_redis";
Enter Send String PING\r\n;
Enter the Receive String PONG;
Press Done.

Assign the configured monitor to the pool.
Configure the monitor using the TMSH utility


# tmsh create ltm monitor tcp tcp_monitor_ping_redis { adaptive disabled defaults-from tcp destination *:* interval 5 ip-dscp 0 recv PONG recv-disable none send "PING\r\n" time-until-up 0 timeout 16 }

Creating a secondary monitor

Note: The send and receive strings must be configured as:
Send string: info replication
Receive string: role:master

Configure the monitor using the setup utility:

Go to Local Traffic ›› Monitors ›› Click Create
Select the type as TCP
Enter the name "tcp_monitor_role_redis"
Enter Send Sting as info replication\r\n
Enter Receive String: role:master
Click Finish
Assign the newly configured monitor to the pool.

Configure the monitor using the TMSH utility


#tmsh create ltm monitor tcp tcp_monitor_role_redis { adaptive disabled defaults-from tcp destination *:* interval 5 ip-dscp 0 recv role:master recv-disable none send "info replication\r\n" time-until-up 0 timeout 16 }

As a result, we should get the following configuration:


ltm monitor tcp tcp_monitor_ping_redis {
adaptive disabled
defaults-from tcp
destination *:*
interval 5
ip-dscp 0
recv PONG
recv-disable none
send "PING\r\n"
time-until-up 0
timeout 16
}
ltm monitor tcp tcp_monitor_role_redis {
adaptive disabled
defaults-from tcp
destination *:*
interval 5
ip-dscp 0
recv role:master
recv-disable none
send "info replication\r\n"
time-until-up 0
timeout 16
}

HAproxy for Redis


sudo apt install haproxy
cd /etc/haproxy
nano haproxy.cfg

global
    log /dev/log	local0
    log /dev/log	local1 notice
    chroot /var/lib/haproxy
    stats socket /run/haproxy/admin.sock mode 660 level admin expose-fd listeners
    stats timeout 30s
    user haproxy
    group haproxy
    daemon

    # Default SSL material locations
    ca-base /etc/ssl/certs
    crt-base /etc/ssl/private

    # Default ciphers to use on SSL-enabled listening sockets.
    # For more information, see ciphers(1SSL). This list is from:
    #  https://hynek.me/articles/hardening-your-web-servers-ssl-ciphers/
    # An alternative list with additional directives can be obtained from
    #  https://mozilla.github.io/server-side-tls/ssl-config-generator/?server=haproxy
    ssl-default-bind-ciphers ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:RSA+AESGCM:RSA+AES:!aNULL:!MD5:!DSS
    ssl-default-bind-options no-sslv3

defaults
    defaults REDIS
    mode tcp
    timeout connect 4s 
    timeout server 30s
    timeout client 30s

frontend ft_redis
    bind {redis_haproxy_ip}:6379 name redis
    default_backend bk_redis

backend bk_redis
    option tcp-check
    tcp-check send PING\r\n
    tcp-check expect string +PONG
    tcp-check send info\ replication\r\n
    tcp-check expect string role:master
    tcp-check send QUIT\r\n
    tcp-check expect string +OK
    server redis1 {redis_node_1_ip}:6379 maxconn 4096 check inter 2s
    server redis2 {redis_node _2_ip}:6379 maxconn 4096 check inter 2s
    server redis3 {redis_node _3_ip}:6379 maxconn 4096 check inter 2s

Now we can try making a request to haproxy from any redis host:


redis-cli -h "iphaproxy" info replication

We will see that it reaches the redis master.

Postgres-node
On each PostgreSQL node: PG_node:


sudo apt update
sudo hostnamectl set-hostname nodeN
sudo apt install net-tools
sudo apt install postgresql -y
sudo apt-get install postgresql-server-dev-10
sudo systemctl stop postgresql
sudo ln -s /usr/lib/postgresql/{version}/bin/* /usr/sbin/
sudo apt -y install python python3-pip
sudo apt install python3-testresources
sudo pip3 install --upgrade setuptools 
sudo pip3 install psycopg2
sudo pip3 install patroni
sudo pip3 install python-etcd

sudo nano /etc/patroni.yml


scope: postgres
namespace: /db/
name: postgresql0
restapi:
    listen: {pg_node_ip}:8008
    connect_address: {pg_node_ip}:8008

etcd:
    host: {postgres_haproxy_ip}:2379

bootstrap:
    dcs:
        ttl: 30
        loop_wait: 10
        retry_timeout: 10
        maximum_lag_on_failover: 1048576
        postgresql:
            use_pg_rewind: true
    initdb:
    - encoding: UTF8
    - data-checksums
    pg_hba:
    - host replication replicator 127.0.0.1/32 md5
    - host replication replicator {pg_node_1_ip}/0 md5
    - host replication replicator {pg_node_2_ip}/0 md5
    - host replication replicator {pg_node_3_ip}/0 md5
    - host all all 0.0.0.0/0 md5
    users:
        admin:
            password: admin
            options:
                - createrole
                - createdb
postgresql:
    listen: {pg_node_ip}:5432
    connect_address: {pg_node_ip}:5432
    data_dir: /data/patroni
    pgpass: /tmp/pgpass
    authentication:
        replication:
            username: replicator
            password: rep-pass
        superuser:
            username: postgres
            password: secretpassword
    parameters:
        unix_socket_directories: '.'
tags:
    nofailover: false
    noloadbalance: false
    clonefrom: false
    nosync: false


sudo mkdir /data/patroni -p
sudo chown postgres:postgres /data/patroni
sudo chmod 700 /data/patroni
sudo nano /etc/systemd/system/patroni.service


[Unit]
Description=Runners to orchestrate a high-availability PostgreSQL
After=syslog.target network.target

[Service]
Type=simple

User=postgres
Group=postgres

ExecStart=/usr/local/bin/patroni /etc/patroni.yml

KillMode=process

TimeoutSec=30

Restart=no

[Install]
WantedBy=multi-user.targ


sudo systemctl start patroni
sudo systemctl status patroni

HAproxy for Postgres


sudo apt update
sudo apt install etcd -y
sudo apt install haproxy -y

sudo nano /etc/default/etcd

ETCD_LISTEN_PEER_URLS="http://{postgres_haproxy_ip}:2380"

ETCD_LISTEN_CLIENT_URLS="http://localhost:2379,http://{postgres_haproxy_ip}:2379"

ETCD_INITIAL_ADVERTISE_PEER_URLS="http://{postgres_haproxy_ip}:2380"

ETCD_INITIAL_CLUSTER="etcd0=http://{postgres_haproxy_ip}:2380,"

ETCD_ADVERTISE_CLIENT_URLS="http://{postgres_haproxy_ip}:2379"

ETCD_INITIAL_CLUSTER_TOKEN="cluster1"

ETCD_INITIAL_CLUSTER_STATE="new"

sudo systemctl restart etcd

sudo nano /etc/haproxy/haproxy.cfg

global
    maxconn 100

defaults
    log global
    mode tcp
    retries 2
    timeout client 30m
    timeout connect 4s
    timeout server 30m
    timeout check 5s

listen stats
    mode http
    bind *:7000
    stats enable
    stats uri /

listen postgres
    bind *:5000
    option httpchk
    http-check expect status 200
    default-server inter 3s fall 3 rise 2 on-marked-down shutdown-sessions
    server postgresql_node_1 {postgresql_node_1}:5432 maxconn 100 check port 8008
    server postgresql_node_2 {postgresql_node_2}:5432 maxconn 100 check port 8008
    server postgresql_node_3 {postgresql_node_3}:5432 maxconn 100 check port 8008
sudo systemctl restart haproxy

Installing and configuring PIX Master


cd ~ 
sudo apt-get update 
sudo apt-get install apt-transport-https 
sudo apt-get update 
wget https://packages.microsoft.com/config/debian/10/packages-microsoft-prod.deb -O packages-microsoft-prod.deb 
sudo dpkg -i packages-microsoft-prod.deb 
sudo apt-get update 
sudo apt-get install -y dotnet-sdk-6.0

cd /var/
sudo mkdir PIX_master
sudo chmod 777 PIX_master

Copy the folder with the master to /var/PIX_master.
You can move the folder with the unpacked project using the command:


sudo cp -a <folder> <target folder>

where <folder> is the path to the transferred folder,
<target folder> is the path where the application will be deployed.

In the example below, the path /var/netcore/ will be used as the working folder of the application.


wget http://pix.ru/download/master
mv master master.zip
unzip master.zip 

sudo cp -a /home/user/pix-master/ /var/PIX_master

sudo mkdir /var/PIX_master/Keys
sudo chmod 777 /var/PIX_master/Keys

sudo nano /var/PIX_master/appsettings.json


{
  "Provider": "PostgreSQL",
  "ConnectionStrings": {
    "DefaultConnection": "Server=(localdb)\\MSSQLLocalDB;Database=master_test;Trusted_Connection=True;MultipleActiveResultSets=true;Persist Security Info=False;",
    "PostgreSqlConnection": "User ID=postgres;Password={pg_password};Host={postgres_endpoint_ip};Port={postgres_port};Database=master_test;Pooling=true;"
  },
  "Database": {
    "CommandTimeout": "180"
  },
  "Logging": {
    "LogLevel": {
      "Default": "Warning",
      "Hangfire": "Information"
    }
  },
  "AllowedHosts": "*",
  "PasswordValidationOptions": {
    "RequiredLength": "8",
    "RequireNonAlphanumeric": "true",
    "RequireLowercase": "true",
    "RequireUppercase": "true",
    "RequireDigit": "true"
  },
  "UseSwagger": "false",
  "SecurityConfiguration": {
    "ActiveDirectoryOnly": "false", //default "false"
    "AutoLogon": "true", //default "true"
    "ExpirationTimeMinutes": 1440, //default 1440
    "MaxAgeMinutes": 14400 //Default 14400
  },

  "isCluster": "true",
  "TokenKey": "xecretKeywqejane",

  "LdapProvider": {
    "Enabled": "false",
    "LdapController": "84.201.129.72", //or use domain name "dc.domain.com"
    "LdapPort": "389", //Default non-secure port "389"
    "BindingID": "pix-user-master", //bind login
    "BindingSecret": "Admin1Default@", //bind secure password
    "Domain": "pix.demo" // e.g. domain.com
  },

  "RedisConnection": "{redis_endpoint_ip}:{redis_endpoint_port}" //Redis Connection String
}


sudo nano /etc/systemd/system/MasterCluster.service


[Unit]
Description=ASP .NET Web Application
[Service]
#Set the path to the directory with the installed Wizard
WorkingDirectory=/var/PIX_master
ExecStart=/usr/bin/dotnet /var/PIX_master/Master.dll --urls=http://0.0.0.0:5000
Restart=always
RestartSec=10
SyslogIdentifier=master-cluster
User=www-data
Environment=ASPNETCORE_ENVIRONMENT=Production
[Install]
WantedBy=multi-user.target


sudo systemctl enable MasterCluster.service
sudo systemctl start MasterCluster.service

Nginx for PIX Master


sudo apt-get install nginx
cd /etc/nginx
sudo nano nginx.conf

Make sure that the value is set


gzip off;


sudo nano sites-enabled/default


upstream master {
        ip_hash;
        server {master_node_1_ip}:{master_node_1_port} max_fails=3 fail_timeout=60s;
        server {master_node_2_ip}:{master_node_2_port} max_fails=3 fail_timeout=60s;
}
server {
        listen 443 ssl;
        server_name cluster-test.pixrpa.ru
        ssl on;
        ssl_protocols TLSv1 TLSv1.1 TLSv1.2 TLSv1.3;
        ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES$
        ssl_prefer_server_ciphers on;
        location /{
                proxy_pass http://master;
                proxy_http_version 1.1;
                proxy_set_header Host $host;
                proxy_set_header X-Real-IP $remote_addr;
                proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
                proxy_set_header X-Client-Verify SUCCESS;
                proxy_set_header X-Client-DN $ssl_client_s_dn;
                proxy_set_header X-SSL-Subject $ssl_client_s_dn;
                proxy_set_header X-SSL-Issuer $ssl_client_i_dn;
                proxy_set_header X-Forwarded-Proto http;
                proxy_set_header Upgrade $http_upgrade;
                proxy_set_header Connection "upgrade";
                proxy_read_timeout 1800;
                proxy_connect_timeout 1800;

        }
        location /.well-known/pki-validation/60B42675A03DE2C522168412ED85F5B5.txt {
                alias /var/www/comodo/60B42675A03DE2C522168412ED85F5B5.txt;
        }

    ssl_certificate /etc/letsencrypt/live/cluster-test.pixrpa.ru/fullchain.pem; # managed by Certbot
    ssl_certificate_key /etc/letsencrypt/live/cluster-test.pixrpa.ru/privkey.pem; # managed by Certbot
}


sudo systemctl enable nginx
sudo systemctl start nginx