Cloud at home with minimum toil - PXE / Proxmox / Saltstack / k3s

#!/bin/sh -e
#
# rc.local
#
# This script is executed at the end of each multiuser runlevel.
# any non 0 exit will be reported as failure by systemd

# set new interfaces file if any
if [ -f /etc/network/interfaces.install ]; then
    echo "found a network interfaces file to install"
    rm /etc/network/interfaces
    mv /etc/network/interfaces.install /etc/network/interfaces
    systemctl restart networking
fi

# /dev/sda is fully dedicated to pve root
if [ -e /dev/pve/data ]; then
    echo "found /dev/pve/data LVM partition: remove it and extend /dev/pve/root"
    lvresize -l +100%FREE /dev/pve/root
    resize2fs /dev/mapper/pve-root
    sleep 10
fi

# remove enterprise repo and enable the commnunity one's
if [ -f /etc/apt/sources.list.d/pve-enterprise.list ]; then
    echo "Enterprise repository found: disable it and enable commnunity repo"
    rm /etc/apt/sources.list.d/pve-enterprise.list
    rm /etc/apt/sources.list.d/ceph.list
    echo "deb http://download.proxmox.com/debian/pve bookworm pve-no-subscription" >> /etc/apt/sources.list

    echo "Trigger update and full upgrade"
    apt update && apt -y full-upgrade
fi

# remove the subscription NAG uppon login
if [ ! -f /usr/share/javascript/proxmox-widget-toolkit/proxmoxlib.js.bak ]; then
    echo "subscription NAG found, removing it"
    sed -Ezi.bak "s/(Ext.Msg.show\(\{\s+title: gettext\('No valid sub)/void\(\{ \/\/\1/g" /usr/share/javascript/proxmox-widget-toolkit/proxmoxlib.js
    systemctl restart pveproxy.service
fi

# install saltstack Onedir repo
if [ ! -f  /etc/apt/keyrings/salt-archive-keyring.pgp ]; then
    echo "Saltstack keyring not found, install it"
    wget --tries=3 --waitretry=3 --no-dns-cache --retry-on-host-error -O /etc/apt/keyrings/salt-archive-keyring.pgp https://packages.broadcom.com/artifactory/api/security/keypair/SaltProjectKey/public
fi

if [ ! -f /etc/apt/sources.list.d/salt.sources ]; then
    wget --tries=3 --waitretry=3 --no-dns-cache --retry-on-host-error -O /etc/apt/sources.list.d/salt.sources https://github.com/saltstack/salt-install-guide/releases/latest/download/salt.sources;
    apt update && apt -y install salt-minion
    mkdir -p /etc/salt/minion.d
    echo "master: saltmaster.local.lan" >> /etc/salt/minion.d/master.conf
    echo "startup_states: highstate" >> /etc/salt/minion.d/minion.conf
    systemctl enable --now salt-minion
fi

# create zfs volumes on first nvme disk if not exists
if [ $(zpool list | grep zfs-storage | awk '{print $1}') = "zfs-storage" ]; then
    # do nothing
    echo "zfs-storage found"
else
    echo "no zfs-storage pool found, creating it on /dev/nvme0n1"

    # erase partition table
    dd if=/dev/zero of=/dev/nvme0n1 bs=512 count=1

    #create the pool
    zpool create -o autotrim=on -o ashift=12 zfs-storage /dev/nvme0n1

    # disable access time and sync for better performnances
    zfs set atime=off zfs-storage
    zfs set sync=disabled zfs-storage

    # set compression
    zfs set compression=lz4 zfs-storage

    pvesm add zfspool storage -pool zfs-storage
fi

if [ -f /etc/default/grub.d/installer.cfg ]; then
    rm /etc/default/grub.d/installer.cfg
    update-grub
fi

# enable wake on lan on all physical interfaces
nics=$(ip -pretty link show | \grep enp | awk '{print $2}' | cut -d':' -f 1)
if [ -n "$nics" ]; then
    for interface in $nics
    do
        echo "Enable WOL for $interface"
        ethtool -s $interface wol g
    done
fi

##########################################################
#
# Physical interfaces : only used though virtual networks
#
##########################################################
auto lo
iface lo inet loopback

# USB 2.5G nic wired to the other node
auto enx00e04c68030c
iface enx00e04c68030c inet manual

# internal 1GB nic bonded with USB 2.5GB
auto enp2s0
iface enp2s0 inet manual

# 2nd USB NIC bonded with internal NIC
auto enx00e04c680b32
iface enx00e04c680b32 inet manual

##########################################################
#
# link aggregations
#
##########################################################
# main bond to host all vlans
auto bond0
    iface bond0 inet manual
    hwaddress xx:xx:xx:xx:xx:xx
    bond-slaves enp2s0 enx00e04c680b32
    bond-miimon 100
    bond-xmit-hash-policy layer2+3
    bond-mode 802.3ad  # LACP

##########################################################
#
# bridges
#
##########################################################
# bridge to host all vlan + management (untagged vlan 40)
auto vmbr0
iface vmbr0 inet dhcp
    bridge-ports bond0
    bridge-stp off
    bridge-fd 0
    bridge-vlan-aware yes
    bridge-vids 2-4094    # vlan are handled on VM network card
    bridge_ageing 0

# Network for replication on dedicated 2.5G NIC
auto vmbr1
    iface vmbr1 inet static
    address 10.0.0.2/24
    bridge-ports enx00e04c680b32
    bridge-stp off
    bridge-fd 0

#### create cluster if it does not exists
if [ ! -f /etc/pve/corosync.conf ]; then
    pvecm create cluster --link0 address=aaa.bbb.ccc.ddd
    echo "migration: secure,network=10.0.0.0/24" >> /etc/pve/datacenter.cfg
fi

#### add qdevice for corosync quorum
if [ ! -f /sbin/corosync-qdevice ]; then
    apt -y install corosync-qdevice
fi

#### setup storages
if [ -z "$(grep 'dir: vz' /etc/pve/storage.cfg)" ]; then
    # dedicated directory on zfs for iso and lxc templates
    zfs create zfs-storage/vz
    pvesm add dir vz --path /zfs-storage/vz --content 'vztmpl,iso,snippets'

    # no content allowed on system disk
    pvesm set local --content ''

    # NL drives for nas and backup
    mkdir -p /nas-storage /nas-mirror
    echo "LABEL=NAS-STORAGE     /nas-storage    ext4    nofail,nodev,nosuid,relatime,noexec,async           0 2" >> /etc/fstab
    echo "LABEL=NAS-MIRROR      /nas-mirror     ext4    nofail,noauto,nodev,nosuid,relatime,noexec,async    0 2" >> /etc/fstab
    systemctl daemon-reload
    mount /nas-storage
    pvesm add dir backup --path /nas-storage/backup --content 'backup' --nodes 'node-1'
fi

#### install corosync-qnetd package for external vote support on the cluster
if [ ! -f /usr/bin/corosync-qnetd ]; then
    apt -y install corosync-qnetd
fi

if [ -z "$(grep 'dir: vz' /etc/pve/storage.cfg)" ]; then
    # only template and iso allowed on system disk
    pvesm set local --content 'vztmpl,iso'
    # zfs-storage host vm and lxc contents
    pvesm set storage --content 'images,rootdir'
fi

#### create saltstack lxc if not exists
if [ "$(pct list | grep saltmaster | awk '{print $1}')" -eq 100 ]; then
    echo "found saltmaster lxc"
else
    pveam update

    # download the needed template
    TEMPLATE=$(pveam available --section system | grep debian-12 | awk '{print $2}')
    pveam download local $TEMPLATE

    # create the container
    pct create 100 /var/lib/vz/template/cache/$TEMPLATE \
    --cores 2 \
    --memory 2048 \
    --storage storage \
    --description "saltstack master for home cloud" \
    --hostname "saltmaster" \
    --onboot 1 \
    --ostype debian \
    --password "SALTMASTER_ROOT_PWD" \
    --swap 0 \
    --features nesting=1 \
    --net0 name=eth0,bridge=vmbr0,ip=dhcp,tag=xx

    sleep 2
    pct start 100

    # prepare and upgrade OS
    pct exec 100 -- bash -c "apt update && apt -y upgrade && apt -y install git gpg sudo"

    # install saltstack onedir repo
    pct exec 100 -- bash -c '\
    wget --tries=3 --waitretry=3 --no-dns-cache --retry-on-host-error -O /etc/apt/keyrings/salt-archive-keyring-2023.gpg https://repo.saltproject.io/salt/py3/debian/12/amd64/SALT-PROJECT-GPG-PUBKEY-2023.gpg
    echo "deb [signed-by=/etc/apt/keyrings/salt-archive-keyring-2023.gpg arch=amd64] https://repo.saltproject.io/salt/py3/debian/12/amd64/latest bookworm main" | tee /etc/apt/sources.list.d/salt.list'

    # install needed packages
    pct exec 100 -- bash -c "apt update && apt -y install salt-master salt-minion"
    pct exec 100 -- bash -c "salt-pip install pyinotify croniter IPy"

    # clone the saltstack IAC git repo
    pct exec 100 -- bash -c "\
    rm -Rf /srv/*
    git clone https://XXX:SALTMASTER_GIT_PWD/XXX/saltstack /srv"

    # configure saltmaster
    pct exec 100 -- bash -c '\
    mkdir -p /etc/salt/master.d
    echo "cli_summary: True" >> /etc/salt/master.d/saltmaster.conf
    echo "auto_accept: True" >> /etc/salt/master.d/saltmaster.conf
    echo "state_output: changes" >> /etc/salt/master.d/saltmaster.conf
    systemctl enable --now salt-master'

    sleep 5

    # configure saltminion
    pct exec 100 -- bash -c '\
    mkdir -p /etc/salt/minion.d
    echo "master: saltmaster.local.lan" >> /etc/salt/minion.d/minion.conf
    echo "startup_states: highstate" >> /etc/salt/minion.d/minion.conf
    systemctl enable salt-minion'

     # configure gpg
    pct exec 100 -- bash -c 'echo "SALTMASTER_GPG_PUB" > /tmp/pubkey.asc'
    pct exec 100 -- bash -c 'echo "SALTMASTER_GPG_PRIV" > /tmp/privkey.secret.b64'
    pct exec 100 -- bash -c '\
        mkdir -p /etc/salt/gpgkeys
        chmod 0700 /etc/salt/gpgkeys
        echo 'homedir /etc/salt/gpgkeys' >> /root/.gnupg
        base64 -d /tmp/privkey.secret.b64 > /tmp/privkey.secret
        gpg --homedir /etc/salt/gpgkeys --import /tmp/privkey.secret
        gpg --homedir /etc/salt/gpgkeys --import /tmp/pubkey.asc
        echo 'gpg_keydir: /etc/salt/gpgkeys' >/etc/salt/master.d/gpg-pillar.conf
        chown -R salt:salt /etc/gpgkeys
        rm /tmp/privkey.*
        rm /tmp/pubkey.*'

    pct reboot 100

    # reload salt-minion to register correctly
    systemctl restart salt-minion
fi

iac_backend:
# custom scripts to install to /usr/loc/bin
  framework:
    - set_lxc.sh
    - set_vm.sh
    - set_templates.sh
    - set_common.sh
    - set_zfspv.sh

# sysctl settings to set
  sysctl:
    vm.swappiness: 1
    net.ipv6.conf.all.disable_ipv6: 1
    vm.min_free_kbytes: 262144
    vm.dirty_ratio: 20

##############################################################
# Deploy framework to handle IAC on Proxmox:
# - define cluster hosts in /etc/hosts
# - install requiered packages
# - set a new rc.local to remove the one used during first boot
# - install all custom script used to manage operations
# - tune some parameters for better performances
# - deploy a set of udevrules for better power management
##############################################################

proxmox-cluster-hosts:
  file.managed:
    - name: /etc/hosts
    - source: salt://iac_backend/files/hosts
    - user: root
    - group: root
    - mode: 644

proxmox_packages:
  pkg.installed:
    - pkgs:
      - libguestfs-tools

proxmox-rc-local:
  file.managed:
    - name: /etc/rc.local
    - source: salt://iac_backend/files/rc.local
    - user: root
    - group: root
    - mode: 750

{% if salt['pillar.get']('iac_backend:framework', none) is not none %}
proxmox_scripts:
  file.managed:
    - names: {% for file in pillar['iac_backend']['framework'] %}
        - /usr/local/bin/{{ file }}:
            - source: salt://iac_backend/files/{{ file }}
            - mode: 750
             {%- endfor -%}
{% endif %}

{% if salt['pillar.get']('iac_backend:sysctl', none) is not none %}
proxmox_sysctl_settings:
  sysctl.present:
    - names: {% for setting in pillar['iac_backend']['sysctl'] %}
      - {{ setting }}:
        - value: {{ pillar['iac_backend']['sysctl'][setting] }}
             {%- endfor -%}
{% endif %}

proxmox_udev:
  cmd.run:
    - name: udevadm control --reload-rules
    - onchanges:
      - file: proxmox_udev_rules

proxmox_udev_rules:
  file.managed:
    - names:
      - /etc/udev/rules.d/99-powermgmt.rules:
        - source: salt://iac_backend/files/99-powermgmt.rules
        - mode: 640

#!/bin/bash

### COMMON VALUES ###########################
DEBIAN12_CMD_INSTALL_SALTMINION=('\
        wget --tries=3 --waitretry=3 --no-dns-cache --retry-on-host-error -O /etc/apt/keyrings/salt-archive-keyring.pgp https://packages.broadcom.com/artifactory/api/security/keypair/SaltProjectKey/public; \
        wget --tries=3 --waitretry=3 --no-dns-cache --retry-on-host-error -O /etc/apt/sources.list.d/salt.sources https://github.com/saltstack/salt-install-guide/releases/latest/download/salt.sources; \
        apt -qq update > /dev/null'
)

DEBIAN12_CMD_CONFIG_SALTMINION=('\
        mkdir -p /etc/salt/minion.d; \
        echo "master: saltmaster.local.lan" >> /etc/salt/minion.d/minion.conf; \
        systemctl enable salt-minion'
)

### GLOBAL VARIABLES ###########################
CHANGED="no"
COMMENT=""
REPORT=""
SPECIAL_ARGS=""
MANDATORY=""
declare -A arguments
declare -A specials

### FUNCTIONS ##################################

# print a message that can be interpreted by saltstack cmd.script state
# $1=exit code
ExitMessage()
{
    echo
    echo $REPORT
    echo "changed=$CHANGED comment='$COMMENT'"
    exit $1
}

# populate the arguments array waiting for key=value pairs
# $1 are key exceptions that are mapped in $specials array
# $2...$x all argumnts to parse
ParseParameters()
{
    # read parameters and store them in a hashmap
    for arg in "$@"
    do
        # separate argumente name and value (expecting name=value)
        name=${arg%%=*}
        value=${arg#*=}

        # extract special parameter for future usage
        if [[ $SPECIAL_ARGS == *"$name"*  ]]; then
            specials[$name]=$value
        else
            # drop description as it causes problem to pass it to cli commands (quotes)
            if [[ $name != 'description' ]]; then
                arguments[$name]=$value
            fi
        fi
    done
}

# check that mandatory parameters are set
CheckMandatory()
{
    for key in $MANDATORY
    do
        if [[ -z ${arguments[$key]}  ]] && [[ -z ${specials[$key]} ]] ; then
            COMMENT="'$key' parameter is mandatory"
            ExitMessage 2
        fi
    done

}

# parse a config output from proxmox cli command (qm config or pct config)
# Call ConfigValue() function (defined in host script) for each parsed value.
#
# $1 = config string
Parseconfig()
{
    OIFS="$IFS"
    IFS=$'\n'

    for setting in $1
    do
        name=${setting%%:*}
        value=${setting#*': '}

        ConfigValue "$name" "$value"
    done
    IFS="$OIFS"
}

#!/bin/bash

source /usr/local/bin/set_common.sh

### GLOBAL VARIABLES ###########################
CREATE=""
OSVER=""
SPECIAL_ARGS="osver"
MANDATORY="id"

### FUNCTIONS ##################################
Help()
{
   # Display Help
   echo
   echo "Unsure a proxmox container exists as defined."
   echo "Either create it, modify it or do nothing"
   echo
   echo "Syntax: $0 parameter1=value1 [paramter2=value2 [...] ]"
   echo "Parameters and values are passed directly to 'pct'commands."
   echo
   echo There is one special parameter for creation:
   echo "  osver: defines the version of 'ostype'. E.g: 12, base_20240911"
   echo
   echo IMPORTANTS NOTES:
   echo   - description parameter is always dropped
   echo   - rootfs parameter is ignored if container esists
   echo
}

# compare given value and set new config if changed
# $1 = name
# $2 = value
ConfigValue()
{
    # call pct set if anything changed, but for rootfs setting
    if [[ ! -z ${arguments[$1]} ]] && [[ ${arguments[$1]} != $2 ]] && [[ $1 != "rootfs" ]] ; then
        pct set ${arguments['id']} --$1 "${arguments[$1]}"
        CHANGED="yes"
        REPORT="$REPORT $1='${arguments[$1]}'"
    fi

    if [[ $CHANGED == "yes" ]]; then
        COMMENT="Container ${arguments[id]} updated"
    else
        COMMENT="Container ${arguments[id]} already configured"
    fi
}

### SCRIPT LOGIC ################################

# Display help if needed
if [[ ($# -lt 1) || $1 == "help" ]] ; then
    Help
    exit 2
fi

ParseParameters "$@"
CheckMandatory

# check if container exists
if [[ "$(pct list | grep ${arguments['id']} | awk '{print $1}')" -eq ${arguments['id']} ]]; then
    Parseconfig "$(pct config ${arguments['id']})"

# when container does not exists we create it
else
    # to create a container, we need osver (e.g. 12) and ostype (e.g. debian) to find the required template (e.g. debian-12)
    if [[ -z ${specials['osver']} ]] || [[ -z ${arguments['ostype']} ]] ; then
        COMMENT="osver and ostype parameters are mandatory for non existing container"
        ExitMessage 2
    fi

    # build 'pct create' arguments as well as saltstack report for cmd.script state
    for param in ${!arguments[@]}
    do
        if [[ $param != 'id' ]]; then
            CREATE="$CREATE --$param ${arguments[$param]}"
            REPORT="$REPORT $param='${arguments[$param]}'"
        fi
    done

    COMMENT="Container ${arguments['id']} created with"

    # get template filename or download it if not found
    TEMPLATE=$(pveam available --section system | grep "${arguments['ostype']}-${specials['osver']}" | awk '{print $2}')

    TEMPLATE_PATH="/var/lib/vz/template/cache"
    TEMPLATE_STORAGE_NAME="local"
    # define wether to use default local storage of zfs if available
    if [[ -d "/zfs-storage/vz" ]]; then
        TEMPLATE_PATH="/zfs-storage/vz/template/cache"
        TEMPLATE_STORAGE_NAME="vz"
    fi

    if [[ ! -f "$TEMPLATE_PATH/$TEMPLATE" ]]; then
        pveam download $TEMPLATE_STORAGE_NAME $TEMPLATE > /dev/null
        COMMENT="$COMMENT new template: $TEMPLATE"
    else
        COMMENT="$COMMENT existing template"
    fi

    # create container
    pct create ${arguments['id']} $TEMPLATE_PATH/$TEMPLATE $CREATE > /dev/null

    # if container has been created, install salt-minion (depending on OS)
    if [[ $? -eq 0 ]]; then
        CHANGED="yes"

        if [[ "${arguments['ostype']}-${specials['osver']}" == "debian-12" ]]; then

            # set salt-minion requirements
            pct exec ${arguments['id']} -- bash -c "${DEBIAN12_CMD_INSTALL_SALTMINION[@]}"
            # upgrade and install needed packages
            pct exec ${arguments['id']} -- bash -c "apt -yqq upgrade > /dev/null && apt -yqq install salt-minion > /dev/null"
            # configure saltminion
            pct exec ${arguments['id']} -- bash -c "${DEBIAN12_CMD_CONFIG_SALTMINION[@]}"
            # reboot container to make it ready
            pct reboot ${arguments['id']}

            COMMENT="$COMMENT. Salt-minion installed"
        fi
    else
        exit 1
    fi

fi

# print saltstack readble message then exit with success
ExitMessage 0

#!/bin/bash

source /usr/local/bin/set_common.sh

### GLOBAL VARIABLES ###########################
MANDATORY="id name image"

### FUNCTIONS ##################################
Help()
{
   # Display Help
   echo
   echo "Unsure a VM Template is present."
   echo "Either create it or do nothing"
   echo
   echo "Syntax: $0 id=<template id> name=<template name> image=<http link to cloudinit image> [ param=value ]"
   echo "Where param=value can be any "qm set" / "wm create" attributes"
   echo
   echo "NOTE: Created template will have guest-agent and salt-minion installed"
   echo
}

### SCRIPT LOGIC ################################

# Display help if needed
if [[ ($# -lt 3) || $1 == "help" ]] ; then
    Help
    exit 2
fi

ParseParameters "$@"
CheckMandatory

cd /root

# check if template exists
if [[ "$(qm list | grep ${arguments['id']} | awk '{print $1}')" -eq ${arguments['id']} ]]; then
    COMMENT="Image with ID ${arguments['id']} already exists"
    ExitMessage 0
fi

# download image
wget --tries=3 --waitretry=3 --no-dns-cache --retry-on-host-error -nv -c -O "temp.img" ${arguments['image']}
if [[ ! $? -eq 0 ]]; then
    COMMENT="Unable to download image ${arguments['image']}"
    ExitMessage 1
else
    COMMENT="New template created with ID ${arguments['id']} (${arguments['name']})"
fi

# install saltstack on supported images
if [[ ${arguments['image']} == *"debian-12"* ]]; then
    virt-customize -a temp.img --run-command "${DEBIAN12_CMD_INSTALL_SALTMINION[@]}"

    virt-customize -a temp.img --install salt-minion
    if [[ ! $? -eq 0 ]]; then
        COMMENT="Unable to install salt-minion on debian-12 image"
        ExitMessage 1
    fi

    virt-customize -a temp.img --run-command "salt-pip install croniter"
    virt-customize -a temp.img --run-command "${DEBIAN12_CMD_CONFIG_SALTMINION[@]}"

    COMMENT="$COMMENT. Salt-minion installed"
fi

# install guest agent
virt-customize -a temp.img --install qemu-guest-agent
virt-customize -a temp.img --run-command 'systemctl enable qemu-guest-agent'
if [[ ! $? -eq 0 ]]; then
    COMMENT="Unable to install qemu-guest-agent"
    ExitMessage 1
fi

# generate the template
qm create ${arguments['id']} --memory 2048 --core 2 --name ${arguments['name']} --net0 virtio,bridge=vmbr0
qm importdisk ${arguments['id']} temp.img storage > /dev/null
qm set ${arguments['id']}   --scsihw virtio-scsi-pci \
                            --scsi0 storage:vm-${arguments['id']}-disk-0 \
                            --ide2 storage:cloudinit \
                            --boot c --bootdisk scsi0 \
                            --serial0 socket --vga serial0 \
                            --agent enabled=1

qm template ${arguments['id']}
rm /root/temp.img

CHANGED="yes"
ExitMessage 0

#!/bin/bash

source /usr/local/bin/set_common.sh

### GLOBAL VARIABLES ###########################
SPECIAL_ARGS="sshkey id name template start"
MANDATORY="id name template"

### FUNCTIONS ##################################
Help()
{
   # Display Help
   echo
   echo "Unsure a proxmox VM exists as defined."
   echo "Either create it, modify it or do nothing"
   echo
   echo "Syntax: $0 id=<id> name=<name> template=<id> [start=<0 | 1>] parameter1=value1 [paramter2=value2 [...] ]"
   echo "Parameters and values are passed directly to 'qm' commands."
   echo
   echo "IMPORTANTS NOTES:"
   echo "  - description parameter is always dropped"
   echo "  - sshkey is ignored and always defaulted to ~/.ssh/id_rsa.pub"
   echo
}

# compare given value and set new config if changed
# $1 = name
# $2 = value
ConfigValue()
{
    # call pct set if anything changed, but for rootfs setting
    if [[ ! -z ${arguments[$1]} ]] && [[ ${arguments[$1]} != $2 ]] ; then
        # special handling for scsi0: only size is changeable
        if [[ $1 == "scsi0" ]]; then
            disksize=$(echo "${arguments[$1]}" | \grep -oP '(?<=size\=).*(?=G)')
            qm resize ${specials['id']} scsi0 "${disksize}G"
        else
            qm set ${specials['id']} --$1 "${arguments[$1]}"
        fi
        CHANGED="yes"
        REPORT="$REPORT $1='${arguments[$1]}'"
    fi

    if [[ $CHANGED == "yes" ]]; then
        COMMENT="VM ${specials[id]} updated"
    else
        COMMENT="VM ${specials[id]} already configured"
    fi
}

### SCRIPT LOGIC ################################

# Display help if needed
if [[ ($# -lt 3) || $1 == "help" ]] ; then
    Help
    exit 2
fi

ParseParameters "$@"
CheckMandatory

# check if vm exists
if [[ "$(qm list | grep ${specials['id']} | awk '{print $1}')" -eq ${specials['id']} ]]; then
    Parseconfig "$(qm config ${specials['id']})"
else
    # clone template
    qm clone ${specials['template']} ${specials['id']} --name ${specials['name']}
    if [[ ! $? -eq 0 ]]; then
        COMMENT="Error cloning template"
        ExitMessage 1
    fi
    CHANGED="yes"

    # set ssh key
    qm set ${specials['id']} --sshkey ~/.ssh/id_rsa.pub

    # apply all setings from argumemts
    for param in ${!arguments[@]}
    do
        if [[ $param == "scsi0" ]]; then
            disksize=$(echo "${arguments[$param]}" | \grep -oP '(?<=size\=).*(?=G)')
            qm resize ${specials['id']} scsi0 "${disksize}G"
        else
            qm set ${specials['id']} --$param ${arguments[$param]}
        fi
        REPORT="$REPORT $param='${arguments[$param]}'"
    done

    if [[ ! -z ${specials['start']} ]] && [[ ${specials['start']} -eq 1 ]] ; then
        qm start ${specials['id']}
        if [[ $? -eq 0 ]]; then
            COMMENT="$COMMENT. VM started"
        else
            COMMENT="$COMMENT. Error trying to start the VM"
        fi
    fi
fi

# print saltstack readble message then exit with success
ExitMessage 0

templates:
  tpl-debian-12: >
    id=9200
    name=debian-12
    image=https://cloud.debian.org/images/cloud/bookworm/latest/debian-12-generic-amd64.qcow2

map:
  lxc-goldorak:
    password: |
      -----BEGIN PGP MESSAGE-----

      [...]
      -----END PGP MESSAGE-----
    settings: >
      id=300
      hostname=goldorak
      cores=2
      memory=1024
      rootfs=storage:6
      net0=name=eth0,bridge=vmbr0,ip=dhcp,tag=2,type=veth
      ostype=debian
      osver=12
      onboot=1
      swap=0
      features=nesting=1
      start=1

  vm-k3s-master-1:
    password: |
      -----BEGIN PGP MESSAGE-----

      [...]
      -----END PGP MESSAGE-----
    settings: >
      id=1200
      name=k3s-master-1
      template=9200
      cores=4
      scsi0=storage:base-9200-disk-0/vm-1200-disk-0,size=20G
      memory=12288
      net0=model=virtio,bridge=vmbr0,tag=3
      ipconfig0=ip=dhcp
      searchdomain=local.lan
      onboot=1
      start=1

# handle VM templates
{% if pillar.get('templates', none) is not none %}
{% for id in pillar['templates'] %}
{{ id }}:
    cmd.script:
        - name: /usr/local/bin/set_templates.sh
        - stateful: True
        - require:
            - file: proxmox_scripts
            - pkg: proxmox_packages
        - args: {{ pillar['templates'][id] }}
{% endfor %}
{% endif %}

# handle VM and LXC creations and updates
{% if pillar.get('map', none) is not none %}
{% for id in pillar['map'] %}
{% if id.startswith('lxc')      %}
{%   set password='password='   %}
{%   set name='set_lxc.sh'      %}
{% elif id.startswith('vm')     %}
{%   set password='cipassword=' %}
{%   set name='set_vm.sh'       %}
{% endif %}
{% set password=password~pillar['map'][id]['password'] %}
{{ id }}:
    cmd.script:
        - name: /usr/local/bin/{{ name }}
        - stateful: True
        - require:
            - file: proxmox_scripts
            - pkg: proxmox_packages
        - args: >
            {{ password }}
            {{ pillar['map'][id]['settings'] }}
{% endfor %}
{% endif %}

#!yaml|gpg

# if defined AND set to true, this setting is used to trigger
# samba password handling. Else, only system password are set
pdbedit: True

revokedusers:
  - debian

users:
  test:
    fullname: full test
    uid: 1000
    shell: /bin/bash
    ssh-keys:
      - ssh-ed25519 [...] test@my_computer
    groups:
      - adm
      - plugdev
      - sudo
      - staff
      - my_group
    password: |
      -----BEGIN PGP MESSAGE-----

      [...]
      -----END PGP MESSAGE-----

groups:
  my_group:
    gid: 1003

##############################################################
# Handle users on all managed hosts
# - Remove users present in 'revokedusers' pillar (including ssh key)
# - Add groups present in 'groups' pillar
# - Add users present in 'users' pillar (including ssh key)
# - Handle samba passwords if needed ('pdbedit' pillar)
##############################################################

# delete users and groups that have been revoked
# unsure user is no more referenced
{% if pillar.get('revokedusers', none) is not none %}
{% for user in pillar['revokedusers'] %}
{{user}}:
  user.absent: []
  group.absent: []

# unsure user ssh key is no more used by root
{% if salt['pillar.get']('revokedusers:ssh-keys', none) is not none %}
{{user}}_root_key:
  ssh_auth.absent:
    - user: root
    - names:
      {% for key in pillar['revokedusers']['ssh-keys'] %}
      - {{ key }}
      {% endfor %}

# unsure user ssh key is no more used for this user
{{user}}_key:
  ssh_auth.absent:
    - user: {{user}}
    - names:
      {% for key in pillar['revokedusers']['ssh-keys'] %}
      - {{ key }}
      {% endfor %}
{% endif %}
{% endfor %}
{% endif %}

# set groups that must be defined
{% if pillar.get('groups', none) is not none %}
{% for group in pillar['groups'] %}
{{ group }}:
  group.present:
    - gid: {{ pillar['groups'][group]['gid'] }}
{% endfor %}
{% endif %}

# Set users that must be defined
{% if pillar.get('users', none) is not none %}
{% for user in pillar['users'] %}
{{ user }}:
  group.present:
      - gid: {{ pillar['users'][user]['uid'] }}

  user.present:
    - fullname: {{ pillar['users'][user]['fullname'] }}
    - uid: {{ pillar['users'][user]['uid'] }}
    - gid: {{ pillar['users'][user]['uid'] }}
    - shell: {{ pillar['users'][user]['shell'] }}
    - password: {{ pillar['users'][user]['password'] }}
    - hash_password: True
    {% if salt['pillar.get']('users:' + user + ':groups', none) is not none -%}
    - groups:
      {% for group in pillar['users'][user]['groups'] -%}
      - {{ group }}
      {% endfor %}
    {% endif %}

{% if pillar.get('pdbedit', none) is not none %}
  pdbedit.managed:
    - password: {{ pillar['users'][user]['password'] }}
{% endif %}

{% if salt['pillar.get']('users:' + user + ':ssh-keys', none) is not none -%}
{{user}}_root_key:
  ssh_auth.present:
    - user: root
    - names:
      {% for key in pillar['users'][user]['ssh-keys'] -%}
      - {{ key }}
      {% endfor %}

{{user}}_key:
  ssh_auth.present:
    - user: {{user}}
    - names:
      {% for key in pillar['users'][user]['ssh-keys'] -%}
      - {{ key }}
      {% endfor %}
{% endif %}

{% endfor %}
{% endif %}

#!yaml|gpg

zabbix-common:
  server: 'monitor.local.lan'
  repo: ' https://repo.zabbix.com/zabbix/7.0/debian/pool/main/z/zabbix-release/zabbix-release_latest+debian12_all.deb'

zabbix-server:
  db_name: 'zabbix'
  db_user: 'zabbix'
  db_script: '/usr/share/zabbix-sql-scripts/postgresql/server.sql.gz'
  db_password: |
    -----BEGIN PGP MESSAGE-----

    [...]
    -----END PGP MESSAGE-----

##############################################################
# Deploy Zabbix server or agent regarding zabbix-* pillars.
# ALL:
# - install the repo
# - ensure all required services are running
#
# SERVER:
# - Init PostgreSQL database
# - install all needed packages
# - customize configuration files with db, user and password values
# - set nginx to use servername defined in 'server' pillar and to listen to port 80
#
# AGENT:
# - Deploy customized configuration files with server fqdn (or 127.0.0.1 on zabbix server)
#
##############################################################

# zabbix repository
zabbix-repo-install:
  cmd.run:
    - name: "wget -O /tmp/zabbix-release.deb {{ pillar['zabbix-common']['repo'] }} && dpkg -i /tmp/zabbix-release.deb && apt-get update"
    - unless: 'dpkg -l | grep zabbix-release'

# zabbix server
{% if pillar.get('zabbix-server', none) is not none %}

zabbix-set-locale:
  file.line:
    - name: /etc/locale.gen
    - match: 'en_US.UTF-8 UTF-8'
    - mode: replace
    - content: 'en_US.UTF-8 UTF-8'
  cmd.run:
    - name: locale-gen

zabbix-set-default-locale:
  file.line:
    - name: /etc/default/locale
    - match: 'LANG='
    - mode: replace
    - content: 'LANG=en_US.UTF-8'

zabbix-set_pgsql:
  file.managed:
    - name: /usr/local/bin/set_pgsql.sh
    - source: salt://services/files/set_pgsql.sh
    - user: root
    - group: root
    - mode: 750

zabbix-server-install:
  pkg.installed:
    - pkgs:
      - zabbix-server-pgsql
      - zabbix-frontend-php
      - php8.2-pgsql
      - zabbix-nginx-conf
      - zabbix-sql-scripts
      - postgresql-all
    - require:
      - cmd: zabbix-repo-install

zabbix-postgresql:
  service.running:
    - name: postgresql
    - enable: True
    - require:
      - pkg: zabbix-server-install

zabbix-db-user-creation:
  cmd.script:
    - name: /usr/local/bin/set_pgsql.sh
    - stateful: True
    - require:
        - file: zabbix-set_pgsql
        - pkg: zabbix-server-install
    - args: {{ pillar['zabbix-server']['db_name'] }} {{ pillar['zabbix-server']['db_user'] }} {{ pillar['zabbix-server']['db_password'] }} {{ pillar['zabbix-server']['db_script'] }}

/etc/zabbix/zabbix_server.conf:
  file.managed:
    - source: salt://services/files/zabbix_server.conf
    - template: jinja
    - context:
      DBPassword: {{ pillar['zabbix-server']['db_password'] }}
      DBName: {{ pillar['zabbix-server']['db_name'] }}
      DBUser: {{ pillar['zabbix-server']['db_user'] }}
    - require:
        - pkg: zabbix-server-install

/etc/zabbix/web/zabbix.conf.php:
  file.managed:
    - source: salt://services/files/zabbix.conf.php
    - template: jinja
    - context:
      DBPassword: {{ pillar['zabbix-server']['db_password'] }}
      DBName: {{ pillar['zabbix-server']['db_name'] }}
      DBUser: {{ pillar['zabbix-server']['db_user'] }}
    - user: www-data
    - group: www-data
    - mode: 600
    - require:
        - pkg: zabbix-server-install

zabbix-nginx.conf-port:
  file.line:
    - name: /etc/zabbix/nginx.conf
    - match: '8080'
    - mode: replace
    - content: 'listen 80;'
    - require:
        - pkg: zabbix-server-install

zabbix-nginx.conf-host:
  file.line:
    - name: /etc/zabbix/nginx.conf
    - match: 'example.com'
    - mode: replace
    - content: 'server_name {{ pillar['zabbix-common']['server'] }};'
    - require:
        - pkg: zabbix-server-install

zabbix-nginx-default-config:
  file.absent:
    - name: /etc/nginx/sites-enabled/default
    - require:
      - pkg: zabbix-server-install

zabbix-server:
  service.running:
    - enable: True
    - require:
      - pkg: zabbix-server-install

zabbix-server-nginx:
  service.running:
    - name: nginx
    - enable: True
    - require:
      - pkg: zabbix-server-install

zabbix-server-php:
  service.running:
    - name: php8.2-fpm
    - enable: True
    - require:
      - pkg: zabbix-server-install
{% endif %}

# zabbix agent
{% if pillar.get('zabbix-common', none) is not none %}
zabbix-agent-install:
  pkg.installed:
    - pkgs:
      - zabbix-agent2
    - require:
      - cmd: zabbix-repo-install

/etc/zabbix/zabbix_agent2.conf:
  file.managed:
    - source: salt://services/files/zabbix_agent2.conf
    - template: jinja
    - context:
    {% if pillar.get('zabbix-server', none) is none %}
      server: {{ pillar['zabbix-common']['server'] }}
    {% else %}
      server: 127.0.0.1
    {% endif %}
    - require:
        - pkg: zabbix-agent-install

zabbix-agent2:
  service.running:
    - enable: True
    - require:
      - file: /etc/zabbix/zabbix_agent2.conf
    - watch:
      - file: /etc/zabbix/zabbix_agent2.conf
{% endif %}

<?php
// Zabbix GUI configuration file.

$DB['TYPE']                     = 'POSTGRESQL';
$DB['SERVER']                   = 'localhost';
$DB['PORT']                     = '0';
$DB['DATABASE']                 = '{{ DBName }}';
$DB['USER']                     = '{{ DBUser }}';
$DB['PASSWORD']                 = '{{ DBPassword }}';

// Schema name. Used for PostgreSQL.
$DB['SCHEMA']                   = '';

// Used for TLS connection.
$DB['ENCRYPTION']               = true;
$DB['KEY_FILE']                 = '';
$DB['CERT_FILE']                = '';
$DB['CA_FILE']                  = '';
$DB['VERIFY_HOST']              = false;
$DB['CIPHER_LIST']              = '';

// Vault configuration. Used if database credentials are stored in Vault secrets manager.
$DB['VAULT']                    = '';
$DB['VAULT_URL']                = '';
$DB['VAULT_PREFIX']             = '';
$DB['VAULT_DB_PATH']            = '';
$DB['VAULT_TOKEN']              = '';
$DB['VAULT_CERT_FILE']          = '';
$DB['VAULT_KEY_FILE']           = '';
// Uncomment to bypass local caching of credentials.
// $DB['VAULT_CACHE']           = true;

// Uncomment and set to desired values to override Zabbix hostname/IP and port.
// $ZBX_SERVER                  = '';
// $ZBX_SERVER_PORT             = '';

$ZBX_SERVER_NAME                = '{{ grains['host'] }}';

LogFile=/var/log/zabbix/zabbix_server.log
PidFile=/run/zabbix/zabbix_server.pid
SocketDir=/run/zabbix
DBName={{ DBName }}
DBUser={{ DBUser }}
DBPassword={{ DBPassword }}
SNMPTrapperFile=/var/log/snmptrap/snmptrap.log
Timeout=20
FpingLocation=/usr/bin/fping
Fping6Location=/usr/bin/fping6
LogSlowQueries=3000
StatsAllowedIP=127.0.0.1
EnableGlobalScripts=0

Server={{ server }}
ServerActive={{ server }}
Hostname={{ grains['host'] }}
LogFile=/var/log/zabbix/zabbix_agent2.log
PidFile=/run/zabbix/zabbix_agent2.pid
PluginSocket=/run/zabbix/zabbix_agent2.sock
{% if salt.pillar.get('k3s-labels:pv', none) == 'enabled' -%}
DenyKey=vfs.*[*]
{% endif %}

#!/bin/bash

### GLOBAL VARIABLES ###########################
CHANGED="no"
COMMENT=""
DBNAME=""
USER=""
PASSWD=""

### FUNCTIONS ##################################
Help()
{
   # Display Help
   echo
   echo "Unsure a PostgreSQL DB and user exists"
   echo "Either create it, modify it or do nothing"
   echo
   echo "Syntax: $0 <dbname> <user> <password> [sql_script.gz]"
   echo
   echo "IMPORTANTS NOTES:"
   echo "   Mind that any givn password can be shown in process list for a short time and can also be storedin bash history"
   echo
}

# print a message that can be interpreted by saltstack cmd.script state
# $1=exit code
ExitMessage()
{
    echo
    echo "changed=$CHANGED comment='$COMMENT'"
    exit $1
}

### SCRIPT LOGIC ################################

# check if a record exists in given table return 1 if found
# $1: table
# $2: field to use for earching (where)
# $3: record to search
Pg_Record_exists()
{
    table="$1"
    field="$2"
    record="$3"

    sudo -u postgres psql -tc "SELECT 1 FROM $table WHERE $field = '$record'"
}

# Display help if needed
if [[ ($# -lt 3) || $1 == "help" ]] ; then
    Help
    exit 2
fi

DBNAME="$1"
USER="$2"
PASSWD="$3"

if [[ "$(Pg_Record_exists pg_roles rolname $USER | awk '{print $1}')" -eq 1 ]]; then
    COMMENT="User $USER already exists."
else
    sudo -u postgres psql -c "CREATE ROLE $USER LOGIN PASSWORD '$PASSWD';"
    CHANGED="yes"
    COMMENT="User $USER created."
fi

if [[ "$(Pg_Record_exists pg_database datname $DBNAME | awk '{print $1}')" -eq 1 ]]; then
    COMMENT="$COMMENT Database $DBNAME already exists."
else
    sudo -u postgres psql -c "CREATE DATABASE $DBNAME OWNER $USER LOCALE 'en_US.UTF-8' ENCODING UTF8;"
    CHANGED="yes"
    COMMENT="$COMMENT Database $DBNAME created."

    # if an sql script path has been given, try to execute it on the created db
    if [[ $# -eq 4 ]]; then
        zcat "$4" | sudo -u $USER psql -q $DBNAME
        if [[ $? -eq 0 ]]; then
            COMMENT="$COMMENT given SQL script executed"
        else
            COMMENT="$COMMENT failed to execute given SQL script"
        fi
    fi
fi

ExitMessage 0

#!/bin/bash

source /usr/local/bin/set_common.sh

### GLOBAL VARIABLES ###########################
VOLUME_PREFIX="pv_"
declare -A PV

### FUNCTIONS ##################################
Help()
{
   # Display Help
   echo
   echo "Syncronize all zfs volume that begin with '$VOLUME_PREFIX' with all given hosts"
   echo "All hosts are scanned and the source of each volume is determined by checking bytes written."
   echo
   echo "The host on which a volume has the highest number of written bytes is considered as the source."
   echo "All others hosts will be considered as destinations on which data must be updated"
   echo "If not change has been made on the volume, no operation is done"
   echo
   echo "This command depends on pve-zsync, it must be present and no check is done for that"
   echo
   echo "for each found volumes, the output will be:"
   echo "datetime  volume  written  source_host  status".
   echo ""
   echo "examples".
   echo "2024-11-09T19:01:02,506680931+01:00   zfs-storage/pv_myvolume  10.0.0.1=674537 / 10.0.0.1=7656 / => 10.0.0.1    OK".
   echo "2024-11-09T19:01:02,506680931+01:00   zfs-storage/pv_myvolume  10.0.0.1=0 / 10.0.0.2=0   no_change   OK".
   echo
   echo "status can be: OK, ERROR or SKIPPED (if source is not localhost)"
   echo
   echo "Syntax: $0 <host1> <host2> [host3 [...]]"
   echo
}

# $1 = zfs volume
# $2 = prefix
GetZFSSnapshots()
{
    prefix="$2"
    $prefix zfs list -H -t snapshot -o name -s creation | \grep "$1"
}

### SCRIPT LOGIC ################################

# Display help if needed
if [[ ($# -lt 2) || $1 == "help" ]] ; then
    Help
    exit 2
fi

# scan all zfs volume used as pv
for pv_path in $(zfs list -H | \grep pv_ | awk '{print $1}')
do
    echo -n "$(date -Ins)   $pv_path    "
    lastWrittenValue=0
    sourceHost=""

    # get amount of written bytes on all servers and set source host regarding the highest bytes written value
    for current_host in "$@"
    do
        # if not localhost, call command throught ssh
        prefix=""
        if [[ $(hostname --all-ip-addresses || hostname -I) != *"$current_host"* ]]; then
            prefix="ssh $current_host"
        fi

        # if this current scanned host has more written bytes for this volume, consider it as source
        written=$($prefix zfs get -H -p written $pv_path | awk '{print $3}')
        if [[ $written -gt $lastWrittenValue ]]; then
            sourceHost=$current_host
        fi

        echo -n "$current_host=$written / "

        lastWrittenValue=$written
        PV[$current_host]=$written
    done

    # if source host has not been determined, it means that no bytes have been written since the previous sync
    if [[ -z "${sourceHost}" ]]; then
        echo "=> no_change    OK"
        continue
    else
        #echo -n "${PV[$sourceHost]}    $sourceHost"
        echo -n "=> $sourceHost "
    fi

    # sync to all sources if sourceHost is localhost
    if [[ $(hostname --all-ip-addresses || hostname -I) == *"$sourceHost"* ]]; then
        for current_host in "$@"
        do
            if [[ $current_host != $sourceHost ]]; then
                # Synchronise from source to destination
                pve-zsync sync --source $pv_path --dest $current_host:$(dirname $pv_path) --maxsnap 5
                if [[ $? -eq 0 ]]; then
                    echo "  OK"
                else
                    echo "  ERROR"
                fi
            fi
        done
    else
        echo "  SKIPPED"
    fi
done

exit 0

##############################################################
# Define zfs volumes to be used as persistents volumes in k3s
# size is in Gb
# scsiid must be unique for each volume
##############################################################
pvdef:
  mariadb:
    zfsmountpoint: zfs-storage
    size: 4
    fstype: ext4
    scsiid: 1

##############################################################
# Define zfs volumes to be used as persistents volumes in k3s
# size is in Gb
# scsiid must be unique for each volume
##############################################################
pvdef:
  mariadb:
    zfsmountpoint: zfs-storage
    size: 4
    fstype: ext4
    scsiid: 1

  mariadbbackup:
    zfsmountpoint: zfs-storage
    size: 1
    fstype: ext4
    scsiid: 5

##############################################################
# list of hosts ip addresses on which all persistents volumes
# must be synchronized through pve-zsync using zfs_sync.sh
# ipaddr: ip address used for zfs to send / receive sync data
# cronsync: cronexpression at which the host execute zfs_sync.sh
#
# WARNING: be sure that no host launch the sync at the same time !
#
##############################################################
pvhosts:
    hulk-1:
        ipaddr: 10.0.0.1
        cronsync: '0-59/10 * * * *'
    hulk-2:
        ipaddr: 10.0.0.2
        cronsync: '5-59/10 * * * *'

##############################################################
# list of persistants volumes and their Virtual Machine
# attachement (ID)
##############################################################
pvmap:
  mariadb: 1200
  mariadbbackup: 1200

# handle k3s persistent volumes creation
{% if pillar.get('pvmap', none) is not none %}
{% for app in pillar['pvmap'] %}
pv_{{ app }}_{{ grains['host'] }}:
    cmd.script:
        - name: /usr/local/bin/set_zfspv.sh
        - stateful: True
        - args: >
            zfsmountpoint={{ pillar['pvdef'][app]['zfsmountpoint'] }}
            appname={{ app }}
            size={{ pillar['pvdef'][app]['size'] }}
            fstype={{ pillar['pvdef'][app]['fstype'] }}
            vmid={{  pillar['pvmap'][app] }}
            scsiid={{ pillar['pvdef'][app]['scsiid'] }}

# try to restore a backup if persistent volume has just been created
{% if pillar.get('has_nas_storage', false) is true %}
pv_{{ app }}_{{ grains['host'] }}_restore:
    cmd.run:
        - names:
            - /usr/local/bin/nas.sh restore {{ pillar['pvdef'][app]['zfsmountpoint'] }} pv_{{ app }}
            - /usr/local/bin/zfs_sync.sh {% for pvhost in pillar['pvhosts'] %}{{ pvhost }} {% endfor %} >> /var/log/zfs_sync.log
        - onchanges:
            - cmd: pv_{{ app }}_{{ grains['host'] }}
{% endif %}

{% endfor %}
{% endif %}

{% if pillar.get('pvhosts', none) is not none %}
{% set currentlocalhost = grains['localhost'] %}

iac_zfssync_packages:
  pkg.installed:
    - pkgs:
      - pve-zsync

iac_zfssync_cmd:
  file.managed:
    - names:
      - /usr/local/bin/zfs_sync.sh:
        - source: salt://iac_backend/files/zfs_sync.sh
        - mode: 750
      - /etc/logrotate.d/zfs_sync:
        - source: salt://iac_backend/files/zfs_sync.logrotate
        - mode: 644

iac_zfssync_cronjob:
  schedule.present:
    - function: cmd.run
    - job_args:
      - '/usr/local/bin/zfs_sync.sh {% for pvhost in pillar['pvhosts'] %}{{ pillar['pvhosts'][pvhost]['ipaddr'] }} {% endfor %} >> /var/log/zfs_sync.log'
    - maxrunning: 1
    - cron: '{{ pillar['pvhosts'][currentlocalhost]['cronsync'] }}'

{% endif %}

#!/bin/bash

source /usr/local/bin/set_common.sh

### GLOBAL VARIABLES ###########################
VOLUME_PREFIX="pv_"
MANDATORY="zfsmountpoint appname size fstype vmid scsiid"

### FUNCTIONS ##################################
Help()
{
   # Display Help
   echo
   echo "Unsure a ZFS volume for kubernetes PV exists and is provided to guest VM"
   echo "Either create and provide it, or do nothing"
   echo
   echo "Syntax: $0 parameter1=value1 paramter2=value2 ..."
   echo
   echo "where valid parameters are (all are mandatory):"
   echo "  zfsmountpoint:  root zfs mountpoint on which to create the volume"
   echo "  appname:        name of the application in kubernetes (eg. mariadb)"
   echo "  size:           size of the volume (eg. 8 for 8 Gigabytes)"
   echo "  fstype:         filesystem to use to format the partition (eg. ext4)"
   echo "  vmid:           VMID of the VM to which the volume must be added"
   echo "  scsiid:         SCSI disk number to use on guest VM (eg. 1 for scsi1) "
   echo
}

### SCRIPT LOGIC ################################

# Display help if needed
if [[ ($# -lt 6) || $1 == "help" ]] ; then
    Help
    exit 2
fi

ParseParameters "$@"
CheckMandatory

ZFSVolumePath="${arguments['zfsmountpoint']}/${VOLUME_PREFIX}${arguments['appname']}"
ZFSBlockDevice="/dev/zvol/$ZFSVolumePath"
ZFSBlockDevicePart1="${ZFSBlockDevice}-part1"
MkfsCMD="mkfs.${arguments['fstype']}"
ProductName="${VOLUME_PREFIX}${arguments['appname']}"

# create volume, partition and format it if not exists
if [[ ! -b "$ZFSBlockDevice" ]]; then
    # Create ZFS volume
    zfs create -V ${arguments['size']}G $ZFSVolumePath
    if [[ ! $? -eq 0 ]]; then
        CHANGED="no"
        COMMENT="Error creating zfs volume $ZFSVolumePath"
        ExitMessage 1
    fi

    # create one partition the volume
    while [ ! -b "$ZFSBlockDevice" ]; do
        sleep 1
    done
    parted "$ZFSBlockDevice" -- mklabel msdos
    parted -a minimal "$ZFSBlockDevice" -s -- mkpart primary 0 -1
    if [[ ! $? -eq 0 ]]; then
        CHANGED="no"
        COMMENT="Error creating partition on zfs volume ${VOLUME_PREFIX}${arguments['appname']}"
        ExitMessage 1
    fi

    # format the new partition when it appears
    while [ ! -b "$ZFSBlockDevicePart1" ]; do
        sleep 1
    done
    sleep 2 # still need to wait a bit for the part to settle
    $MkfsCMD "$ZFSBlockDevicePart1"
    if [[ ! $? -eq 0 ]]; then
        CHANGED="no"
        COMMENT="Error with $MkfsCMD $ZFSBlockDevicePart1"
        ExitMessage 1
    fi

    CHANGED="yes"
    COMMENT="$ZFSVolumePath created."
else
    CHANGED="no"
    COMMENT="$ZFSVolumePath already exists."
fi

# add volume as disk to the guest VM
if [[ "$(qm config ${arguments['vmid']} | grep ${arguments['appname']})" != *"${arguments['appname']},"* ]]; then
    qm set ${arguments['vmid']} --scsi${arguments['scsiid']} $ZFSBlockDevice,ssd=1,product="$ProductName"
    if [[ ! $? -eq 0 ]]; then
        CHANGED="no"
        COMMENT="Error adding $ZFSBlockDevice as scsi${arguments['scsiid']} on VM ${arguments['vmid']}"
        ExitMessage 1
    fi

    CHANGED="yes"
    COMMENT="$COMMENT Disk scsi${arguments['scsiid']} added to VM ${arguments['vmid']}"
else
    CHANGED="no"
    COMMENT="$COMMENT Guest ${arguments['vmid']} already use it"
fi

# print saltstack readble message then exit with success
ExitMessage 0

#!/bin/bash

### GLOBAL VARIABLES ###########################
LOG_TMP_FILE="/tmp/nas_last_sync.log"
SYNC_SOURCE="/nas-storage"
SYNC_DEST="/nas-mirror"
BACKUP_DIR="$SYNC_SOURCE/backup"
DATE_FORMAT="/bin/date +%d.%m.%Y-%H:%M:%S"
SUBJECT_SYNC="NAS Sync result"
SUBJECT_REPORT="NAS storage report"
SUBJECT_BACKUP="NAS zfs pv_* backup"
SUBJECT_RESTORE="NAS zfs pv_* restore"
MIRROR_DISK_LABEL="NAS-MIRROR"
VOL_TO_RESTORE=""
ZFS_POOL=""
BACKUP_NB=4
NL=$'\n'

### FUNCTIONS ##################################
Help()
{
   # Display Help
   echo
   echo "Perform NAS operations on NL disks"
   echo
   echo "Syntax: $0 <command>"
   echo "where command can be"
   echo "- sync:    launch disks synchdonisation"
   echo "- report:  build usage report and send it"
   echo "- backup:  backup all zfs pv_* volumes to $BACKUP_DIR"
   echo "- restore <zfs pool> <vol>: restore content of vol from $BACKUP_DIR"
   echo "- help:    display this message"
   echo
}

# send a email to sysadmin
# $1 = subject
# $2 = content (string or file)
mailadmin()
{
  if [ -f "$2" ]; then
    mail -s "$1" root < "$2"
  else
    echo "$2" | mail -s "$1" root
  fi
}

# umount, send mail then exit with given code
# $1 = exit code
# $2 = path to umount
# $3 = subject
# $4 = content (string or file)
mailandexit()
{
    if [[ ! -z "$2" ]];then
        umount -q $2
    fi
    mailadmin "$3" "$4"
    exit $1
}

# snychornize nas-storage to nas-mirror if disk is present
Sync()
{
    if [ -e /dev/disk/by-label/$MIRROR_DISK_LABEL ] ; then
        mount -v $SYNC_DEST > "$LOG_TMP_FILE" 2>&1

        echo "---------------------------------------------------------" >> "$LOG_TMP_FILE"
        rsync -a -v --delete-after $SYNC_SOURCE/ $SYNC_DEST/ | awk '{ print "['$($DATE_FORMAT)'] " $0 }' >> "$LOG_TMP_FILE" 2>&1
        echo "---------------------------------------------------------" >> "$LOG_TMP_FILE"

        umount -v $SYNC_DEST >> "$LOG_TMP_FILE" 2>&1

        mailadmin "$SUBJECT_SYNC" "$LOG_TMP_FILE"
    fi
}

# generate a simple usage report of nas storage
Report()
{

    message="$(df -h $SYNC_SOURCE)"
    message="$message$NL --------------------------------------------------------------$NL"
    for dir in $(ls $SYNC_SOURCE)
    do
        message="$message $(du -h -d 1 $SYNC_SOURCE/$dir)$NL$NL"
    done

    mailadmin "$SUBJECT_REPORT" "$message"
}

# backup all zfs pv_* vol
backup_zfspv()
{
    message="ZFS pv_* backup:$NL"
    message="$message --------------------------------------------------------------$NL"

    for pv_path in $(zfs list -H | \grep pv_ | awk '{print $1}')
    do
        mountpath="/dev/zvol/${pv_path}-part1"
        base="$(basename $pv_path)"
        filename="$base-$(date -I).tar.gz"
        message="$message  $filename  "

        # do nothing if backup destination already exists
        if [[ -f "$BACKUP_DIR/$filename" ]]; then
            message="$message INFO: file already exists $NL"
            continue
        fi

        # mount the source to backup
        mkdir -p /media/$pv_path
        mount -o ro $mountpath /media/$pv_path
        if [[ ! $? -eq 0 ]]; then
            message="$message ERROR: could not mount $mountpath $NL"
            continue
        fi

        # if volume is empty, do nothing
        if [[ $(ls /media/$pv_path | wc -l) -lt 2 ]] then
            message="$message INFO: empty source, nothing done $NL"
            umount /media/$pv_path
            continue
        fi

        # do the backup
        cd /media/$pv_path
        tar -czf "$BACKUP_DIR/$filename" *
        if [[ ! $? -eq 0 ]]; then
            message="$message ERROR: creating tar.gz archive on $BACKUP_DIR $NL"
            umount /media/$pv_path
            continue
        fi

        # report size
        message="$message$(du -h "$BACKUP_DIR/$filename" | awk '{print $1}') $NL"
        cd ..
        umount /media/$pv_path
        if [[ $? -eq 0 ]]; then
            rmdir /media/$pv_path
        fi

        # if there are more backup files than BACKUP_NB, delete the oldest
        if [[ $(ls $BACKUP_DIR/$base* | wc -l) -gt $BACKUP_NB ]] then
            file_to_delete=$(ls $BACKUP_DIR/$base* | sort | head -1)
            rm $file_to_delete
        fi
    done

    mailadmin "$SUBJECT_BACKUP" "$message"
}

# restore a given zfs volume
restore_zfspv()
{
    if [[ -z "$VOL_TO_RESTORE" || -z "$ZFS_POOL" ]]; then
        Help
        exit 0
    fi

    message="Restore $VOL_TO_RESTORE:$NL"
    message="$message --------------------------------------------------------------$NL"

    # get lastest backup if any
    file_to_restore=$(ls $BACKUP_DIR/$VOL_TO_RESTORE* 2>/dev/null | sort -r | head -1)
    if [[ ! -f $file_to_restore ]]; then
        message="$message INFO: no backup found in $BACKUP_DIR"
        mailandexit 0 "" "$SUBJECT_RESTORE" "$message"
    fi

    # mount vol to restore
    mountpath="/dev/zvol/${ZFS_POOL}/${VOL_TO_RESTORE}-part1"
    mkdir -p /media/$VOL_TO_RESTORE
    mount $mountpath /media/$VOL_TO_RESTORE
    if [[ ! $? -eq 0 ]]; then
        message="$message ERROR: could not mount $mountpath"
        mailandexit 1 /media/$VOL_TO_RESTORE "$SUBJECT_RESTORE" "$message"
    fi

    # if volume not empty, do nothing
    if [[ $(ls /media/$VOL_TO_RESTORE | wc -l) -gt 2 ]] then
        message="$message WARN: target not empty, nothing done $NL"
        mailandexit 0 /media/$VOL_TO_RESTORE "$SUBJECT_RESTORE" "$message"
    fi

    # restore
    message="$message $(tar -zxvf $file_to_restore -C /media/$VOL_TO_RESTORE/)$NL$NL"
    if [[ ! $? -eq 0 ]]; then
        message="$message ERROR: could not restore $file_to_restore"
        mailandexit 1 /media/$VOL_TO_RESTORE "$SUBJECT_RESTORE" "$message"
    fi

    # umount
    umount /media/$VOL_TO_RESTORE
    if [[ $? -eq 0 ]]; then
        rmdir /media/$VOL_TO_RESTORE
    fi

    message="$message restored $file_to_restore into $mountpath $NL"
    mailadmin "$SUBJECT_RESTORE" "$message"
}

### SCRIPT LOGIC ################################
case "$1" in
    sync)
        Sync
        ;;
    report)
        Report
        ;;
    backup)
        backup_zfspv
        ;;
    restore)
        ZFS_POOL="$2"
        VOL_TO_RESTORE="$3"
        restore_zfspv
        ;;
    *)
        Help
        ;;
esac

#!jinja|yaml|gpg

##############################################################
# reverse proxy configuration with in-memory cache
# secrets are encrypted with GPG
#
# rproxy_conf :
# - ssl_certificate and ssl_certificate_key are GPG encrypted data for let's encrypt. Wildcard certificate is assumed (same for all sites)
#
# rproxy_sites:
# - type: used for specific settings and naxsi rules file name naxsi_<type>.rules
#
##############################################################

rproxy_conf:
  naxsi_package: https://github.com/wargio/naxsi/releases/download/1.6/debian-bookworm-libnginx-mod-http-naxsi_1.6_amd64.deb
  worker_connections: 512   # maximum number of simultaneous connections that can be opened by a worker process
  resolver: aaa.bbb.ccc.ddd  # ip address of dns server to use to resolve backend fqdn
  cache_max_age: 14d        # delete file older than x days
  cache_max_size: 400m      # maximum size the cache can take ()
  dh4096: |
    -----BEGIN PGP MESSAGE-----

    [...]
    -----END PGP MESSAGE-----

  ssl_certificate: |
    -----BEGIN PGP MESSAGE-----

    [...]
    -----END PGP MESSAGE-----

  ssl_certificate_key: |
    -----BEGIN PGP MESSAGE-----

    [...]
    -----END PGP MESSAGE-----

rproxy_certbot:
  rsa-key-size: 4096
  cert-name: all-sites
  email: registered@email.com
  domains: www.mydomain.com,www.myother-domain.org
  deploy-hook: systemctl restart nginx

rproxy_sites:
  mydomain.com:
    type: wordpress
    cache_static_files: enabled
    cache_dynamic_files: enabled
    backends:
      - k3s-master-1.local.lan:8001
      - k3s-master-2.local.lan:8001

  myother-domain.org:
    type: wordpress
    cache_static_files: enabled
    cache_dynamic_files: enabled
    backends:
      - k3s-master-1.local.lan:8002
      - k3s-master-2.local.lan:8002

##############################################################
# Deploy everything that is required to run nginx as a reverseproxy + waf + cache:
# - needed packages
# - external deb package for naxsi extention
# - custom config for nginx service in order to create /run/nginx directory for tmpfs caching
# - install naxsi rules files and cache configurations
# - make custom nginx config file regarding pillar values
# - set let's encrypt required certiitates file based on pillar GPG encrypted values
# - create site configuration file for each site declared in pillar
# - set a cronjob that runs certbot everyday to try to renew the wildcards certificate
##############################################################

# install packages
rproxy_packages:
  pkg.installed:
    - pkgs:
      - nginx
      - certbot
      - python3-certbot-nginx

# install naxsi extention, assume debian host
rproxy_sources_packages:
  cmd.run:
    - name: wget {{ pillar['rproxy_conf']['naxsi_package'] }} -O /root/naxsi.deb && dpkg -i /root/naxsi.deb && rm /root/naxsi.deb
    - unless: test -f /usr/lib/nginx/modules/ngx_http_naxsi_module.so

# tune systemd nginx service config
rproxy_systemd_config:
  file.managed:
    - name: /etc/systemd/system/nginx.service.d/custom-service.conf
    - source: salt://services/files/rproxy/custom-service.conf
    - makedirs: True

# install available naxsi rules
rproxy_naxsi_rules:
  file.recurse:
    - name: /etc/nginx/naxsi-rules
    - source: salt://services/files/rproxy/naxsi-rules

# install available reverse proxy cache configurations
rproxy_cache_conf:
  file.recurse:
    - name: /etc/nginx/reverse-conf
    - source: salt://services/files/rproxy/reverse-conf

# deploy nginx configuration
rproxy_conf:
  file.managed:
    - name: /etc/nginx/nginx.conf
    - source: salt://services/files/rproxy/nginx.conf
    - template: jinja
    - context:
      worker_connections: {{ pillar['rproxy_conf']['worker_connections'] }}
      resolver: {{ pillar['rproxy_conf']['resolver'] }}
      cache_max_age: {{ pillar['rproxy_conf']['cache_max_age'] }}
      cache_max_size: {{ pillar['rproxy_conf']['cache_max_size'] }}

# let's encrypte cert files
rproxy_dh4096:
  file.managed:
    - name: /etc/nginx/dh4096.pem
    - contents: |-
        {{ pillar['rproxy_conf']['dh4096'] | indent(8) }}

rproxy_fullchain:
  file.managed:
    - name: /etc/letsencrypt/live/all-sites/fullchain.pem
    - makedirs: True
    - contents: |-
        {{ pillar['rproxy_conf']['ssl_certificate'] | indent(8) }}

rproxy_privkey:
  file.managed:
    - name: /etc/letsencrypt/live/all-sites/privkey.pem
    - makedirs: True
    - contents: |-
        {{ pillar['rproxy_conf']['ssl_certificate_key'] | indent(8) }}

# generate config files for each declared site
{% for site in pillar['rproxy_sites'] %}
rproxy_site_{{ site }}:
  file.managed:
    - name: /etc/nginx/sites-enabled/{{ site }}.conf
    - source: salt://services/files/rproxy/sites-enabled.conf
    - makedirs: True
    - template: jinja
    - context:
      site: {{ site }}
{% endfor %}

# ensure nginx is running
rproxy_service:
  service.running:
    - name: nginx
    - enable: True
    - restart: True
    - watch:
      - file: /etc/nginx/nginx.conf
      - file: /etc/nginx/sites-enabled/*

# auto renew let's encrypt certificate
rproxy_certbot:
  schedule.present:
    - function: cmd.run
    - job_args:
      - 'certbot --nginx --agree-tos --hsts --staple-ocsp --rsa-key-size {{ pillar['rproxy_certbot']['rsa-key-size'] }} --cert-name {{ pillar['rproxy_certbot']['cert-name'] }} --email {{ pillar['rproxy_certbot']['email'] }} --domains {{ pillar['rproxy_certbot']['domains'] }}'
    - maxrunning: 1
    - cron: '0 5 * * *'