From d9b0af54b153c87a9474f9b3517c96bd2d5d50b4 Mon Sep 17 00:00:00 2001 From: Patrick Spek Date: Tue, 5 Mar 2024 11:52:17 +0100 Subject: Add seaweedfs playbooks --- playbooks.d/seaweedfs-filer/description.txt | 1 + playbooks.d/seaweedfs-filer/etc/defaults | 1 + playbooks.d/seaweedfs-filer/playbook.bash | 94 ++++++++++++++++++++++ playbooks.d/seaweedfs-filer/share/filer.conf | 7 ++ playbooks.d/seaweedfs-filer/share/filer.toml | 3 + playbooks.d/seaweedfs-master/description.txt | 1 + playbooks.d/seaweedfs-master/etc/defaults | 1 + playbooks.d/seaweedfs-master/playbook.bash | 95 ++++++++++++++++++++++ playbooks.d/seaweedfs-master/share/master.conf | 8 ++ playbooks.d/seaweedfs-volume/description.txt | 1 + playbooks.d/seaweedfs-volume/etc/defaults | 1 + playbooks.d/seaweedfs-volume/playbook.bash | 104 +++++++++++++++++++++++++ playbooks.d/seaweedfs-volume/share/volume.conf | 11 +++ 13 files changed, 328 insertions(+) create mode 100644 playbooks.d/seaweedfs-filer/description.txt create mode 100644 playbooks.d/seaweedfs-filer/etc/defaults create mode 100644 playbooks.d/seaweedfs-filer/playbook.bash create mode 100644 playbooks.d/seaweedfs-filer/share/filer.conf create mode 100644 playbooks.d/seaweedfs-filer/share/filer.toml create mode 100644 playbooks.d/seaweedfs-master/description.txt create mode 100644 playbooks.d/seaweedfs-master/etc/defaults create mode 100644 playbooks.d/seaweedfs-master/playbook.bash create mode 100644 playbooks.d/seaweedfs-master/share/master.conf create mode 100644 playbooks.d/seaweedfs-volume/description.txt create mode 100644 playbooks.d/seaweedfs-volume/etc/defaults create mode 100644 playbooks.d/seaweedfs-volume/playbook.bash create mode 100644 playbooks.d/seaweedfs-volume/share/volume.conf (limited to 'playbooks.d') diff --git a/playbooks.d/seaweedfs-filer/description.txt b/playbooks.d/seaweedfs-filer/description.txt new file mode 100644 index 0000000..d14afca --- /dev/null +++ b/playbooks.d/seaweedfs-filer/description.txt @@ -0,0 +1 @@ +Scalable object storage cluster diff --git a/playbooks.d/seaweedfs-filer/etc/defaults b/playbooks.d/seaweedfs-filer/etc/defaults new file mode 100644 index 0000000..206987a --- /dev/null +++ b/playbooks.d/seaweedfs-filer/etc/defaults @@ -0,0 +1 @@ +svc.seaweedfs-filer=seaweedfs-filer diff --git a/playbooks.d/seaweedfs-filer/playbook.bash b/playbooks.d/seaweedfs-filer/playbook.bash new file mode 100644 index 0000000..1a71f07 --- /dev/null +++ b/playbooks.d/seaweedfs-filer/playbook.bash @@ -0,0 +1,94 @@ +#!/usr/bin/env bash + +BASHTARD_PLAYBOOK_VARS[$BASHTARD_PLAYBOOK.ip]="required" + +playbook_add() { + local buffer + + # Make sure seaweedfs is installed + "$BASHTARD_BIN" add seaweedfs || true + + mkdir -pv "/var/lib/seaweedfs/filer" + + # Generate systemd unit files + case "${BASHTARD_PLATFORM[init]}" in + systemd) + cat <<-EOF > "$(config "fs.etcdir")/systemd/system/seaweedfs-filer.service" + [Unit] + Description=SeaweedFS Filer + After=network.target + + [Service] + Type=simple + User=root + Group=root + + ExecStart=$(config "fs.bindir")/weed filer -options="$(config "fs.etcdir")/seaweedfs/filer.conf" + WorkingDirectory=/var/lib/seaweedfs/filer + SyslogIdentifier=seaweedfs + + [Install] + WantedBy=multi-user.target + EOF + ;; + *) die "NYI" + esac + + # Perform initial configuration + playbook_sync + + # Start the service + svc enable seaweedfs-filer + svc start seaweedfs-filer +} + +playbook_sync() { + local buffer + local conf + local peers=() + local hash + + buffer="$(tmpfile)" + conf="$(config "fs.etcdir")/seaweedfs/filer.conf" + hash="$(file_hash "$conf")" + + # Add all registered seaweedfs-master nodes as peers + while read -r host + do + peers+=("[$(config_for "$(basename "$host")" "seaweedfs-master.ip")]:$(config_for "$(basename "$host")" "seaweedfs-master.port" "9333")") + done < <(grep -FHl "$BASHTARD_PLAYBOOK" "$BASHTARD_ETCDIR/registry.d"/*) + + # Generate config files + file_template "filer.conf" \ + ip="$(config "$BASHTARD_PLAYBOOK.ip")" \ + port="$(config "$BASHTARD_PLAYBOOK.port" "8888")" \ + masters="$(join_args "${peers[@]}")" \ + s3_port="$(config "$BASHTARD_PLAYBOOK.s3.port" "8333")" \ + > "$buffer" + + file_template "filer.toml" \ + > "$(config "fs.etcdir")/seaweedfs/filer.toml" + + [[ "$(file_hash "$buffer")" == "$hash" ]] && return + + mv -- "$buffer" "$conf" + + [[ "$BASHTARD_ACTION" == "add" ]] && return + + # Reload service + svc restart seaweedfs-filer +} + +playbook_del() { + # Stop service + svc stop seaweedfs-filer + svc disable seaweedfs-filer + + # Remove systemd unit file + rm -fr -- "$(config "fs.etcdir")/systemd/system/seaweedfs-filer.service" + + # Remove configuration files + rm -fr -- \ + "$(config "fs.etcdir")/seaweedfs/filer.conf" \ + "$(config "fs.etcdir")/seaweedfs/filer.toml" +} diff --git a/playbooks.d/seaweedfs-filer/share/filer.conf b/playbooks.d/seaweedfs-filer/share/filer.conf new file mode 100644 index 0000000..e9ac6e8 --- /dev/null +++ b/playbooks.d/seaweedfs-filer/share/filer.conf @@ -0,0 +1,7 @@ +ip=[${ip}] +port=${port} + +master=${masters} + +s3=true +s3.port=${s3_port} diff --git a/playbooks.d/seaweedfs-filer/share/filer.toml b/playbooks.d/seaweedfs-filer/share/filer.toml new file mode 100644 index 0000000..8d40cb9 --- /dev/null +++ b/playbooks.d/seaweedfs-filer/share/filer.toml @@ -0,0 +1,3 @@ +[leveldb3] +enabled = true +dir = "./filerldb3" diff --git a/playbooks.d/seaweedfs-master/description.txt b/playbooks.d/seaweedfs-master/description.txt new file mode 100644 index 0000000..d14afca --- /dev/null +++ b/playbooks.d/seaweedfs-master/description.txt @@ -0,0 +1 @@ +Scalable object storage cluster diff --git a/playbooks.d/seaweedfs-master/etc/defaults b/playbooks.d/seaweedfs-master/etc/defaults new file mode 100644 index 0000000..2578831 --- /dev/null +++ b/playbooks.d/seaweedfs-master/etc/defaults @@ -0,0 +1 @@ +svc.seaweedfs-master=seaweedfs-master diff --git a/playbooks.d/seaweedfs-master/playbook.bash b/playbooks.d/seaweedfs-master/playbook.bash new file mode 100644 index 0000000..34e46c1 --- /dev/null +++ b/playbooks.d/seaweedfs-master/playbook.bash @@ -0,0 +1,95 @@ +#!/usr/bin/env bash + +BASHTARD_PLAYBOOK_VARS[$BASHTARD_PLAYBOOK.ip]="required" + +playbook_add() { + local buffer + + # Make sure seaweedfs is installed + "$BASHTARD_BIN" add seaweedfs || true + + # Create directories used by seaweedfs + mkdir -pv -- "/var/lib/seaweedfs/master" + + # Generate systemd unit files + case "${BASHTARD_PLATFORM[init]}" in + systemd) + cat <<-EOF > "$(config "fs.etcdir")/systemd/system/seaweedfs-master.service" + [Unit] + Description=SeaweedFS Master + After=network.target + + [Service] + Type=simple + User=root + Group=root + + ExecStart=$(config "fs.bindir")/weed master -options="$(config "fs.etcdir")/seaweedfs/master.conf" + WorkingDirectory=/var/lib/seaweedfs + SyslogIdentifier=seaweedfs + + [Install] + WantedBy=multi-user.target + EOF + ;; + *) die "NYI" + esac + + # Perform initial configuration + playbook_sync + + # Start the service + svc enable seaweedfs-master + svc start seaweedfs-master +} + +playbook_sync() { + local buffer + local conf + local peers=() + local hash + + buffer="$(tmpfile)" + conf="$(config "fs.etcdir")/seaweedfs/master.conf" + hash="$(file_hash "$conf")" + + # Add all registered seaweedfs-master nodes as peers + while read -r host + do + # Except this node itself + [[ "$(basename "$host")" == "${BASHTARD_PLATFORM[fqdn]}" ]] && continue + + peers+=("$(config_for "$(basename "$host")" "$BASHTARD_PLAYBOOK.ip")") + done < <(grep -FHl "$BASHTARD_PLAYBOOK" "$BASHTARD_ETCDIR/registry.d"/*) + + # Generate config file + file_template "master.conf" \ + ip="$(config "$BASHTARD_PLAYBOOK.ip")" \ + port="$(config "$BASHTARD_PLAYBOOK.port" "9333")" \ + peers="$(join_args "${peers[@]}")" \ + mdir="$(config "$BASHTARD_PLAYBOOK.mdir" "/var/lib/seaweedfs/master/mdir")" \ + replication="$(config "$BASHTARD_PLAYBOOK.replication" "000")" \ + volume_size="$(config "$BASHTARD_PLAYBOOK.volume-size" "1024")" \ + > "$buffer" + + [[ "$(file_hash "$buffer")" == "$hash" ]] && return + + mv -- "$buffer" "$conf" + + [[ "$BASHTARD_ACTION" == "add" ]] && return + + # Reload service + svc restart seaweedfs-master +} + +playbook_del() { + # Stop service + svc stop seaweedfs-master + svc disable seaweedfs-master + + # Remove systemd unit file + rm -fr -- "$(config "fs.etcdir")/systemd/system/seaweedfs-master.service" + + # Remove configuration files + rm -fr -- "$(config "fs.etcdir")/seaweedfs/master.conf" +} diff --git a/playbooks.d/seaweedfs-master/share/master.conf b/playbooks.d/seaweedfs-master/share/master.conf new file mode 100644 index 0000000..4126635 --- /dev/null +++ b/playbooks.d/seaweedfs-master/share/master.conf @@ -0,0 +1,8 @@ +ip=[${ip}] +port=${port} + +peers=${peers} +mdir=${mdir} + +defaultReplication=${replication} +volumeSizeLimitMB=${volume_size} diff --git a/playbooks.d/seaweedfs-volume/description.txt b/playbooks.d/seaweedfs-volume/description.txt new file mode 100644 index 0000000..d14afca --- /dev/null +++ b/playbooks.d/seaweedfs-volume/description.txt @@ -0,0 +1 @@ +Scalable object storage cluster diff --git a/playbooks.d/seaweedfs-volume/etc/defaults b/playbooks.d/seaweedfs-volume/etc/defaults new file mode 100644 index 0000000..2578831 --- /dev/null +++ b/playbooks.d/seaweedfs-volume/etc/defaults @@ -0,0 +1 @@ +svc.seaweedfs-master=seaweedfs-master diff --git a/playbooks.d/seaweedfs-volume/playbook.bash b/playbooks.d/seaweedfs-volume/playbook.bash new file mode 100644 index 0000000..55e9d52 --- /dev/null +++ b/playbooks.d/seaweedfs-volume/playbook.bash @@ -0,0 +1,104 @@ +#!/usr/bin/env bash + +BASHTARD_PLAYBOOK_VARS[$BASHTARD_PLAYBOOK.dc]="required" +BASHTARD_PLAYBOOK_VARS[$BASHTARD_PLAYBOOK.ip]="required" +BASHTARD_PLAYBOOK_VARS[$BASHTARD_PLAYBOOK.rack]="required" + +playbook_add() { + local buffer + + # Make sure seaweedfs is installed + "$BASHTARD_BIN" add seaweedfs || true + + # Generate systemd unit files + case "${BASHTARD_PLATFORM[init]}" in + systemd) + cat <<-EOF > "$(config "fs.etcdir")/systemd/system/seaweedfs-volume@.service" + [Unit] + Description=SeaweedFS Volume + After=network.target + + [Service] + Type=simple + User=root + Group=root + + ExecStart=$(config "fs.bindir")/weed volume -options="$(config "fs.etcdir")/seaweedfs/volume-%i.conf" + WorkingDirectory=/var/lib/seaweedfs/volume/%i + SyslogIdentifier=seaweedfs + + [Install] + WantedBy=multi-user.target + EOF + ;; + *) die "NYI" + esac + + # Perform initial configuration + playbook_sync + + # Start services + while read -r volume + do + systemctl enable --now "seaweedfs-volume@$volume" + done < <(config_subkeys "$BASHTARD_PLAYBOOK.volumes") +} + +playbook_sync() { + local buffer + local conf + local peers=() + local hash + + buffer="$(tmpfile)" + + # Add all registered seaweedfs-master nodes as peers + while read -r host + do + peers+=("[$(config_for "$(basename "$host")" "seaweedfs-master.ip")]:$(config_for "$(basename "$host")" "seaweedfs-master.port" "9333")") + done < <(grep -FHl "seaweedfs-master" "$BASHTARD_ETCDIR/registry.d"/*) + + while read -r volume + do + conf="$(config "fs.etcdir")/seaweedfs/volume-$volume.conf" + hash="$(file_hash "$conf")" + + info "$BASHTARD_PLAYBOOK/sync/$volume" "Updating $conf" + + # Generate config file + file_template "volume.conf" \ + dc="$(config "$BASHTARD_PLAYBOOK.dc")" \ + dir="/var/lib/seaweedfs/volume/$volume" \ + disk="$(config "$BASHTARD_PLAYBOOK.volume.$volume.disk" "hdd")" \ + filesize_limit="$(config "$BASHTARD_PLAYBOOK.filesize-limit" "256")" \ + free_space="$(config "$BASHTARD_PLAYBOOK.free-space" "10")" \ + ip="$(config "$BASHTARD_PLAYBOOK.ip")" \ + mserver="$(join_args "${peers[@]}")" \ + port="$(config "$BASHTARD_PLAYBOOK.volume.$volume.port" "8080")" \ + rack="$(config "$BASHTARD_PLAYBOOK.rack")" \ + > "$buffer" + + # Write config file + [[ "$(file_hash "$buffer")" == "$hash" ]] && continue + info "$BASHTARD_PLAYBOOK/sync/$volume" "Configuration file changed" + mv -- "$buffer" "$conf" + + # Restart volume server + info "$BASHTARD_PLAYBOOK/sync/$volume" "Restarting volume $volume" + [[ "$BASHTARD_ACTION" == "sync" ]] && systemctl restart "seaweedfs-volume@$volume" + done < <(config_subkeys "$BASHTARD_PLAYBOOK.volumes") +} + +playbook_del() { + # Stop services + while read -r volume + do + systemctl disable --now "seaweedfs-volume@$volume" + done < <(config_subkeys "$BASHTARD_PLAYBOOK.volumes") + + # Remove systemd unit file + rm -fr -- "$(config "fs.etcdir")/systemd/system/seaweedfs-volume@.service" + + # Remove configuration files + rm -fr -- "$(config "fs.etcdir")/seaweedfs/volume.conf" +} diff --git a/playbooks.d/seaweedfs-volume/share/volume.conf b/playbooks.d/seaweedfs-volume/share/volume.conf new file mode 100644 index 0000000..3b0c816 --- /dev/null +++ b/playbooks.d/seaweedfs-volume/share/volume.conf @@ -0,0 +1,11 @@ +ip=[${ip}] +port=${port} +mserver=${mserver} + +dir=${dir} + +dataCenter=${dc} +rack=${rack} +disk=${disk} +fileSizeLimitMB=${filesize_limit} +minFreeSpace=${free_space} -- cgit v1.1