Browse Source

new: [mysql,rsync-backup] added host install script

Signed-off-by: Valentin Lab <valentin.lab@kalysto.org>
0k/dev/master
Valentin Lab 4 years ago
parent
commit
bea7a24e80
  1. 86
      mysql/hooks/install.d/60-backup.sh
  2. 82
      mysql/resources/bin/mysql-backup
  3. 144
      rsync-backup-target/resources/bin/compose-add-rsync-key
  4. 68
      rsync-backup/hooks/install.d/60-install.sh
  5. 196
      rsync-backup/resources/bin/mirror-dir

86
mysql/hooks/install.d/60-backup.sh

@ -0,0 +1,86 @@
set -eux ## important for unbound variable ?
## Require these to be set
# MYSQL_ROOT_PASSWORD=
# MYSQL_CONTAINER=
[ "${MYSQL_ROOT_PASSWORD}" ] || {
echo "Error: you must set \$MYSQL_ROOT_PASSWORD prior to running this script." >&2
exit 1
}
[ "${MYSQL_CONTAINER}" ] || {
echo "Error: you must set \$MYSQL_CONTAINER prior to running this script." >&2
exit 1
}
##
## Init, to setup passwordless connection to mysql
##
type -p mysql >/dev/null || apt-get install -y mysql-client </dev/null
if ! [ -e "/root/.my.cnf" ]; then
cat <<EOF > ~/.my.cnf
[client]
password=${MYSQL_ROOT_PASSWORD}
EOF
chmod 600 ~/.my.cnf
fi
##
## installation of the mysql-backup script
##
apt-get install -y kal-shlib-{core,pretty,common} </dev/null
cp -f resources/bin/mysql-backup /usr/local/sbin/mysql-backup
##
## Connection to cron
##
depends cron
cat <<EOF > /etc/cron.d/mysql-backup
SHELL=/bin/bash
PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
0 * * * * root /usr/local/sbin/mysql-backup --host \$(docker-ip "$MYSQL_CONTAINER" 2>/dev/null | sed -r 's/ +/ /g' | cut -f 3 -d " ") | logger -t mysql-backup
EOF
##
## Connection with backup
##
if type -p mirror-dir >/dev/null 2>&1; then
[ -d "/etc/mirror-dir" ] || {
echo "'mirror-dir' is installed but no '/etc/mirror-dir' was found." >&2
exit 1
}
depends shyaml
if ! sources=$(shyaml get-values default.sources < /etc/mirror-dir/config.yml); then
echo "Couldn't query 'default.sources' in '/etc/mirror-dir/config.yml'." >&2
exit 1
fi
if ! echo "$sources" | grep "^/var/backups/mysql$" 2>/dev/null; then
sed -i '/sources:/a\ - /var/backups/mysql' /etc/mirror-dir/config.yml
cat <<EOF >> /etc/mirror-dir/config.yml
/var/backups/mysql:
exclude:
- "/*.inprogress"
EOF
fi
else
echo "warn: 'mirror-dir' not installed, backup won't be sent" >&2
fi

82
mysql/resources/bin/mysql-backup

@ -0,0 +1,82 @@
#!/bin/bash
. /etc/shlib
include common
include pretty
usage="$exname [--host HOST] [DATABASE...]"
DBS=()
host=
while [ "$1" ]; do
case "$1" in
"--help"|"-h")
print_usage
exit 0
;;
"--host")
host="$2"
shift
;;
*)
DBS+=("$1")
;;
esac
shift
done
mysql_opts=()
if [ "$host" ]; then
mysql_opts+=(-h "$host")
fi
m() {
mysql "${mysql_opts[@]}" -Bs "$@"
}
md() {
mysqldump "${mysql_opts[@]}" "$@"
}
mysql_databases() {
echo "SHOW DATABASES" | m
}
mysql_tables() {
local db="$1"
echo "SHOW TABLES" | m "$db"
}
if [ "${#DBS[@]}" == 0 ]; then
DBS=($(mysql_databases)) || exit 1
fi
mkdir -p /var/backups/mysql
for db in "${DBS[@]}"; do
if [[ "$db" == "information_schema" || "$db" == "performance_schema" || "$db" == "mysql" ]]; then
continue
fi
echo "Dumping database $db..." >&2
# omitting all the rotation logic
dst=/"var/backups/mysql/$db"
[ -d "$dst.old" ] && rm -rf "$dst.old"
[ -d "$dst" ] && mv "$dst" "$dst.old"
mkdir -p "$dst.inprogress"
(( start = SECONDS ))
md "$db" --routines --no-data --add-drop-database --database "$db" | gzip --rsyncable > "$dst.inprogress/schema.sql.gz"
tables=$(mysql_tables "$db")
for table in $tables; do
backup_file="$dst.inprogress/${table}.sql.gz"
echo " Dumping $table into ${backup_file}"
md "$db" "$table" | gzip --rsyncable > "$backup_file" || break
done
mv "$dst.inprogress" "$dst"
[ -d "$dst.old" ] && rm -rf "$dst.old"
(( elapsed = SECONDS - start ))
echo " ..dumped $db to $dst ($(du -sh "$dst" | cut -f 1) in ${elapsed}s)" >&2
done

144
rsync-backup-target/resources/bin/compose-add-rsync-key

@ -0,0 +1,144 @@
#!/bin/bash
. /etc/shlib
[[ "${BASH_SOURCE[0]}" != "${0}" ]] && SOURCED=true
include common
include pretty
include cmdline
desc='Adds YAML key quite crudely in given compose.yml.'
help="\
$WHITE$exname$NORMAL will find a add or replace SSH key to
given identifier for rsync-backup-target charm.
For now, it only use detection of '## INSERTION-POINT' to manage
edition of 'compose.yml', and verification it didn't break anything
before overwriting.
"
[ "$SOURCED" ] && return 0
##
## Command line processing
##
# remove all lines from regex to regex (not included).
remove_lines_from_to() {
local from="$1" to="$2"
sed -r "/$from/,/$to/{/$to/"'!'"d};/$from/d"
}
check_valid_yaml() {
shyaml get-value >/dev/null 2>&1;
}
cmdline.spec.gnu
cmdline.spec.reporting
service_name=${SERVICE_NAME:-rsync-backup-target}
compose_file=${COMPOSE_FILE:-/etc/compose/compose.yml}
cmdline.spec::valued:--compose-file,-f:usage() {
echo "Compose file location. Defaults to '/etc/compose/compose.yml'"; }
cmdline.spec::valued:--compose-file,-f:run() { compose_file="$1"; }
cmdline.spec::valued:--service-name,-s:usage() {
echo "YAML service name in compose file to check for existence of key. Defaults to 'rsync-backup-target'"; }
cmdline.spec::valued:--service-name,-s:run() { service_name="$1"; }
cmdline.spec::cmd:__main__:run() {
: :posarg: DOMAIN 'domain identifier'
: :posarg: SSH_PUBLIC_KEY 'ssh public key'
if ! existing_domains=$(shyaml keys "${service_name//./\\.}.options.keys" < "$compose_file"); then
err "Couldn't query file '$compose_file' for keys of" \
"service ${DARKYELLOW}${service_name}${NORMAL}."
exit 1
fi
content=$(cat "$compose_file")
if echo "$existing_domains" | grep "^${DOMAIN}$" >/dev/null 2>&1; then
if ! prev_key=$(shyaml get-value "${service_name//./\\.}.options.keys.${DOMAIN//./\\.}" \
< "$compose_file"); then
err "Couldn't query file '$compose_file' for key of domain '$DOMAIN'."
exit 1
fi
if [ "${prev_key}" == "$SSH_PUBLIC_KEY" ]; then
echo "Key was already setup."
exit 0
fi
content=$(echo "$content" | remove_lines_from_to '^ '"${DOMAIN//./\\.}"': ".*\\$' \
'^ ([A-Za-z0-9.-]+: "|## END MARKER)')
if [ -z "$content" ]; then
err "Didn't manage to remove key to compose file '$DOMAIN' in '$compose_file'."
exit 1
fi
if [ "$content" == "$(cat "$compose_file")" ]; then
err "Couldn't remove previous key for '$DOMAIN' in '$compose_file'."
exit 1
fi
## check we didn't break yaml
if ! echo "$content" | check_valid_yaml; then
err "Couldn't safely remove previous key for '$DOMAIN' in '$compose_file'."
exit 1
fi
fi
excerpt=$(cat <<EOF
$DOMAIN: "$(
echo "$SSH_PUBLIC_KEY" |
fold -w 67 | sed -r 's/$/\\/g;2,$ s/^/ /g;$,$ s/\\$//g')"
EOF
)
excerpt="${excerpt//\\/\\\\\\}"
content="$(echo "$content" | sed -r '/^ ## INSERTION-POINT/a\'"${excerpt}")"
if [ -z "$content" ]; then
err "Didn't manage to add key to compose file '$DOMAIN' in '$compose_file'."
exit 1
fi
if ! echo "$content" | check_valid_yaml; then
err "Couldn't safely add key to compose file '$DOMAIN' in '$compose_file'."
exit 1
fi
if diff=$(diff -u "$compose_file" <(echo "$content")); then
err "No difference, this is not expected, and something went wrong."
exit 1
fi
echo "${WHITE}Applying these changes:${NORMAL}"
echo "$diff"
cp "$compose_file" "${compose_file}.old"
echo "$content" > "$compose_file"
## reloading (could be much faster)
compose --debug down && compose --debug up
if [ "$?" == 0 ]; then
echo "Added key, and restarted service ${DARKYELLOW}$service_name${NORMAL}."
else
echo "something went wrong ! Should check the state of '$DOMAIN' !!"
exit 1
fi
}
cmdline::parse "$@"

68
rsync-backup/hooks/install.d/60-install.sh

@ -0,0 +1,68 @@
#!/bin/bash
set -eux
[ "${DOMAIN}" ] || {
echo "Error: you must set \$DOMAIN prior to running this script." >&2
exit 1
}
[ "${BACKUP_SERVER}" ] || {
echo "Error: you must set \$BACKUP_SERVER prior to running this script." >&2
exit 1
}
## rsync
type -p rsync >/dev/null 2>&1 || apt-get install -y rsync </dev/null
## creating rsync user
mkdir -p /var/lib/rsync
getent group rsync >/dev/null ||
groupadd -r rsync
getent passwd rsync >/dev/null ||
useradd -r rsync -d /var/lib/rsync -g rsync
chown rsync:rsync /var/lib/rsync
## rsync ssh key creation
[ -e /var/lib/rsync/.ssh/id_rsa ] ||
su -c 'ssh-keygen -t rsa -N "" -f ~/.ssh/id_rsa -q -C rsync@'"$DOMAIN" - rsync
dest="$BACKUP_SERVER"
if [[ "$dest" == *"/"* ]]; then
dest="${dest%/*}"
fi
if [[ "$dest" == *":"* ]]; then
ssh_options+=("-p" "${dest#*:}")
dest="${dest%%:*}"
fi
ssh-keyscan "${ssh_options[@]}" -H "${dest}" > /var/lib/rsync/.ssh/known_hosts
apt-get install kal-shlib-process </dev/null
ln -sf "$PWD/resources/bin/mirror-dir" /usr/local/sbin/mirror-dir
if ! [ -e /etc/mirror-dir/config.yml ]; then
mkdir -p /etc/mirror-dir
cat <<EOF > /etc/mirror-dir/config.yml
default:
sources:
- /etc
EOF
fi
cat <<EOF > /etc/cron.d/mirror-dir
SHELL=/bin/bash
PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
$((RANDOM % 60)) * * * * root mirror-dir -h "$DOMAIN" -d "$BACKUP_SERVER" -u rsync 2>&1 | logger -t mirror-dir
EOF

196
rsync-backup/resources/bin/mirror-dir

@ -0,0 +1,196 @@
#!/bin/bash
#:-
. /etc/shlib
#:-
include common
include parse
include process
depends shyaml lock
[ "$UID" != "0" ] && echo "You must be root." && exit 1
##
## Here's an example crontab:
##
## SHELL=/bin/sh
## PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin
##
## 49 */2 * * * root mirror-dir -d core-05.0k.io:10023 -u rsync /etc /home /opt/apps 2>&1 | logger -t mirror-dir
##
usage="usage: $exname -d DEST1 [-d DEST2 [...]] [-u USER] [DIR1 [DIR2 ...]]
Preserve as much as possible the source structure, keeping hard-links, acl,
exact numerical uids and gids, and being able to resume in very large files.
Options:
DIR1 ... DIRn
Local directories that should be mirrored on destination(s).
examples: /etc /home /var/backups
If no directories are provided, the config file root
entries will be used all as destination to copy.
-d DESTn
Can be repeated. Specifies host destination towards which
files will be send. Note that you can specify port number after
a colon and a bandwidth limit for rsync after a '/'.
examples: -d liszt.musicalta:10022 -d 10.8.0.19/200
-u USER (default: 'backuppc')
Local AND destination user to log as at both ends to transfer file.
This local user need to have a NOPASSWD ssh login towards it's
account on destination. This destination account should have
full permissions access without passwd to write with rsync-server
in the destination directory.
-h STORE (default is taken of the hostname file)
Set the destination store, this is the name of the directory where
the files will all directories will be copied. Beware ! if 2 hosts
use the same store, this means they'll conflictingly update the
same destination directory. Only use this if you know what you
are doing.
"
dests=()
source_dirs=()
hostname=
while [ "$#" != 0 ]; do
case "$1" in
"-d")
dests+=("$2")
shift
;;
"-h")
hostname="$2"
shift
;;
"-u")
user="$2"
shift
;;
*)
source_dirs+=("$1")
;;
esac
shift
done
if test -z "$hostname"; then
hostname=$(hostname)
fi
if test -z "$hostname"; then
die "Couldn't figure a valid hostname. Please specify one with \`\`-h STORENAME\`\`."
fi
user=${user:-backuppc}
dest_path=/var/mirror/$hostname
config_file="/etc/$exname/config.yml"
if [ "${#source_dirs[@]}" == 0 ]; then
if [ -e "$config_file" ]; then
echo "No source provided on command line.. "
echo " ..so reading '$config_file' for default sources..."
source_dirs=($(eval echo $(shyaml get-values default.sources < "$config_file")))
fi
if [ "${#source_dirs[@]}" == 0 ]; then
err "You must specify at least one source directory to mirror" \
"on command line (or in a config file)."
print_usage
exit 1
fi
fi
echo "Sources directories are: ${source_dirs[@]}"
if [ "${#dests[@]}" == 0 ]; then
err "You must specify at least a destination."
print_usage
exit 1
fi
rsync_options=(${RSYNC_OPTIONS:-})
ssh_options=(${SSH_OPTIONS:-})
get_exclude_patterns() {
local dir="$1"
[ -e "$config_file" ] || return
cat "$config_file" | shyaml get-values-0 "$(echo "$dir" | sed -r 's%\.%\\.%g').exclude"
}
for dest in "${dests[@]}"; do
for d in "${source_dirs[@]}"; do
current_rsync_options=("${rsync_options[@]}")
if [[ "$dest" == *"/"* ]]; then
current_rsync_options+=("--bwlimit" "${dest##*/}")
dest="${dest%/*}"
fi
if [[ "$dest" == *":"* ]]; then
ssh_options+=("-p" "${dest#*:}")
dest="${dest%%:*}"
fi
dirpath="$(dirname "$d")"
if [ "$dirpath" == "/" ]; then
dir="/$(basename "$d")"
else
dir="$dirpath/$(basename "$d")"
fi
[ -d "$dir" ] || {
warn "ignoring '$dir' as it is not existing."
continue
}
lock_label=$exname-$hostname-$(echo "$dest" | md5_compat | cut -f 1 -d " ")
exclude_patterns="$(get_exclude_patterns "$dir")"
tmp_exclude_patterns=/tmp/${lock_label}.$(echo "$d" | md5_compat | cut -f 1 -d " ").exclude_patterns.tmp
if [ "$exclude_patterns" ]; then
echo "Adding exclude patterns..."
## Adding the base of the dir if required... seems necessary with
## the rsync option that replicate the full path.
while read-0 exclude_dir; do
if [[ "$exclude_dir" == "/"* ]]; then
echo -en "$dir""$(echo "$exclude_dir" | cut -c 1-)\0"
else
echo -en "$exclude_dir\0"
fi
done < <(get_exclude_patterns "$dir") > "$tmp_exclude_patterns"
cat "$tmp_exclude_patterns" | xargs -0 -n 1 echo
current_rsync_options=("-0" "--exclude-from"="$tmp_exclude_patterns" "${current_rsync_options[@]}")
else
echo "No exclude patterns for '$dir'."
fi
echo ---------------------------------
date
echo nice -n 15 rsync "${current_rsync_options[@]}" -azvARH -e "'sudo -u $user ssh ${ssh_options[*]}'" --delete --delete-excluded --partial --partial-dir .rsync-partial --numeric-ids "$dir/" "$user@$dest":"$dest_path"
lock "$lock_label" -v -D -k -- \
nice -n 15 \
rsync "${current_rsync_options[@]}" -azvARH \
-e "sudo -u $user ssh ${ssh_options[*]}" \
--delete --delete-excluded --partial --partial-dir .rsync-partial \
--numeric-ids "$dir/" "$user@$dest":"$dest_path"
rm -fv "$tmp_exclude_patterns"
done
done
Loading…
Cancel
Save