Browse Source
new: [mysql,rsync-backup] added host install script
new: [mysql,rsync-backup] added host install script
Signed-off-by: Valentin Lab <valentin.lab@kalysto.org>lokavaluto/dev/master
5 changed files with 576 additions and 0 deletions
-
86mysql/hooks/install.d/60-backup.sh
-
82mysql/resources/bin/mysql-backup
-
144rsync-backup-target/resources/bin/compose-add-rsync-key
-
68rsync-backup/hooks/install.d/60-install.sh
-
196rsync-backup/resources/bin/mirror-dir
@ -0,0 +1,86 @@ |
|||
|
|||
set -eux ## important for unbound variable ? |
|||
|
|||
## Require these to be set |
|||
# MYSQL_ROOT_PASSWORD= |
|||
# MYSQL_CONTAINER= |
|||
|
|||
[ "${MYSQL_ROOT_PASSWORD}" ] || { |
|||
echo "Error: you must set \$MYSQL_ROOT_PASSWORD prior to running this script." >&2 |
|||
exit 1 |
|||
} |
|||
|
|||
[ "${MYSQL_CONTAINER}" ] || { |
|||
echo "Error: you must set \$MYSQL_CONTAINER prior to running this script." >&2 |
|||
exit 1 |
|||
} |
|||
|
|||
|
|||
## |
|||
## Init, to setup passwordless connection to mysql |
|||
## |
|||
|
|||
type -p mysql >/dev/null || apt-get install -y mysql-client </dev/null |
|||
|
|||
if ! [ -e "/root/.my.cnf" ]; then |
|||
cat <<EOF > ~/.my.cnf |
|||
[client] |
|||
password=${MYSQL_ROOT_PASSWORD} |
|||
EOF |
|||
chmod 600 ~/.my.cnf |
|||
fi |
|||
|
|||
## |
|||
## installation of the mysql-backup script |
|||
## |
|||
|
|||
|
|||
apt-get install -y kal-shlib-{core,pretty,common} </dev/null |
|||
cp -f resources/bin/mysql-backup /usr/local/sbin/mysql-backup |
|||
|
|||
|
|||
## |
|||
## Connection to cron |
|||
## |
|||
|
|||
|
|||
depends cron |
|||
cat <<EOF > /etc/cron.d/mysql-backup |
|||
SHELL=/bin/bash |
|||
PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin |
|||
|
|||
0 * * * * root /usr/local/sbin/mysql-backup --host \$(docker-ip "$MYSQL_CONTAINER" 2>/dev/null | sed -r 's/ +/ /g' | cut -f 3 -d " ") | logger -t mysql-backup |
|||
|
|||
EOF |
|||
|
|||
|
|||
## |
|||
## Connection with backup |
|||
## |
|||
|
|||
if type -p mirror-dir >/dev/null 2>&1; then |
|||
[ -d "/etc/mirror-dir" ] || { |
|||
echo "'mirror-dir' is installed but no '/etc/mirror-dir' was found." >&2 |
|||
exit 1 |
|||
} |
|||
depends shyaml |
|||
|
|||
if ! sources=$(shyaml get-values default.sources < /etc/mirror-dir/config.yml); then |
|||
echo "Couldn't query 'default.sources' in '/etc/mirror-dir/config.yml'." >&2 |
|||
exit 1 |
|||
fi |
|||
|
|||
if ! echo "$sources" | grep "^/var/backups/mysql$" 2>/dev/null; then |
|||
sed -i '/sources:/a\ - /var/backups/mysql' /etc/mirror-dir/config.yml |
|||
cat <<EOF >> /etc/mirror-dir/config.yml |
|||
/var/backups/mysql: |
|||
exclude: |
|||
- "/*.inprogress" |
|||
EOF |
|||
fi |
|||
else |
|||
echo "warn: 'mirror-dir' not installed, backup won't be sent" >&2 |
|||
fi |
|||
|
|||
|
|||
|
|||
@ -0,0 +1,82 @@ |
|||
#!/bin/bash |
|||
|
|||
. /etc/shlib |
|||
|
|||
include common |
|||
include pretty |
|||
|
|||
usage="$exname [--host HOST] [DATABASE...]" |
|||
|
|||
|
|||
DBS=() |
|||
host= |
|||
while [ "$1" ]; do |
|||
case "$1" in |
|||
"--help"|"-h") |
|||
print_usage |
|||
exit 0 |
|||
;; |
|||
"--host") |
|||
host="$2" |
|||
shift |
|||
;; |
|||
*) |
|||
DBS+=("$1") |
|||
;; |
|||
esac |
|||
shift |
|||
done |
|||
|
|||
|
|||
mysql_opts=() |
|||
if [ "$host" ]; then |
|||
mysql_opts+=(-h "$host") |
|||
fi |
|||
|
|||
m() { |
|||
mysql "${mysql_opts[@]}" -Bs "$@" |
|||
} |
|||
|
|||
md() { |
|||
mysqldump "${mysql_opts[@]}" "$@" |
|||
} |
|||
|
|||
mysql_databases() { |
|||
echo "SHOW DATABASES" | m |
|||
} |
|||
|
|||
mysql_tables() { |
|||
local db="$1" |
|||
echo "SHOW TABLES" | m "$db" |
|||
} |
|||
|
|||
|
|||
if [ "${#DBS[@]}" == 0 ]; then |
|||
DBS=($(mysql_databases)) || exit 1 |
|||
fi |
|||
|
|||
mkdir -p /var/backups/mysql |
|||
|
|||
for db in "${DBS[@]}"; do |
|||
if [[ "$db" == "information_schema" || "$db" == "performance_schema" || "$db" == "mysql" ]]; then |
|||
continue |
|||
fi |
|||
echo "Dumping database $db..." >&2 |
|||
# omitting all the rotation logic |
|||
dst=/"var/backups/mysql/$db" |
|||
[ -d "$dst.old" ] && rm -rf "$dst.old" |
|||
[ -d "$dst" ] && mv "$dst" "$dst.old" |
|||
mkdir -p "$dst.inprogress" |
|||
(( start = SECONDS )) |
|||
md "$db" --routines --no-data --add-drop-database --database "$db" | gzip --rsyncable > "$dst.inprogress/schema.sql.gz" |
|||
tables=$(mysql_tables "$db") |
|||
for table in $tables; do |
|||
backup_file="$dst.inprogress/${table}.sql.gz" |
|||
echo " Dumping $table into ${backup_file}" |
|||
md "$db" "$table" | gzip --rsyncable > "$backup_file" || break |
|||
done |
|||
mv "$dst.inprogress" "$dst" |
|||
[ -d "$dst.old" ] && rm -rf "$dst.old" |
|||
(( elapsed = SECONDS - start )) |
|||
echo " ..dumped $db to $dst ($(du -sh "$dst" | cut -f 1) in ${elapsed}s)" >&2 |
|||
done |
|||
@ -0,0 +1,144 @@ |
|||
#!/bin/bash |
|||
|
|||
|
|||
. /etc/shlib |
|||
|
|||
[[ "${BASH_SOURCE[0]}" != "${0}" ]] && SOURCED=true |
|||
|
|||
include common |
|||
include pretty |
|||
include cmdline |
|||
|
|||
desc='Adds YAML key quite crudely in given compose.yml.' |
|||
help="\ |
|||
|
|||
$WHITE$exname$NORMAL will find a add or replace SSH key to |
|||
given identifier for rsync-backup-target charm. |
|||
|
|||
For now, it only use detection of '## INSERTION-POINT' to manage |
|||
edition of 'compose.yml', and verification it didn't break anything |
|||
before overwriting. |
|||
|
|||
" |
|||
|
|||
|
|||
[ "$SOURCED" ] && return 0 |
|||
|
|||
## |
|||
## Command line processing |
|||
## |
|||
|
|||
|
|||
# remove all lines from regex to regex (not included). |
|||
remove_lines_from_to() { |
|||
local from="$1" to="$2" |
|||
sed -r "/$from/,/$to/{/$to/"'!'"d};/$from/d" |
|||
} |
|||
|
|||
check_valid_yaml() { |
|||
shyaml get-value >/dev/null 2>&1; |
|||
} |
|||
|
|||
|
|||
cmdline.spec.gnu |
|||
cmdline.spec.reporting |
|||
|
|||
service_name=${SERVICE_NAME:-rsync-backup-target} |
|||
compose_file=${COMPOSE_FILE:-/etc/compose/compose.yml} |
|||
|
|||
cmdline.spec::valued:--compose-file,-f:usage() { |
|||
echo "Compose file location. Defaults to '/etc/compose/compose.yml'"; } |
|||
cmdline.spec::valued:--compose-file,-f:run() { compose_file="$1"; } |
|||
|
|||
cmdline.spec::valued:--service-name,-s:usage() { |
|||
echo "YAML service name in compose file to check for existence of key. Defaults to 'rsync-backup-target'"; } |
|||
cmdline.spec::valued:--service-name,-s:run() { service_name="$1"; } |
|||
|
|||
cmdline.spec::cmd:__main__:run() { |
|||
|
|||
: :posarg: DOMAIN 'domain identifier' |
|||
: :posarg: SSH_PUBLIC_KEY 'ssh public key' |
|||
|
|||
if ! existing_domains=$(shyaml keys "${service_name//./\\.}.options.keys" < "$compose_file"); then |
|||
err "Couldn't query file '$compose_file' for keys of" \ |
|||
"service ${DARKYELLOW}${service_name}${NORMAL}." |
|||
exit 1 |
|||
fi |
|||
|
|||
content=$(cat "$compose_file") |
|||
if echo "$existing_domains" | grep "^${DOMAIN}$" >/dev/null 2>&1; then |
|||
|
|||
if ! prev_key=$(shyaml get-value "${service_name//./\\.}.options.keys.${DOMAIN//./\\.}" \ |
|||
< "$compose_file"); then |
|||
err "Couldn't query file '$compose_file' for key of domain '$DOMAIN'." |
|||
exit 1 |
|||
fi |
|||
|
|||
if [ "${prev_key}" == "$SSH_PUBLIC_KEY" ]; then |
|||
echo "Key was already setup." |
|||
exit 0 |
|||
fi |
|||
|
|||
content=$(echo "$content" | remove_lines_from_to '^ '"${DOMAIN//./\\.}"': ".*\\$' \ |
|||
'^ ([A-Za-z0-9.-]+: "|## END MARKER)') |
|||
|
|||
if [ -z "$content" ]; then |
|||
err "Didn't manage to remove key to compose file '$DOMAIN' in '$compose_file'." |
|||
exit 1 |
|||
fi |
|||
|
|||
if [ "$content" == "$(cat "$compose_file")" ]; then |
|||
err "Couldn't remove previous key for '$DOMAIN' in '$compose_file'." |
|||
exit 1 |
|||
fi |
|||
## check we didn't break yaml |
|||
if ! echo "$content" | check_valid_yaml; then |
|||
err "Couldn't safely remove previous key for '$DOMAIN' in '$compose_file'." |
|||
exit 1 |
|||
fi |
|||
fi |
|||
|
|||
excerpt=$(cat <<EOF |
|||
$DOMAIN: "$( |
|||
echo "$SSH_PUBLIC_KEY" | |
|||
fold -w 67 | sed -r 's/$/\\/g;2,$ s/^/ /g;$,$ s/\\$//g')" |
|||
|
|||
EOF |
|||
) |
|||
excerpt="${excerpt//\\/\\\\\\}" |
|||
content="$(echo "$content" | sed -r '/^ ## INSERTION-POINT/a\'"${excerpt}")" |
|||
|
|||
if [ -z "$content" ]; then |
|||
err "Didn't manage to add key to compose file '$DOMAIN' in '$compose_file'." |
|||
exit 1 |
|||
fi |
|||
|
|||
if ! echo "$content" | check_valid_yaml; then |
|||
err "Couldn't safely add key to compose file '$DOMAIN' in '$compose_file'." |
|||
exit 1 |
|||
fi |
|||
|
|||
if diff=$(diff -u "$compose_file" <(echo "$content")); then |
|||
err "No difference, this is not expected, and something went wrong." |
|||
exit 1 |
|||
fi |
|||
echo "${WHITE}Applying these changes:${NORMAL}" |
|||
echo "$diff" |
|||
|
|||
cp "$compose_file" "${compose_file}.old" |
|||
echo "$content" > "$compose_file" |
|||
|
|||
## reloading (could be much faster) |
|||
compose --debug down && compose --debug up |
|||
|
|||
if [ "$?" == 0 ]; then |
|||
echo "Added key, and restarted service ${DARKYELLOW}$service_name${NORMAL}." |
|||
else |
|||
echo "something went wrong ! Should check the state of '$DOMAIN' !!" |
|||
exit 1 |
|||
fi |
|||
} |
|||
|
|||
|
|||
cmdline::parse "$@" |
|||
|
|||
@ -0,0 +1,68 @@ |
|||
#!/bin/bash |
|||
|
|||
|
|||
set -eux |
|||
|
|||
|
|||
[ "${DOMAIN}" ] || { |
|||
echo "Error: you must set \$DOMAIN prior to running this script." >&2 |
|||
exit 1 |
|||
} |
|||
|
|||
[ "${BACKUP_SERVER}" ] || { |
|||
echo "Error: you must set \$BACKUP_SERVER prior to running this script." >&2 |
|||
exit 1 |
|||
} |
|||
|
|||
|
|||
## rsync |
|||
type -p rsync >/dev/null 2>&1 || apt-get install -y rsync </dev/null |
|||
|
|||
## creating rsync user |
|||
mkdir -p /var/lib/rsync |
|||
|
|||
getent group rsync >/dev/null || |
|||
groupadd -r rsync |
|||
|
|||
getent passwd rsync >/dev/null || |
|||
useradd -r rsync -d /var/lib/rsync -g rsync |
|||
|
|||
chown rsync:rsync /var/lib/rsync |
|||
|
|||
## rsync ssh key creation |
|||
[ -e /var/lib/rsync/.ssh/id_rsa ] || |
|||
su -c 'ssh-keygen -t rsa -N "" -f ~/.ssh/id_rsa -q -C rsync@'"$DOMAIN" - rsync |
|||
|
|||
dest="$BACKUP_SERVER" |
|||
if [[ "$dest" == *"/"* ]]; then |
|||
dest="${dest%/*}" |
|||
fi |
|||
|
|||
if [[ "$dest" == *":"* ]]; then |
|||
ssh_options+=("-p" "${dest#*:}") |
|||
dest="${dest%%:*}" |
|||
fi |
|||
|
|||
ssh-keyscan "${ssh_options[@]}" -H "${dest}" > /var/lib/rsync/.ssh/known_hosts |
|||
|
|||
apt-get install kal-shlib-process </dev/null |
|||
ln -sf "$PWD/resources/bin/mirror-dir" /usr/local/sbin/mirror-dir |
|||
|
|||
|
|||
|
|||
if ! [ -e /etc/mirror-dir/config.yml ]; then |
|||
mkdir -p /etc/mirror-dir |
|||
cat <<EOF > /etc/mirror-dir/config.yml |
|||
default: |
|||
sources: |
|||
- /etc |
|||
EOF |
|||
fi |
|||
|
|||
cat <<EOF > /etc/cron.d/mirror-dir |
|||
SHELL=/bin/bash |
|||
PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin |
|||
|
|||
$((RANDOM % 60)) * * * * root mirror-dir -h "$DOMAIN" -d "$BACKUP_SERVER" -u rsync 2>&1 | logger -t mirror-dir |
|||
|
|||
EOF |
|||
@ -0,0 +1,196 @@ |
|||
#!/bin/bash |
|||
|
|||
|
|||
#:- |
|||
. /etc/shlib |
|||
#:- |
|||
|
|||
include common |
|||
include parse |
|||
include process |
|||
|
|||
depends shyaml lock |
|||
|
|||
[ "$UID" != "0" ] && echo "You must be root." && exit 1 |
|||
|
|||
## |
|||
## Here's an example crontab: |
|||
## |
|||
## SHELL=/bin/sh |
|||
## PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin |
|||
## |
|||
## 49 */2 * * * root mirror-dir -d core-05.0k.io:10023 -u rsync /etc /home /opt/apps 2>&1 | logger -t mirror-dir |
|||
## |
|||
|
|||
|
|||
usage="usage: $exname -d DEST1 [-d DEST2 [...]] [-u USER] [DIR1 [DIR2 ...]] |
|||
|
|||
Preserve as much as possible the source structure, keeping hard-links, acl, |
|||
exact numerical uids and gids, and being able to resume in very large files. |
|||
|
|||
Options: |
|||
DIR1 ... DIRn |
|||
Local directories that should be mirrored on destination(s). |
|||
|
|||
examples: /etc /home /var/backups |
|||
|
|||
If no directories are provided, the config file root |
|||
entries will be used all as destination to copy. |
|||
|
|||
-d DESTn |
|||
Can be repeated. Specifies host destination towards which |
|||
files will be send. Note that you can specify port number after |
|||
a colon and a bandwidth limit for rsync after a '/'. |
|||
|
|||
examples: -d liszt.musicalta:10022 -d 10.8.0.19/200 |
|||
|
|||
-u USER (default: 'backuppc') |
|||
|
|||
Local AND destination user to log as at both ends to transfer file. |
|||
This local user need to have a NOPASSWD ssh login towards it's |
|||
account on destination. This destination account should have |
|||
full permissions access without passwd to write with rsync-server |
|||
in the destination directory. |
|||
|
|||
-h STORE (default is taken of the hostname file) |
|||
|
|||
Set the destination store, this is the name of the directory where |
|||
the files will all directories will be copied. Beware ! if 2 hosts |
|||
use the same store, this means they'll conflictingly update the |
|||
same destination directory. Only use this if you know what you |
|||
are doing. |
|||
|
|||
" |
|||
|
|||
dests=() |
|||
source_dirs=() |
|||
hostname= |
|||
while [ "$#" != 0 ]; do |
|||
case "$1" in |
|||
"-d") |
|||
dests+=("$2") |
|||
shift |
|||
;; |
|||
"-h") |
|||
hostname="$2" |
|||
shift |
|||
;; |
|||
"-u") |
|||
user="$2" |
|||
shift |
|||
;; |
|||
*) |
|||
source_dirs+=("$1") |
|||
;; |
|||
esac |
|||
shift |
|||
done |
|||
|
|||
|
|||
if test -z "$hostname"; then |
|||
hostname=$(hostname) |
|||
fi |
|||
|
|||
if test -z "$hostname"; then |
|||
die "Couldn't figure a valid hostname. Please specify one with \`\`-h STORENAME\`\`." |
|||
fi |
|||
|
|||
user=${user:-backuppc} |
|||
dest_path=/var/mirror/$hostname |
|||
|
|||
config_file="/etc/$exname/config.yml" |
|||
|
|||
if [ "${#source_dirs[@]}" == 0 ]; then |
|||
if [ -e "$config_file" ]; then |
|||
echo "No source provided on command line.. " |
|||
echo " ..so reading '$config_file' for default sources..." |
|||
source_dirs=($(eval echo $(shyaml get-values default.sources < "$config_file"))) |
|||
fi |
|||
if [ "${#source_dirs[@]}" == 0 ]; then |
|||
err "You must specify at least one source directory to mirror" \ |
|||
"on command line (or in a config file)." |
|||
print_usage |
|||
exit 1 |
|||
fi |
|||
fi |
|||
echo "Sources directories are: ${source_dirs[@]}" |
|||
|
|||
if [ "${#dests[@]}" == 0 ]; then |
|||
err "You must specify at least a destination." |
|||
print_usage |
|||
exit 1 |
|||
fi |
|||
|
|||
rsync_options=(${RSYNC_OPTIONS:-}) |
|||
ssh_options=(${SSH_OPTIONS:-}) |
|||
|
|||
get_exclude_patterns() { |
|||
local dir="$1" |
|||
[ -e "$config_file" ] || return |
|||
cat "$config_file" | shyaml get-values-0 "$(echo "$dir" | sed -r 's%\.%\\.%g').exclude" |
|||
} |
|||
|
|||
for dest in "${dests[@]}"; do |
|||
for d in "${source_dirs[@]}"; do |
|||
current_rsync_options=("${rsync_options[@]}") |
|||
|
|||
if [[ "$dest" == *"/"* ]]; then |
|||
current_rsync_options+=("--bwlimit" "${dest##*/}") |
|||
dest="${dest%/*}" |
|||
fi |
|||
|
|||
if [[ "$dest" == *":"* ]]; then |
|||
ssh_options+=("-p" "${dest#*:}") |
|||
dest="${dest%%:*}" |
|||
fi |
|||
|
|||
dirpath="$(dirname "$d")" |
|||
if [ "$dirpath" == "/" ]; then |
|||
dir="/$(basename "$d")" |
|||
else |
|||
dir="$dirpath/$(basename "$d")" |
|||
fi |
|||
|
|||
[ -d "$dir" ] || { |
|||
warn "ignoring '$dir' as it is not existing." |
|||
continue |
|||
} |
|||
|
|||
lock_label=$exname-$hostname-$(echo "$dest" | md5_compat | cut -f 1 -d " ") |
|||
|
|||
exclude_patterns="$(get_exclude_patterns "$dir")" |
|||
|
|||
tmp_exclude_patterns=/tmp/${lock_label}.$(echo "$d" | md5_compat | cut -f 1 -d " ").exclude_patterns.tmp |
|||
if [ "$exclude_patterns" ]; then |
|||
echo "Adding exclude patterns..." |
|||
|
|||
## Adding the base of the dir if required... seems necessary with |
|||
## the rsync option that replicate the full path. |
|||
while read-0 exclude_dir; do |
|||
if [[ "$exclude_dir" == "/"* ]]; then |
|||
echo -en "$dir""$(echo "$exclude_dir" | cut -c 1-)\0" |
|||
else |
|||
echo -en "$exclude_dir\0" |
|||
fi |
|||
done < <(get_exclude_patterns "$dir") > "$tmp_exclude_patterns" |
|||
cat "$tmp_exclude_patterns" | xargs -0 -n 1 echo |
|||
current_rsync_options=("-0" "--exclude-from"="$tmp_exclude_patterns" "${current_rsync_options[@]}") |
|||
else |
|||
echo "No exclude patterns for '$dir'." |
|||
fi |
|||
|
|||
echo --------------------------------- |
|||
date |
|||
|
|||
echo nice -n 15 rsync "${current_rsync_options[@]}" -azvARH -e "'sudo -u $user ssh ${ssh_options[*]}'" --delete --delete-excluded --partial --partial-dir .rsync-partial --numeric-ids "$dir/" "$user@$dest":"$dest_path" |
|||
|
|||
lock "$lock_label" -v -D -k -- \ |
|||
nice -n 15 \ |
|||
rsync "${current_rsync_options[@]}" -azvARH \ |
|||
-e "sudo -u $user ssh ${ssh_options[*]}" \ |
|||
--delete --delete-excluded --partial --partial-dir .rsync-partial \ |
|||
--numeric-ids "$dir/" "$user@$dest":"$dest_path" |
|||
|
|||
rm -fv "$tmp_exclude_patterns" |
|||
done |
|||
done |
|||
Write
Preview
Loading…
Cancel
Save
Reference in new issue