From 9ae15014b1e80f0fe9363119a0b16f2754980773 Mon Sep 17 00:00:00 2001 From: Javier Sánchez Parra Date: Fri, 3 Sep 2021 10:24:52 +0200 Subject: #1062 Support tiptorrent image transfer mode Add support for tiptorrent image transfer mode to client's bash scripts (aka cloning engine). If desired, clients receive images to restore with the new unicast transfer mode tiptorrent. See also: http://git.soleta.eu/tiptorrent/ #1062 Add tiptorrent support to WebConsole Add tiptorrent option to WebConsole restore image form. #1062 Launch tiptorrent on oglive init This commit adds tiptorrent launch to oglive init process. Clients launch tiptorrent daemon if they have cache partition. #1062 Fix minor tiptorrent integration bugs * Fix md5sum pipe typo * Use correct variable for checksum download #1062 Calculate checksum on image creation Otherwise, OpenGnsys depends on cron and bittorrent specific scripts to obtain images checksums. Support cache cleaning for tiptorrent image parts Tiptorrent downloads and saves images to the cache in parts. With this commit, WebConsole shows and deletes all parts as one. #1062 Rename TIPTORRENT-CACHE in restore image form This commit changes TIPTORRENT for TIPTORRENT-CACHE to keep naming consistency. #1062 tiptorrent stores whole file instead of chunks Tiptorrent now stores the whole file instead of the split chunks. Removes any chunk related code from cloning engine scripts. Requires newer version of tiptorrent installed (tiptorrent-static >= 1.0.0-6). Always compute MD5 of cached image Do not rely on MD5 checksum file from the cache on tiptorrent transfers. Otherwise, cloning engine could restore a corrupted image. #1062 Check tiptorrent exit code Otherwise, the image restoration continues in failed downloads. --- client/engine/Image.lib | 92 ++++++++++++++++++++++++++++++++++---- client/shared/etc/init/default.sh | 3 ++ client/shared/scripts/deployImage | 5 +++ client/shared/scripts/restoreImage | 5 +++ 4 files changed, 97 insertions(+), 8 deletions(-) (limited to 'client') diff --git a/client/engine/Image.lib b/client/engine/Image.lib index 941deb91..728eb24a 100755 --- a/client/engine/Image.lib +++ b/client/engine/Image.lib @@ -102,6 +102,48 @@ esac [ -n "$PARAM1" ] && echo "$PARAM1 $PARAM2 $PARAM3 $IMGFILE" } +function ogTiptorrentGet () +{ +local MAXTRIES err +i=0 +MAXTRIES=3 +while [ "$i" -lt $MAXTRIES ] +do + wget --spider -q "$ogrepo:9999/$1" && break + i=$(( i + 1 )) +done +if [ $i -eq 3 ]; then + ogRaiseError "$OG_ERR_NOTFOUND" "Error checking $1" + return $? +fi +pushd /opt/opengnsys/cache/opt/opengnsys/images > /dev/null || return 1 +/opt/opengnsys/bin/tiptorrent-client "$ogrepo" "$1" > /dev/null +err=$? +popd > /dev/null || return 1 +if [ $err -ne 0 ]; then + ogRaiseError "$OG_ERR_NOTFOUND" "Error downloading $1 image, tiptorrent-client failed. See /var/log/syslog on the client for details." +fi +return $err +} + +function ogTiptorrentGetToCache () +{ +local MAXTRIES MD5FILE MD5IMG +if ! wget -qO "$1".full.sum "$ogrepo:9999/$2.full.sum"; then + ogRaiseError "$OG_ERR_NOTFOUND" "Error downloading $2 checksum" + return $? +fi +MD5FILE=$(cat "$1".full.sum) +if ! ogTiptorrentGet "$2"; then + return 1; +fi +MD5IMG=$(md5sum "$IMGPATH" | cut -f1 -d " ") +if [ "$MD5IMG" != "$MD5FILE" ]; then + ogRaiseError "$OG_ERR_NOTFOUND" "Error downloading $2, checksum mismatch" + return $? +fi +return 0 +} #/** # ogRestoreImageSyntax path_filename path_device [str_tools] [str_compressionlevel] @@ -121,7 +163,8 @@ esac #*/ ## function ogRestoreImageSyntax () { -local TOOL COMPRESSOR LEVEL PART IMGFILE FILEHEAD INFOIMG +local TOOL COMPRESSOR LEVEL PART IMGFILE FILEHEAD INFOIMG IMGPATH IMGCACHEDIR +local MD5FILE MD5WGET # Si se solicita, mostrar ayuda. @@ -131,8 +174,8 @@ if [ "$*" == "help" ]; then return fi -# Error si no se reciben entre 2 y 4 parámetros. -[ $# -ge 2 -a $# -le 4 ] || ogRaiseError $OG_ERR_FORMAT "$*" || return $? +# Error si no se reciben entre 2 y 5 parámetros. +[ $# -ge 2 -a $# -le 5 ] || ogRaiseError $OG_ERR_FORMAT "$*" || return $? # controlamos que el parametro 1 (imagen) es tipo file. [ -f $1 ] || ogRaiseError $OG_ERR_NOTFOUND "$1" || return $? @@ -147,9 +190,8 @@ if [ "$#" -eq 2 ]; then ogRestoreImageSyntax $IMGFILE $PART $TOOL $COMPRESSOR fi - # Si cuatro parametros genera sintaxis -if [ "$#" -eq 4 ]; then +if [ "$#" -eq 4 ] || [ "$#" -eq 5 ]; then IMGFILE=$1 PART=$2 # comprobamos parametro herramienta compresion. @@ -198,7 +240,35 @@ if [ "$#" -eq 4 ]; then ;; esac - echo "$COMPRESSOR $IMGFILE $MBUFFER $TOOL" + if [ "$5" == "true" ]; then + IMGFILE=${1/\/opt\/opengnsys\/images\//} + if ogFindCache &>/dev/null; then + IMGCACHEDIR=$(ogGetMountPoint $(ogFindCache))/opt/opengnsys/images + IMGPATH="$IMGCACHEDIR/$IMGFILE" + if [ -f "$IMGPATH" ]; then + MD5FILE=$(md5sum "$IMGPATH" | cut -f1 -d " ") + MD5WGET=$(wget -qO - "$ogrepo:9999/$IMGFILE.full.sum") + if [ "$MD5FILE" != "$MD5WGET" ]; then + ogTiptorrentGetToCache "$IMGPATH" "$IMGFILE" + fi + else + mkdir -p "$IMGCACHEDIR" + ogTiptorrentGetToCache "$IMGPATH" "$IMGFILE" + fi + if [ $? -ne 0 ]; then + ogRaiseError "$OG_ERR_NOTFOUND" "Failed tiptorrent to cache $TOOL" + return $? + fi + CAT="cat $IMGPATH |" + COMPRESSOR="$COMPRESSOR -" + echo "$CAT $COMPRESSOR $MBUFFER $TOOL" + else + ogRaiseError "$OG_ERR_NOTFOUND" "Tiptorrent needs cache $TOOL" + return $? + fi + else + echo "$COMPRESSOR $IMGFILE $MBUFFER $TOOL" + fi fi } @@ -363,6 +433,7 @@ ERRCODE=$? if [ $ERRCODE == 0 ]; then echo "$(ogGetImageInfo $IMGFILE):$(ogGetHostname)" > $IMGFILE.info cp -f $IMGFILE.info /tmp/image.info + md5sum "$IMGFILE" | cut -f1 -d " " > "$IMGFILE".full.sum else ogRaiseError $OG_ERR_IMAGE "$1 $2 $IMGFILE" rm -f "$IMGFILE" @@ -746,7 +817,7 @@ if [ "$*" == "help" ]; then return fi # Error si no se reciben 4 parámetros. -[ $# == 4 ] || ogRaiseError $OG_ERR_FORMAT || return $? +[ $# == 4 ] || [ $# == 5 ] || ogRaiseError $OG_ERR_FORMAT || return $? # Procesar parámetros. PART="$(ogDiskToDev $3 $4)" || return $(ogRaiseError $OG_ERR_NOTFOUND " $3 $4"; echo $?) #IMGTYPE=$(ogGetImageType "$1" "$2") @@ -779,7 +850,12 @@ fi # Solicitamos la generación de la instruccion a ejecutar # Atención: no se comprueba el tipo de sistema de archivos. # Atención: no se comprueba incongruencia entre partición e imagen. -PROGRAM=`ogRestoreImageSyntax $IMGFILE $PART` +INFOIMG=$(ogGetImageInfo $IMGFILE) || ogRaiseError $OG_ERR_NOTFOUND "No Image $1" || return $? +TOOL=`echo $INFOIMG | cut -f1 -d:` +COMPRESSOR=`echo $INFOIMG | cut -f2 -d:` +PROGRAM=`ogRestoreImageSyntax $IMGFILE $PART $TOOL $COMPRESSOR $5` +echo PROGRAMA +echo "$PROGRAM" # Desmontar y bloquear partición. ogUnmount $3 $4 2>/dev/null || return $(ogRaiseError $OG_ERR_PARTITION " $3 $4"; echo $?) diff --git a/client/shared/etc/init/default.sh b/client/shared/etc/init/default.sh index ed5527b9..fd88bb53 100755 --- a/client/shared/etc/init/default.sh +++ b/client/shared/etc/init/default.sh @@ -19,6 +19,9 @@ echo "${MSG_LAUNCHCLIENT:-.}" [ -f /usr/local/etc/${LANG%_*}.qmap ] && export QWS_KEYBOARD="TTY:keymap=/usr/local/etc/${LANG%_*}.qmap" if [ "$ogstatus" != "offline" ]; then + if ogFindCache &>/dev/null; then + /opt/opengnsys/bin/tiptorrent --root /opt/opengnsys/cache/opt/opengnsys/images -n 3 & + fi GROUP_CONFIG=/opt/opengnsys/ogClient/cfg/${group/ /_}.json PYMAJMIN=`python3 --version | cut -d" " -f2 | cut -d. -f1,2` export PYTHONPATH=/opt/opengnsys/lib/python$PYMAJMIN diff --git a/client/shared/scripts/deployImage b/client/shared/scripts/deployImage index d74d3ac2..e34c908f 100755 --- a/client/shared/scripts/deployImage +++ b/client/shared/scripts/deployImage @@ -121,6 +121,9 @@ case "$MODE" in UNICAST-DIRECT) NEXTOPERATION=UNICAST ;; + TIPTORRENT) + NEXTOPERATION=TIPTORRENT + ;; # Si protocolo es torrent|torrent-cache o multicast|multicast-cache TORRENT|TORRENT-CACHE|MULTICAST|MULTICAST-CACHE|UNICAST|UNICAST-CACHE) @@ -198,6 +201,8 @@ case "$NEXTOPERATION" in PARAMS="CACHE $IMGNAME $DISK $PART" ;; UNICAST) PARAMS="$REPO $IMGNAME $DISK $PART" ;; + TIPTORRENT) + PARAMS="$REPO $IMGNAME $DISK $PART TIPTORRENT" ;; MULTICAST) PARAMS="$REPO $IMGNAME $DISK $PART $PROTO $PROTOOPT" ;; esac diff --git a/client/shared/scripts/restoreImage b/client/shared/scripts/restoreImage index d9e7f584..537bcbb9 100755 --- a/client/shared/scripts/restoreImage +++ b/client/shared/scripts/restoreImage @@ -77,6 +77,11 @@ fi # Procesar protocolos de transferencia. case "$PROTO" in + TIPTORRENT) + ogEcho log session "[40] ogRestoreImage $REPO $IMGNAME $DISK $PART true" + ogExecAndLog command ogRestoreImage "$REPO" "$IMGNAME" "$DISK" "$PART" true + RETVAL=$? + ;; UNICAST|UNICAST-DIRECT) # Restaurar la imagen. ogEcho log session "[40] ogRestoreImage $REPO $IMGNAME $DISK $PART UNICAST" -- cgit v1.2.3-18-g5258