#!/bin/bash
######################################
## NodeWeaver - manual DFS orphan image cleanup
######################################
# © NodeWeaver 2026
######################################

## NOTES:
#
# Finds "orphan" image backing files in an OpenNebula image datastore: files
# that physically exist on the DFS but are referenced by NOTHING that matters -
# i.e. they are
#   1. NOT registered (or referenced) in the OpenNebula database (any image), AND
#   2. NOT the target of a symlink in the system datastore 0 (a VM disk binding), AND
#   3. NOT held open by any running qemu-system process on ANY cluster node.
#
# Such files are typically left behind by disk "save_as" operations (e.g.
# saveallrunningvms) whose image object was later deleted while the file on the
# DFS survived. Deleting OpenNebula images/snapshots never reclaims them because
# OpenNebula no longer tracks them - so they have to be found and removed by file.
#
# WHY all three checks: a datastore file can back a LIVE VM with no image object
# at all, because VM disks here are direct symlinks
#   datastores/0/<vmid>/disk.N -> /var/lib/one//datastores/1/<hash>
# and qemu opens the resolved path. "Not a registered image" alone is NOT enough
# to call a file unused - the symlink and the live file-descriptor checks are what
# make a deletion safe.
#
# By default the script only REPORTS (dry-run). It deletes nothing unless invoked
# with --cleanup, and even then only after an explicit typed confirmation and a
# fresh re-verification of every file immediately before removal.
#
# Usage:
#   orphanimagecleaner [--datastore N] [--cleanup]
#     --datastore N   image datastore id to scan (default: 1, the default datastore)
#     --cleanup       after reporting, interactively offer to delete the orphans
#     -h | --help     show this help
#
# Must run on the LEADER (oned and the DFS master live there).

## VARIABLES:

source /opt/scripts/misc/nwbashlib

LOGGERNAME=ORPHANIMAGECLEANER
DSID=1
DO_CLEANUP=false
ONEBASE=/var/lib/one/datastores
WORKDIR="$(mktemp -d /tmp/orphancleanup.XXXXXX)"
trap 'rm -rf "$WORKDIR"' EXIT

REG="$WORKDIR/registered.txt"      # hashes referenced anywhere in the OpenNebula DB
SYM="$WORKDIR/symlinked.txt"       # hashes that are a system-datastore (DS0) symlink target
QEMU="$WORKDIR/qemuopen.txt"       # hashes currently open by a qemu-system process, any node
FILES="$WORKDIR/files.txt"         # hash-named files physically present in the datastore
ORPHANS="$WORKDIR/orphans.txt"     # the result: present-but-unreferenced files
VERIFY_OK=true                     # set false if a node could not be checked -> refuse deletion

## FUNCTIONS:

usage() {
	sed -n '2,/^## VARIABLES:/p' "$0" | grep -E '^#' | sed 's/^# \{0,1\}//'
	exit "${1:-0}"
}

# Emit the set of datastore-$DSID hashes a qemu-system process holds open, on one
# node. Runs locally for this node, over ssh (house NWSSH credentials) otherwise.
# A non-zero return means "could not verify this node" - the caller must treat
# that as unsafe and disable deletion.
scan_qemu_fds_on() {
	local node="$1"
	local remote_cmd
	remote_cmd='for p in $(pgrep -f qemu-system 2>/dev/null); do ls -l /proc/$p/fd 2>/dev/null; done | grep -oE "datastores/'"$DSID"'/[a-f0-9]{32}" | sed "s#.*/##" | sort -u'
	if [ "$node" = "$(hostname)" ]; then
		bash -c "$remote_cmd" </dev/null
		return 0
	fi
	# </dev/null: this runs inside while-read loops; without it ssh would consume
	# the caller's stdin (the orphan list) and break iteration.
	sshpass -p "$NWSSHPASS" ssh $NWSSH_TIMEOUT_OPTS -q -l root "$node" "$remote_cmd" </dev/null
}

## SCRIPT:

# --- argument parsing ---
while [ $# -gt 0 ]; do
	case "$1" in
		--datastore) DSID="$2"; shift 2 ;;
		--cleanup)   DO_CLEANUP=true; shift ;;
		-h|--help)   usage 0 ;;
		*) echo "Unknown argument: $1"; usage 1 ;;
	esac
done

DSDIR="$ONEBASE/$DSID"

# --- guards ---
if [ "$ROLE" != "LEADER" ]; then
	logger -s "___$LOGGERNAME: ERROR this node is '$ROLE', not LEADER - run it on the leader (oned and the DFS master live there). Aborting."
	exit 1
fi
if [ "$DSID" = "0" ]; then
	logger -s "___$LOGGERNAME: ERROR refusing to scan the system datastore 0 (it legitimately holds live VM disks). Aborting."
	exit 1
fi
if [ ! -d "$DSDIR" ]; then
	logger -s "___$LOGGERNAME: ERROR datastore directory $DSDIR does not exist. Aborting."
	exit 1
fi
if ! onedatastore show "$DSID" >/dev/null 2>&1; then
	logger -s "___$LOGGERNAME: ERROR datastore $DSID is unknown to OpenNebula. Aborting."
	exit 1
fi

logger -s "___$LOGGERNAME: scanning datastore $DSID ($DSDIR) for orphan image files"

# --- 1. hashes referenced anywhere in the OpenNebula database (conservative: any
#        mention of this datastore's path in any image or VM keeps the file) ---
{ oneimage list -x 2>/dev/null; onevm list -x 2>/dev/null; } \
	| grep -oE "datastores/$DSID/[a-f0-9]{32}" | sed 's#.*/##' | sort -u > "$REG"

# --- 2. system-datastore (DS0) symlink targets that point into this datastore
#        (these are live VM disk bindings, even with no image object) ---
find "$ONEBASE/0" -type l 2>/dev/null -exec readlink {} \; \
	| grep -oE "datastores/$DSID/[a-f0-9]{32}" | sed 's#.*/##' | sort -u > "$SYM"

# --- 3. files actually open by a qemu-system process on EVERY cluster node ---
: > "$QEMU"
for NODE in $(tellmenodes | xargs); do
	if NODE_FDS="$(scan_qemu_fds_on "$NODE")"; then
		echo "$NODE_FDS" >> "$QEMU"
	else
		VERIFY_OK=false
		logger -s "___$LOGGERNAME: WARNING could not read qemu file descriptors on node '$NODE' - cleanup will be DISABLED for safety."
	fi
done
grep -E '^[a-f0-9]{32}$' "$QEMU" | sort -u > "$QEMU.sorted" && mv "$QEMU.sorted" "$QEMU"

# --- candidate files: hash-named regular files only (skip .snap dirs, skip
#     non-image bookkeeping files like images-list.txt) ---
find "$DSDIR" -maxdepth 1 -type f -regextype posix-extended -regex '.*/[a-f0-9]{32}' \
	-printf '%f\n' 2>/dev/null | sort -u > "$FILES"

# --- orphans = present files - referenced - symlinked - qemu-open ---
comm -23 "$FILES" "$REG" | comm -23 - "$SYM" | comm -23 - "$QEMU" > "$ORPHANS"

ORPHAN_COUNT=$(wc -l < "$ORPHANS")
echo
echo "Datastore $DSID ($DSDIR)"
echo "  image files present : $(wc -l < "$FILES")"
echo "  referenced in DB    : $(wc -l < "$REG")"
echo "  DS0 symlink targets : $(wc -l < "$SYM")"
echo "  open by live qemu   : $(wc -l < "$QEMU")"
echo "  ORPHANS (unreferenced): $ORPHAN_COUNT"
echo

if [ "$ORPHAN_COUNT" -eq 0 ]; then
	logger -s "___$LOGGERNAME: no orphans found in datastore $DSID. Done."
	exit 0
fi

# --- report the orphans with size and age, and the reclaimable total ---
TOTAL=0
printf '%-34s %15s   %s\n' "FILE" "BYTES" "MTIME"
while read -r h; do
	[ -z "$h" ] && continue
	sz=$(stat -c '%s' "$DSDIR/$h" 2>/dev/null)
	mt=$(stat -c '%y' "$DSDIR/$h" 2>/dev/null | cut -d. -f1)
	TOTAL=$((TOTAL + ${sz:-0}))
	printf '%-34s %15s   %s\n' "$h" "${sz:-?}" "$mt"
done < "$ORPHANS"
echo
echo "  Reclaimable (apparent): $(numfmt --to=iec --suffix=B "$TOTAL" 2>/dev/null || echo "${TOTAL}B")"
echo

if [ "$DO_CLEANUP" != "true" ]; then
	echo "Dry-run only. Re-run with --cleanup to delete these files."
	logger -s "___$LOGGERNAME: dry-run: $ORPHAN_COUNT orphan(s), ${TOTAL} bytes reclaimable in datastore $DSID."
	exit 0
fi

if [ "$VERIFY_OK" != "true" ]; then
	echo "Cleanup DISABLED: at least one node's qemu state could not be verified."
	echo "A file open on an unverified node could be silently destroyed. Fix node access and retry."
	logger -s "___$LOGGERNAME: ERROR cleanup aborted - incomplete cross-node verification."
	exit 1
fi

echo "About to DELETE $ORPHAN_COUNT file(s) from datastore $DSID."
echo "This is irreversible. Type exactly  iamserious  to proceed (anything else cancels):"
read -r CONFIRM
if [ "$CONFIRM" != "iamserious" ]; then
	echo "Cancelled, nothing deleted."
	exit 0
fi

# --- delete, re-verifying each file one last time right before removal
#     (guards against a VM being deployed onto a file between scan and confirm).
#     The orphan list is read on fd 3 so nothing inside the loop can consume it. ---
DELETED=0; FREED=0; SKIPPED=0
while read -r h <&3; do
	[ -z "$h" ] && continue
	# fresh point-in-time safety re-check for this single file
	inuse=false
	failnode=""
	if grep -qxF "$h" "$REG" \
	   || find "$ONEBASE/0" -type l -exec readlink {} \; 2>/dev/null | grep -q "datastores/$DSID/$h"; then
		inuse=true
	fi
	# qemu open-fd re-check on every node; if a node cannot be verified right now,
	# fail safe and treat the file as in-use rather than risk deleting a live disk.
	if [ "$inuse" = false ]; then
		for NODE in $(tellmenodes | xargs); do
			if NODE_FDS="$(scan_qemu_fds_on "$NODE")"; then
				if printf '%s\n' "$NODE_FDS" | grep -qxF "$h"; then inuse=true; break; fi
			else
				failnode="$NODE"; inuse=true; break
			fi
		done
	fi
	if [ "$inuse" = true ]; then
		if [ -n "$failnode" ]; then
			logger -s "___$LOGGERNAME: SKIP $h - could not verify qemu state on node '$failnode'; not deleting."
		else
			logger -s "___$LOGGERNAME: SKIP $h became referenced/in-use since the scan - not deleting."
		fi
		SKIPPED=$((SKIPPED + 1))
		continue
	fi
	sz=$(stat -c '%s' "$DSDIR/$h" 2>/dev/null)
	if rm -f "$DSDIR/$h"; then
		DELETED=$((DELETED + 1)); FREED=$((FREED + ${sz:-0}))
		logger -s "___$LOGGERNAME: deleted orphan $DSDIR/$h (${sz:-?} bytes)"
	else
		logger -s "___$LOGGERNAME: WARNING failed to delete $DSDIR/$h"
	fi
done 3< "$ORPHANS"

echo
echo "Deleted $DELETED file(s), freed $(numfmt --to=iec --suffix=B "$FREED" 2>/dev/null || echo "${FREED}B"); skipped $SKIPPED."
logger -s "___$LOGGERNAME: cleanup done on datastore $DSID: deleted $DELETED, freed ${FREED} bytes, skipped $SKIPPED."