General | Add support for F2FS and Btrfs root file systems

.meta/dietpi-imager

@@ -12,7 +12,7 @@
 	# - Create new .img file from drive
 	#   or use an existing .img file
 	#   or use Clonezilla to generate a bootable installer ISO from drive for x86_64 systems
-	# - Minimises root partition and file system
+	# - Minimises root partition and filesystem
 	# - Hashes and 7z's the final image ready for release
 	#////////////////////////////////////
 
@@ -35,6 +35,7 @@
 	FP_SOURCE=
 	PART_TABLE_TYPE=
 	FP_ROOT_DEV=
+	ROOT_FS_TYPE=
 	CLONING_TOOL='dd'
 	OUTPUT_IMG_EXT='img'
 	OUTPUT_IMG_NAME='DietPi_RPi-ARMv6-Buster'
@@ -50,19 +51,30 @@
 
 	Menu_Source_Type()
 	{
+		main_menu_choice='Source type' # On cancel, keep this entry selected
+
 		G_WHIP_MENU_ARRAY=(
 			'Drive' ': The OS is stored on an attached drive.'
 			'Image' ': The OS is stored as an image file.'
 		)
 		G_WHIP_DEFAULT_ITEM=$SOURCE_TYPE
-		G_WHIP_MENU 'Please select how the input OS is stored:' && { Delete_Loopback; SOURCE_TYPE=$G_WHIP_RETURNED_VALUE FP_SOURCE_IMG='' FP_SOURCE='' FP_ROOT_DEV=''; }
+		G_WHIP_MENU 'Please select how the input OS is stored:' || return
+		Delete_Loopback
+		SOURCE_TYPE=$G_WHIP_RETURNED_VALUE
+		FP_SOURCE_IMG=
+		FP_SOURCE=
+		FP_ROOT_DEV=
+
+		Menu_Source_Path # Directly open this menu next
 	}
 
 	Menu_Source_Path()
 	{
+		main_menu_choice='Source path' # On cancel, keep this entry selected
+
 		if [[ $SOURCE_TYPE == 'Drive' ]]
 		then
-			# Detect drives with a partition table, containing a partition with ext4 file system, excluding the hosts root file system drive
+			# Detect drives with a partition table, containing a partition with ext4 filesystem, excluding the hosts root filesystem drive
 			mapfile -t G_WHIP_MENU_ARRAY < <(mawk -v root="$(lsblk -npo PKNAME "$G_ROOTFS_DEV")" '{if ( $1!=root && $2=="ext4" ) print $1"\n"$2}' < <(lsblk -rnpo PKNAME,FSTYPE) | sort -u)
 
 			if [[ ! ${G_WHIP_MENU_ARRAY[0]} ]]
@@ -80,9 +92,10 @@
 			G_WHIP_DEFAULT_ITEM=$FP_SOURCE
 			G_WHIP_MENU 'Please select the drive you wish to create the image from:
 \nNB: All mounted partitions of the selected drive will be unmounted.' || return
-			FP_SOURCE=$G_WHIP_RETURNED_VALUE FP_ROOT_DEV=''
+			FP_SOURCE=$G_WHIP_RETURNED_VALUE
+			FP_ROOT_DEV=
 
-			G_DIETPI-NOTIFY 2 "Unmounting all file systems below selected $FP_SOURCE ..."
+			G_DIETPI-NOTIFY 2 "Unmounting all filesystems below selected $FP_SOURCE ..."
 			local mountpoint
 			for i in "$FP_SOURCE"?*
 			do
@@ -95,7 +108,8 @@
 			[[ -f $fp_selected ]] && rm $fp_selected # Failsafe
 			/boot/dietpi/dietpi-explorer 1
 			[[ -f $fp_selected && $(<$fp_selected) ]] || { G_DIETPI-NOTIFY 1 'No image file selected, aborting...'; read -rp 'Press any key to return to menu...'; return; }
-			FP_SOURCE_IMG=$(<$fp_selected) FP_ROOT_DEV=''
+			FP_SOURCE_IMG=$(<$fp_selected)
+			FP_ROOT_DEV=
 			rm $fp_selected
 			[[ -f $FP_SOURCE_IMG ]] || { G_DIETPI-NOTIFY 1 "Selected image file ($FP_SOURCE_IMG) does not exist, aborting..."; read -rp 'Press any key to return to menu...'; return; }
 
@@ -108,22 +122,31 @@
 			G_EXEC_NOEXIT=1 G_EXEC partx -u "$FP_SOURCE" || return
 			G_DIETPI-NOTIFY 0 "Mounted the image ($FP_SOURCE_IMG) as loopback device: $FP_SOURCE"
 		fi
+
+		Menu_Source_RootFS # Directly open this menu next
 	}
 
 	Menu_Source_RootFS()
 	{
+		main_menu_choice='Source rootfs' # On cancel, keep this entry selected
+
 		# Detect partitions and list for selection
 		# Coders NB: read/mapfile cannot be easily used here since we need to parse multiple lines and split at newline AND space.
 		# shellcheck disable=SC2207
 		G_WHIP_MENU_ARRAY=($(lsblk -rnpo NAME,SIZE "$FP_SOURCE"?*))
 		# Visually separate dev name and size and add FS type
 		for ((i=1;i<${#G_WHIP_MENU_ARRAY[@]};i+=2)); do G_WHIP_MENU_ARRAY[$i]=": $(lsblk -drno SIZE,FSTYPE "${G_WHIP_MENU_ARRAY[$i-1]}")"; done
 		G_WHIP_DEFAULT_ITEM=$FP_ROOT_DEV
-		G_WHIP_MENU 'Please select the OS root partition:' && FP_ROOT_DEV=$G_WHIP_RETURNED_VALUE
+		G_WHIP_MENU 'Please select the OS root partition:' || return
+		FP_ROOT_DEV=$G_WHIP_RETURNED_VALUE
+
+		Menu_Target_Type # Directly open this menu next
 	}
 
 	Menu_Target_Type()
 	{
+		main_menu_choice='Target type' # On cancel, keep this entry selected
+
 		G_WHIP_MENU_ARRAY=(
 			'dd'         ': Create an .img file to flash to target system drive directly.'
 			'Clonezilla' ': Create an installer .iso file to boot from removeable media. (x86_64 only!)'
@@ -136,12 +159,17 @@
  - Clonezilla: An installer ISO image is created which must be flashed to an external/USB/removeable drive.
 	Boot from the external drive will launch Clonezilla and allow you to install DietPi to any internal drive.
 	This is required e.g. for UEFI images.
-	NB: Only compatible with x86_64 systems!' && CLONING_TOOL=$G_WHIP_RETURNED_VALUE
+	NB: Only compatible with x86_64 systems!' || return
+		CLONING_TOOL=$G_WHIP_RETURNED_VALUE
 		[[ $CLONING_TOOL == 'dd' ]] && OUTPUT_IMG_EXT='img' || OUTPUT_IMG_EXT='iso'
+
+		Menu_Target_Name # Directly open this menu next
 	}
 
 	Menu_Target_Name()
 	{
+		main_menu_choice='Target name' # On cancel, keep this entry selected
+
 		G_WHIP_DEFAULT_ITEM=$OUTPUT_IMG_NAME
 		G_WHIP_INPUTBOX 'Please enter a name for the new image:\n - DietPi_<device>-<arch>-<distro>\n - E.g.: DietPi_RPi-ARMv6-Buster' || return
 		OUTPUT_IMG_NAME=$G_WHIP_RETURNED_VALUE
@@ -154,6 +182,8 @@
 			G_WHIP_YESNO "[WARNING] $PWD/$OUTPUT_IMG_NAME.$OUTPUT_IMG_EXT already exists
 \nDo you want to overwrite or backup the existing file to $PWD/$OUTPUT_IMG_NAME.$OUTPUT_IMG_EXT.bak?" || mv "$OUTPUT_IMG_NAME.$OUTPUT_IMG_EXT"{,.bak}
 		fi
+
+		main_menu_choice='Mount' # Select this entry next
 	}
 
 	Menu_Main()
@@ -174,7 +204,6 @@
 		G_WHIP_DEFAULT_ITEM=$main_menu_choice
 		G_WHIP_BUTTON_CANCEL_TEXT='Exit'
 		G_WHIP_MENU 'Select input parameters and hit "Start" to continue image creation:' || exit 0
-		main_menu_choice=$G_WHIP_RETURNED_VALUE
 
 		case $G_WHIP_RETURNED_VALUE in
 			'Source type') Menu_Source_Type;;
@@ -183,12 +212,33 @@
 			'Target type') Menu_Target_Type;;
 			'Target name') Menu_Target_Name;;
 			'Mount') [[ $MOUNT_IT == 'Off' ]] && MOUNT_IT='On' || MOUNT_IT='Off';;
+			'Start') main_menu_choice='Start';;
 		esac
 	}
 
+	Run_fsck()
+	{
+		if [[ $ROOT_FS_TYPE == 'ext4' ]]
+		then
+			G_EXEC_OUTPUT=1 G_EXEC e2fsck -fp "$FP_ROOT_DEV"
+
+		elif [[ $ROOT_FS_TYPE == 'f2fs' ]]
+		then
+			G_EXEC_OUTPUT=1 G_EXEC fsck.f2fs -f "$FP_ROOT_DEV"
+
+		elif [[ $ROOT_FS_TYPE == 'btrfs' ]]
+		then
+			G_EXEC_OUTPUT=1 G_EXEC btrfs check --repair "$FP_ROOT_DEV"
+		else
+			G_DIETPI-NOTIFY 1 "Unsupported root filesystem type ($PART_TABLE_TYPE), aborting..."
+			exit 1
+		fi
+	}
+
 	Main(){
 
-		local main_menu_choice='Source type'
+		local main_menu_choice
+		Menu_Source_Type
 		until [[ $main_menu_choice == 'Start' ]]
 		do
 			Menu_Main
@@ -201,8 +251,8 @@
 		G_AG_CHECK_INSTALL_PREREQ parted zerofree p7zip $fdisk
 		unset -v fdisk
 
-		# Auto detect partition table type, failsafe detection of MBR to debug possibly other/unknown wording/partition table types
-		PART_TABLE_TYPE=$(sfdisk -l "$FP_SOURCE" | mawk '/^Disklabel type:/{print $3;exit}')
+		# Detect partition table type, failsafe detection of MBR to debug possibly other/unknown wording/partition table types
+		PART_TABLE_TYPE=$(lsblk -no PTTYPE "$FP_ROOT_DEV")
 		if [[ $PART_TABLE_TYPE == 'dos' ]]
 		then
 			G_DIETPI-NOTIFY 2 'MBR partition table detected'
@@ -221,7 +271,10 @@
 			exit 1
 		fi
 
-		G_EXEC_OUTPUT=1 G_EXEC e2fsck -fp "$FP_ROOT_DEV"
+		# Detect root filesystem type
+		ROOT_FS_TYPE=$(lsblk -no FSTYPE "$FP_ROOT_DEV")
+
+		Run_fsck
 
 		# Remount image for any required edits
 		G_EXEC mkdir -p $FP_MNT_TMP
@@ -248,51 +301,95 @@
 		G_EXEC rmdir $FP_MNT_TMP
 		G_EXEC partprobe "$FP_SOURCE" # Failsafe
 		G_EXEC partx -u "$FP_SOURCE" # Failsafe
-		G_EXEC_OUTPUT=1 G_EXEC e2fsck -fp "$FP_ROOT_DEV"
-
-		# Shrink file system to minimum
-		# - Run multiple times until no change is done any more
-		G_DIETPI-NOTIFY 2 'Shrinking RootFS to minimum size...'
-		local out
-		FS_SIZE=0
-		while :
-		do
-			resize2fs -M "$FP_ROOT_DEV" 2>&1 | tee resize2fs_out
-			if out=$(grep -im1 'nothing to do!' resize2fs_out)
-			then
-				FS_SIZE=$(mawk '{print $5}' <<< "$out") # blocks
-				BLOCK_SIZE=${out%%k) *} BLOCK_SIZE=${BLOCK_SIZE##*\(} # KiB
-				# Re-add 4 MiB to be failsafe, was required on Raspbian Buster for successful boot
-				FS_SIZE=$(( $FS_SIZE + 4096/$BLOCK_SIZE )) # blocks
-				rm resize2fs_out
-				G_EXEC resize2fs "$FP_ROOT_DEV" $FS_SIZE
-				G_DIETPI-NOTIFY 0 "Reduced RootFS size to $(( $FS_SIZE * $BLOCK_SIZE / 1024 + 1 )) MiB"
-				FS_SIZE=$(( $FS_SIZE * $BLOCK_SIZE * 2 )) # blocks => 512 byte sectors
-				break
-
-			elif out=$(grep -im1 'no such file or directory' resize2fs_out)
-			then
-				G_DIETPI-NOTIFY 1 'Partition not found, aborting...'
-				exit 1
-			fi
-		done
+		Run_fsck
+
+		# Shrink filesystem to minimum
+		if [[ $ROOT_FS_TYPE == 'ext4' ]]
+		then
+			# Run multiple times until no change is done any more
+			G_DIETPI-NOTIFY 2 'Shrinking root filesystem to minimum size...'
+			local out
+			FS_SIZE=0
+			while :
+			do
+				resize2fs -M "$FP_ROOT_DEV" 2>&1 | tee resize2fs_out
+				if out=$(grep -im1 'nothing to do!' resize2fs_out)
+				then
+					FS_SIZE=$(mawk '{print $5}' <<< "$out") # blocks
+					BLOCK_SIZE=${out%%k) *} BLOCK_SIZE=${BLOCK_SIZE##*\(} # KiB
+					# Re-add 4 MiB to be failsafe, was required on Raspbian Buster for successful boot
+					FS_SIZE=$(( $FS_SIZE + 4096/$BLOCK_SIZE )) # blocks
+					rm resize2fs_out
+					G_EXEC resize2fs "$FP_ROOT_DEV" $FS_SIZE
+					G_DIETPI-NOTIFY 0 "Reduced RootFS size to $(( $FS_SIZE * $BLOCK_SIZE / 1024 + 1 )) MiB"
+					FS_SIZE=$(( $FS_SIZE * $BLOCK_SIZE * 2 )) # blocks => 512 byte sectors
+					break
+
+				elif out=$(grep -im1 'no such file or directory' resize2fs_out)
+				then
+					G_DIETPI-NOTIFY 1 'Partition not found, aborting...'
+					exit 1
+				fi
+			done
+			G_DIETPI-NOTIFY 2 'Overriding root filesystem free space with zeros to purge removed data and allow further archive size reduction...'
+			G_EXEC_OUTPUT=1 G_EXEC zerofree -v "$FP_ROOT_DEV"
+			G_EXEC sync
+
+		elif [[ $ROOT_FS_TYPE == 'f2fs' ]] #&& $(lsblk -rno FSUSE% "$FP_ROOT_DEV") =~ ^(9[5-9]|100)%$ ]]
+		then
+			# F2FS does not support shrinking: https://www.reddit.com/r/archlinux/comments/bpp77f/shrinking_a_f2fs_partition/
+			# Hence copy all data outside, remove and re-create a smaller filesytem, then copy them back in, as long as disk usage is not >=95% already.
+			# The UUID changes and there is currently no way to change it back...
+			#local usage=$(lsblk -rnbo FSUSED "$FP_ROOT_DEV") # bytes
+			#local sector_size=$(lsblk -rnbo LOG-SEC "$FP_ROOT_DEV") # bytes
+			#FS_SIZE=$(( ( $usage + 4*1024**2 ) / $sector_size )) # bytes + 4 MiB buffer => sectors
+			#G_DIETPI-NOTIFY 2 'Copying root filesystem content to temporary directory'
+			G_EXEC mkdir -p ${FP_MNT_TMP}{,_backup}
+			G_EXEC mount -o ro "$FP_ROOT_DEV" $FP_MNT_TMP
+			#G_EXEC cp -a $FP_MNT_TMP/. ${FP_MNT_TMP}_backup/
+			#G_EXEC umount $FP_MNT_TMP
+			#G_DIETPI-NOTIFY 2 'Purging root filesystem'
+			#G_EXEC dd if=/dev/zero of="$FP_ROOT_DEV" bs=4K count=10
+			#G_DIETPI-NOTIFY 2 'Re-creating smaller root filesystem' # Probably sload.f2fs can replace this? https://manpages.debian.org/sload.f2fs
+			#G_EXEC_OUTPUT=1 G_EXEC mkfs.f2fs -w "$sector_size" "$FP_ROOT_DEV" $FS_SIZE
+			#G_DIETPI-NOTIFY 2 'Moving root filesystem content back'
+			#G_EXEC mount "$FP_ROOT_DEV" $FP_MNT_TMP
+			#G_EXEC cp -a ${FP_MNT_TMP}_backup/. $FP_MNT_TMP/
+			#G_EXEC rm -R ${FP_MNT_TMP}_backup
+			G_EXEC fstrim $FP_MNT_TMP
+			G_EXEC sync
+			G_EXEC sleep 1 # Give the system 1 second to avoid "mount is busy"
+			G_EXEC umount $FP_MNT_TMP
+			G_EXEC rmdir $FP_MNT_TMP
+			# Assure root filesystem size is in 512 byte sectors, as this is what sfdisk assmes
+			#FS_SIZE=$(( $FS_SIZE * $sector_size / 512 ))
+		elif [[ $ROOT_FS_TYPE == 'btrfs' ]]
+		then
+			G_DIETPI-NOTIFY 2 'Shrinking root filesystem to minimum size...'
+			G_EXEC mkdir -p $FP_MNT_TMP
+			G_EXEC mount "$FP_ROOT_DEV" $FP_MNT_TMP
+			FS_SIZE=$(( $(btrfs inspect-internal min-dev-size $FP_MNT_TMP) + 4*1024**2 )) # bytes? + 4 MiB buffer
+			G_EXEC_OUTPUT=1 G_EXEC btrfs filesystem resize $FS_SIZE $FP_MNT_TMP
+			G_EXEC fstrim $FP_MNT_TMP
+			G_EXEC sync
+			G_EXEC sleep 1 # Give the system 1 second to avoid "mount is busy"
+			G_EXEC umount $FP_MNT_TMP
+			G_EXEC rmdir $FP_MNT_TMP
+			FS_SIZE=$(( $FS_SIZE / 512 )) # bytes => 512 byte sectors
+		fi
+
+		Run_fsck
 
 		# Only resize partition when new size would be less than current size
-		if (( $(</sys/class/block/"${FP_ROOT_DEV##*/}"/size) > $FS_SIZE ))
+		if [[ $ROOT_FS_TYPE != 'f2fs' ]] && (( $(</sys/class/block/"${FP_ROOT_DEV##*/}"/size) > $FS_SIZE ))
 		then
 			G_DIETPI-NOTIFY 2 "Shrinking root partition to: $(( $FS_SIZE / 2048 + 1 )) MiB"
 			G_EXEC_OUTPUT=1 G_EXEC eval "sfdisk --no-reread --no-tell-kernel -fN${FP_ROOT_DEV: -1} '$FP_SOURCE' <<< ',$FS_SIZE'"
 			G_EXEC partprobe "$FP_SOURCE"
 			G_EXEC partx -u "$FP_SOURCE"
 		fi
 
-		G_DIETPI-NOTIFY 2 'Overriding root partition free space with zeros to purge removed data and allow further archive size reduction...'
-		G_EXEC_OUTPUT=1 G_EXEC zerofree -v "$FP_ROOT_DEV"
-		G_EXEC sync
-
 		# Finished: Derive final image size from last partition end + failsafe buffer
-		G_EXEC partprobe "$FP_SOURCE"
-		G_EXEC partx -u "$FP_SOURCE"
 		IMAGE_SIZE=$(( ( $(sfdisk -qlo End "$FP_SOURCE" | tail -1) + 1 ) * 512 )) # 512 byte sectors => Byte
 		IMAGE_SIZE=$(( $IMAGE_SIZE + ( 512 * 256 ) )) # 64 byte for secondary GPT + safety net
 
@@ -461,11 +558,8 @@ _EOF_
 		# - gdisk will correct this
 		if [[ $PART_TABLE_TYPE == 'gpt' ]]
 		then
-			local loop=$(losetup -f)
-			G_EXEC losetup "$loop" "$OUTPUT_IMG_NAME.$OUTPUT_IMG_EXT"			
-			G_EXEC_DESC='Reapplying GPT partition backup table fix' G_EXEC_OUTPUT=1 G_EXEC sgdisk -e "$loop"
+			G_EXEC_DESC='Reapplying GPT partition backup table fix' G_EXEC_OUTPUT=1 G_EXEC sgdisk -e "$OUTPUT_IMG_NAME.$OUTPUT_IMG_EXT"
 			G_EXEC sync
-			G_EXEC losetup -d "$loop"
 		fi
 
 		# Generate hashes: MD5, SHA1, SHA256

CHANGELOG.txt

@@ -2,6 +2,8 @@ v7.1
 (2021-03-XX)
 
 Changes:
+- DietPi-FS_partition_resize | Added support to automatically resize F2FS and Btrfs filesystems on first boot.
+- DietPi-Drive_Manager | Added support for resizing F2FS and Btrfs filesystems as well as format- and filesystem check & repair support for XFS filesystems.
 - DietPi-Config | Added a safe overclocking profile for RPi 3+ models. Many thanks to @lone for doing long-term stability tests and reporting back the result: https://dietpi.com/phpbb/viewtopic.php?p=32285#p32285
 
 Fixes:
@@ -10,6 +12,7 @@ Fixes:
 As always, many smaller code performance and stability improvements, visual and spelling fixes have been done, too much to list all of them here. Check out all code changes of this release on GitHub: https://github.com/MichaIng/DietPi/pull/XXXX
 
 Known/Outstanding Issues:
+- DietPi-Drive_Manager | Fixed detection and visualisation of loop devices in menu.
 - DietPi-Config | Enabling WiFi + Ethernet adapters, both on different subnets, breaks WiFi connection in some cases: https://github.com/MichaIng/DietPi/issues/2103
 - DietPi-Software | MATE desktop: When logging in as root, desktop items and right-click context menu is missing: https://github.com/MichaIng/DietPi/issues/3160
 - DietPi-Software | Sonarr/Radarr/Mono: With current Mono version 6, import to a file system without UNIX permissions support (exFAT, FAT32/vfat, CIFS mounts and NTFS without "permissions" option) fails, regardless of user/umask mount options: https://github.com/MichaIng/DietPi/issues/3179