SPS · log · files · refs · tags · releases · people · languages · GitHub · commit
68aff5d
setup: seed live recipes and speed src indexing
DistroDeveloperParents
Stat
README.md | 5 +- bin/mkiso | 26 ++++-- bin/setup | 131 +++++++++++++++++++++++++++ bin/src | 40 +++++++-- lib/common.sh | 57 ++++++++++++ lib/mkiso/initrd-init | 1 + lib/recipe.awk | 192 ++++++++++++++++++++++++++++------------ man/mkiso.1 | 4 + man/setup.1 | 14 +++ man/src.1 | 9 +- tests/test_definition_digest.sh | 24 ++++- tests/test_mkiso.sh | 4 + tests/test_recipe_contract.sh | 59 ++++++++++++ tests/test_setup.sh | 78 ++++++++++++++++ 14 files changed, 563 insertions(+), 81 deletions(-)
Diff
diff --git a/README.md b/README.md
index 525b67a..52d02c5 100644
--- a/README.md
+++ b/README.md
@@ -181,8 +181,9 @@ SSH, and more. `setup --list-sets` prints them. There is also an extra-package
checklist (`--extra`) plus locale and login-shell screens.
The installer writes hostname, timezone, keymap, and a first user into the
-target, then runs `src update` and `sget install`. Guided partitioning is
-optional and waits until the last confirmation. Choose systemd or OpenRC
+target, then copies bundled live recipe trees (when present), runs
+`src update`, and `sget install`. Guided partitioning is optional and waits
+until the last confirmation. Choose systemd or OpenRC
as PID 1 (`--init`); those packages conflict. `grub-install` still needs
`--install-bootloader` and `--disk`.
diff --git a/bin/mkiso b/bin/mkiso
index d8265df..cc6c7cf 100755
--- a/bin/mkiso
+++ b/bin/mkiso
@@ -236,10 +236,12 @@ mkiso_copy_path()
mkdir -p "$(dirname "$mkiso_dest")" || return 0
if [ -d "$mkiso_src" ] && [ ! -L "$mkiso_src" ]; then
mkdir -p "$mkiso_dest"
- cp -a "$mkiso_src"/. "$mkiso_dest"/ 2>/dev/null ||
+ cp -a --reflink=auto "$mkiso_src"/. "$mkiso_dest"/ 2>/dev/null ||
+ cp -a "$mkiso_src"/. "$mkiso_dest"/ 2>/dev/null ||
cp -R "$mkiso_src"/. "$mkiso_dest"/ 2>/dev/null || :
else
- cp -a "$mkiso_src" "$mkiso_dest" 2>/dev/null ||
+ cp -a --reflink=auto "$mkiso_src" "$mkiso_dest" 2>/dev/null ||
+ cp -a "$mkiso_src" "$mkiso_dest" 2>/dev/null ||
cp "$mkiso_src" "$mkiso_dest" 2>/dev/null || :
if [ -L "$mkiso_src" ]; then
mkiso_link=$(readlink "$mkiso_src") || return 0
@@ -348,7 +350,8 @@ mkiso_close_libs()
done
[ -n "$mkiso_find" ] || return 0
# shellcheck disable=SC2086
- find $mkiso_find -type f -print 2>/dev/null |
+ find $mkiso_find -type f \( -perm -0100 -o -perm -0010 -o -perm -0001 \) \
+ -print 2>/dev/null |
while IFS= read -r mkiso_elf || [ -n "$mkiso_elf" ]; do
[ -n "$mkiso_elf" ] || continue
# Scripts and data are not DT_NEEDED sources.
@@ -472,6 +475,7 @@ mkiso_copy_plasma()
while IFS= read -r mkiso_tree || [ -n "$mkiso_tree" ]; do
case $mkiso_tree in ''|\#*) continue ;; esac
[ -e "$mkiso_tree" ] || continue
+ printf '%s: copying %s\n' "$sps_prog" "$mkiso_tree" >&2
mkiso_copy_path "$mkiso_tree"
done <"$mkiso_libdir/plasma-trees"
fi
@@ -798,15 +802,21 @@ cp -a "$mkiso_kernel" "$mkiso_iso/live/vmlinuz"
printf '%s: writing squashfs (zstd)\n' "$sps_prog" >&2
# zstd mounts in seconds; xz is what made QEMU sit on loop0 while the
-# 4G Plasma root decompressed. Level 5 is a live-ISO default: fast
-# to build, fast to boot, still smaller than gzip.
+# 4G Plasma root decompressed. Level 3 is a live-ISO default: fast
+# to build, fast to boot, still much smaller than gzip.
+mkiso_squash_jobs=$(nproc 2>/dev/null || printf 2)
+case $mkiso_squash_jobs in
+ ''|*[!0-9]*|0) mkiso_squash_jobs=2 ;;
+esac
if ! mksquashfs "$mkiso_live" "$mkiso_iso/live/filesystem.squashfs" \
- -comp zstd -Xcompression-level 5 -b 262144 -noappend -no-progress \
- -e boot
+ -comp zstd -Xcompression-level 3 -b 262144 -noappend -no-progress \
+ -processors "$mkiso_squash_jobs" \
+ -e boot tmp var/tmp var/cache
then
printf '%s: zstd squashfs failed, trying xz\n' "$sps_prog" >&2
mksquashfs "$mkiso_live" "$mkiso_iso/live/filesystem.squashfs" \
- -comp xz -noappend -no-progress -e boot ||
+ -comp xz -noappend -no-progress -processors "$mkiso_squash_jobs" \
+ -e boot tmp var/tmp var/cache ||
sps_die "$SPS_EX_BUILD" "mksquashfs failed"
fi
diff --git a/bin/setup b/bin/setup
index ea6f2b8..3be9cd8 100755
--- a/bin/setup
+++ b/bin/setup
@@ -17,6 +17,14 @@ if ! . "$SPS_LIBDIR/common.sh"; then
printf '%s\n' "setup: cannot load $SPS_LIBDIR/common.sh" >&2
exit 1
fi
+# Remember caller-supplied paths. sps_load_config exports defaults for the
+# running system; after --target those must not leak into src/sget.
+[ "${SPS_DB+x}" = x ] && setup_keep_db=1 || setup_keep_db=0
+[ "${SPS_CACHE+x}" = x ] && setup_keep_cache=1 || setup_keep_cache=0
+[ "${SPS_BUILD+x}" = x ] && setup_keep_build=1 || setup_keep_build=0
+[ "${SPS_REPO_ROOT+x}" = x ] && setup_keep_repo_root=1 || setup_keep_repo_root=0
+[ "${SPS_CONFIG+x}" = x ] && setup_keep_config=1 || setup_keep_config=0
+[ "${SPS_REPOS_CONFIG+x}" = x ] && setup_keep_repos=1 || setup_keep_repos=0
sps_load_config || exit $?
setup_usage()
@@ -1304,6 +1312,9 @@ setup_apply_config()
fi
setup_write_answers_file "$setup_etc/sps/setup-answers"
+ # Installed-system paths. src update during setup used $setup_root
+ # prefixes so it could index the copied trees before first boot.
+ setup_write_repos_conf ""
if [ -n "$setup_user" ]; then
setup_home=/home/$setup_user
@@ -1402,12 +1413,122 @@ EOF
fi
}
+# Copy bundled live recipe trees into the target and write repos.conf so
+# src update indexes local dir repos. Live ISOs already have those trees
+# at /usr/src/sps/{core,extra}; cloning GitHub during setup is what made
+# "updating source repositories" hang or fail offline.
+setup_tree_has_recipe()
+{
+ setup_probe=$1
+ [ -d "$setup_probe" ] || return 1
+ find "$setup_probe" \( -type d \( -name .git -o -name files -o \
+ -name patches -o -name hooks \) -prune \) -o \
+ -type f -name recipe -print | {
+ IFS= read -r setup_probe_recipe || true
+ [ -n "$setup_probe_recipe" ]
+ }
+}
+
+setup_seed_one_tree()
+{
+ setup_seed_name=$1
+ setup_seed_src=$setup_live_root/usr/src/sps/$setup_seed_name
+ setup_seed_dest=$setup_root/usr/src/sps/$setup_seed_name
+ setup_tree_has_recipe "$setup_seed_src" || return 1
+ setup_seed_src_abs=$(CDPATH= cd -P "$setup_seed_src" 2>/dev/null &&
+ pwd -P) || return 1
+ if [ -d "$setup_seed_dest" ]; then
+ setup_seed_dest_abs=$(CDPATH= cd -P "$setup_seed_dest" \
+ 2>/dev/null && pwd -P) || setup_seed_dest_abs=
+ if [ -n "$setup_seed_dest_abs" ] &&
+ [ "$setup_seed_src_abs" = "$setup_seed_dest_abs" ]; then
+ return 0
+ fi
+ if setup_tree_has_recipe "$setup_seed_dest"; then
+ return 0
+ fi
+ fi
+ printf '%s: copying live %s recipes into the target\n' \
+ "$sps_prog" "$setup_seed_name" >&2
+ mkdir -p "$setup_seed_dest" || return 1
+ cp -a "$setup_seed_src"/. "$setup_seed_dest"/ || return 1
+ rm -rf "$setup_seed_dest/.git"
+ return 0
+}
+
+setup_write_repos_conf()
+{
+ setup_repos_prefix=$1
+ setup_repos_core=/usr/src/sps/core
+ setup_repos_extra=/usr/src/sps/extra
+ if [ -n "$setup_repos_prefix" ]; then
+ setup_repos_core=$setup_repos_prefix$setup_repos_core
+ setup_repos_extra=$setup_repos_prefix$setup_repos_extra
+ fi
+ {
+ if [ "${setup_have_core:-0}" -eq 1 ]; then
+ printf 'dir core %s 100\n' "$setup_repos_core"
+ else
+ printf 'git core https://github.com/RobertFlexx/sps-core.git 100\n'
+ fi
+ if [ "${setup_have_extra:-0}" -eq 1 ]; then
+ printf 'dir extra %s 80\n' "$setup_repos_extra"
+ else
+ printf 'git extra https://github.com/RobertFlexx/sps-extra.git 80\n'
+ fi
+ } >"$setup_root/etc/sps/repos.conf" ||
+ setup_fail "$SPS_EX_PERMISSION" "cannot write target repos.conf"
+}
+
+setup_seed_recipe_trees()
+{
+ setup_live_root=${SPS_LIVE_ROOT:-/}
+ case $setup_live_root in
+ /*) ;;
+ *)
+ setup_fail "$SPS_EX_USAGE" \
+ "SPS_LIVE_ROOT must be an absolute path: $setup_live_root"
+ ;;
+ esac
+ [ "$setup_live_root" = / ] || setup_live_root=${setup_live_root%/}
+
+ mkdir -p "$setup_root/etc/sps" "$setup_root/usr/src/sps" ||
+ setup_fail "$SPS_EX_PERMISSION" "cannot create target repository directories"
+
+ setup_have_core=0
+ setup_have_extra=0
+ if setup_seed_one_tree core; then
+ setup_have_core=1
+ fi
+ if setup_seed_one_tree extra; then
+ setup_have_extra=1
+ fi
+
+ # Paths must exist on this system during src update. apply_config
+ # rewrites them to /usr/src/sps/... for the installed machine.
+ setup_write_repos_conf "$setup_root"
+
+ if [ "$setup_have_core" -eq 1 ] && [ "$setup_have_extra" -eq 1 ]; then
+ printf '%s: using bundled recipes; src update indexes them locally\n' \
+ "$sps_prog" >&2
+ else
+ printf '%s: missing bundled recipes; src update will clone from the network\n' \
+ "$sps_prog" >&2
+ fi
+}
+
# Packages first, then hostname/user so filesystem's passwd is already present.
setup_root=${setup_target%/}
mkdir -p "$setup_root" ||
setup_fail "$SPS_EX_PERMISSION" "cannot create target: $setup_root"
export SPS_ROOT=$setup_root
+[ "$setup_keep_db" -eq 1 ] || unset SPS_DB
+[ "$setup_keep_cache" -eq 1 ] || unset SPS_CACHE
+[ "$setup_keep_build" -eq 1 ] || unset SPS_BUILD
+[ "$setup_keep_repo_root" -eq 1 ] || unset SPS_REPO_ROOT
+[ "$setup_keep_config" -eq 1 ] || unset SPS_CONFIG
+[ "$setup_keep_repos" -eq 1 ] || unset SPS_REPOS_CONFIG
if [ "$setup_kernel" = custom ] && [ -n "$setup_kernel_config" ]; then
mkdir -p "$setup_root/etc/sps"
@@ -1415,6 +1536,8 @@ if [ "$setup_kernel" = custom ] && [ -n "$setup_kernel_config" ]; then
setup_fail "$SPS_EX_PERMISSION" "cannot copy kernel config"
fi
+setup_seed_recipe_trees
+
if [ "$setup_do_update" -eq 1 ]; then
printf '%s: updating source repositories\n' "$sps_prog" >&2
src update || setup_fail "$SPS_EX_NETWORK" "src update failed"
@@ -1437,6 +1560,14 @@ sget install $setup_pkgs ||
# Hostname, user, and notes land after packages so filesystem's passwd wins.
setup_apply_config
+# On a live ISO, rewrite of repos.conf pointed at /usr/src/sps. Rebuild
+# the index so the installed system does not keep /mnt-prefixed recipe paths.
+if [ "$setup_do_update" -eq 1 ] && [ -f /etc/sps/live ]; then
+ printf '%s: reindexing recipes for the installed system\n' "$sps_prog" >&2
+ src update || printf '%s: installed-system reindex failed; run src update after boot\n' \
+ "$sps_prog" >&2
+fi
+
if [ "$setup_flatpak" = yes ]; then
setup_fp=$setup_root/usr/bin/flatpak
if [ -x "$setup_fp" ]; then
diff --git a/bin/src b/bin/src
index c7a8531..8d60b24 100755
--- a/bin/src
+++ b/bin/src
@@ -47,7 +47,7 @@ src_cleanup_update()
if [ -n "${src_work:-}" ]; then
rm -f "$src_work/repos.raw" "$src_work/repos" \
"$src_work/recipes.unsorted" "$src_work/recipes" \
- "$src_work/packages.index"
+ "$src_work/digests" "$src_work/packages.index"
rmdir "$src_work" 2>/dev/null || :
src_work=
fi
@@ -83,12 +83,17 @@ src_sync_git()
src_git_parent=${src_git_checkout%/*}
[ -n "$src_git_parent" ] || src_git_parent=/
sps_mkdir "$src_git_parent"
+ GIT_TERMINAL_PROMPT=0
+ export GIT_TERMINAL_PROMPT
if [ ! -e "$src_git_checkout" ] && [ ! -L "$src_git_checkout" ]; then
src_clone_work=$(sps_mktemp_dir "$src_git_parent" \
"clone-$src_git_name")
src_clone_checkout=$src_clone_work/repository
- if ! git clone -- "$src_git_source" "$src_clone_checkout"; then
+ # Depth 1 is enough for recipes. Later fetches stay full so
+ # fast-forward and divergence checks keep working.
+ if ! git clone --depth 1 --single-branch -- \
+ "$src_git_source" "$src_clone_checkout"; then
sps_die "$SPS_EX_NETWORK" \
"cannot clone repository '$src_git_name'"
fi
@@ -216,6 +221,8 @@ src_update()
fi
: > "$src_work/packages.index" ||
sps_die "$SPS_EX_PERMISSION" "cannot create repository index"
+ : > "$src_work/digests" ||
+ sps_die "$SPS_EX_PERMISSION" "cannot create recipe digest map"
src_default_arch=$SPS_ARCH
src_repo_count=0
@@ -268,6 +275,12 @@ src_update()
break
fi
+ : > "$src_work/digests" || {
+ printf "%s: cannot create recipe digest map\n" "$sps_prog" >&2
+ src_failed=1
+ break
+ }
+ src_repo_packages=0
while IFS= read -r src_recipe || [ -n "$src_recipe" ]
do
[ -n "$src_recipe" ] || continue
@@ -297,21 +310,30 @@ src_update()
src_failed=1
break
fi
+ printf '%s\t%s\n' "$src_recipe" "$src_digest" \
+ >> "$src_work/digests" || {
+ printf "%s: cannot record recipe digest: %s\n" \
+ "$sps_prog" "$src_recipe" >&2
+ src_failed=1
+ break
+ }
+ src_repo_packages=$((src_repo_packages + 1))
+ done < "$src_work/recipes"
+ [ "$src_failed" -eq 0 ] || break
+ if [ "$src_repo_packages" -gt 0 ]; then
if ! awk -v recipe_repo="$src_repo_name" \
-v recipe_priority="$src_repo_priority" \
- -v recipe_path="$src_recipe" \
- -v recipe_sha256="$src_digest" \
+ -v recipe_digest_map="$src_work/digests" \
+ -v recipe_file_list="$src_work/recipes" \
-v recipe_default_arch="$src_default_arch" \
- -f "$SPS_LIBDIR/recipe.awk" "$src_recipe" \
+ -f "$SPS_LIBDIR/recipe.awk" \
>> "$src_work/packages.index"; then
src_failed=1
break
fi
- src_package_count=$((src_package_count + 1))
- done < "$src_work/recipes"
-
- [ "$src_failed" -eq 0 ] || break
+ fi
+ src_package_count=$((src_package_count + src_repo_packages))
src_repo_count=$((src_repo_count + 1))
done < "$src_work/repos"
diff --git a/lib/common.sh b/lib/common.sh
index fa164fe..283c5f8 100644
--- a/lib/common.sh
+++ b/lib/common.sh
@@ -108,6 +108,27 @@ sps_sha256()
fi
}
+# Same digest as sps_sha256, from stdin. Used for the recipe-only definition
+# manifest so src update does not create a throwaway directory per package.
+sps_sha256_stream()
+{
+ if command -v sha256sum >/dev/null 2>&1; then
+ sps_hash_line=$(sha256sum) || return $?
+ printf '%s\n' "${sps_hash_line%% *}"
+ elif command -v sha256 >/dev/null 2>&1; then
+ sha256 -q
+ elif command -v shasum >/dev/null 2>&1; then
+ sps_hash_line=$(shasum -a 256) || return $?
+ printf '%s\n' "${sps_hash_line%% *}"
+ elif command -v openssl >/dev/null 2>&1; then
+ sps_hash_line=$(openssl dgst -sha256) || return $?
+ printf '%s\n' "${sps_hash_line##* }"
+ else
+ sps_die "$SPS_EX_CHECKSUM" \
+ "no SHA-256 utility found (tried sha256sum, sha256, shasum, openssl)"
+ fi
+}
+
# Hash every input which makes up one package definition. The manifest uses
# only fixed, package-relative names, so moving an unchanged package directory
# does not change its digest. files/ and patches/ are copied recursively by
@@ -142,6 +163,42 @@ sps_definition_sha256()
return 1
fi
+ # Recipe-only packages produce a two-line manifest. Hash that without a
+ # workspace directory; src update indexes hundreds of these.
+ sps_definition_need_support=0
+ for sps_definition_support in files patches hooks; do
+ if [ -e "$sps_definition_dir/$sps_definition_support" ] ||
+ [ -L "$sps_definition_dir/$sps_definition_support" ]; then
+ sps_definition_need_support=1
+ break
+ fi
+ done
+ if [ "$sps_definition_need_support" -eq 0 ]; then
+ sps_definition_hash=$(sps_sha256 "$sps_definition_recipe") || {
+ sps_warn 'cannot hash package definition recipe'
+ return 1
+ }
+ sps_definition_hash=$(printf '%s\n' "$sps_definition_hash" |
+ LC_ALL=C tr 'ABCDEF' 'abcdef') || return 1
+ case $sps_definition_hash in
+ ''|*[!0123456789abcdef]*) return 1 ;;
+ esac
+ [ "${#sps_definition_hash}" -eq 64 ] || return 1
+ sps_definition_out=$(printf 'sps-package-definition\t1\nfile\t-\t%s\trecipe\n' \
+ "$sps_definition_hash" | sps_sha256_stream) || {
+ sps_warn 'cannot hash package definition manifest'
+ return 1
+ }
+ sps_definition_out=$(printf '%s\n' "$sps_definition_out" |
+ LC_ALL=C tr 'ABCDEF' 'abcdef') || return 1
+ case $sps_definition_out in
+ ''|*[!0123456789abcdef]*) return 1 ;;
+ esac
+ [ "${#sps_definition_out}" -eq 64 ] || return 1
+ printf '%s\n' "$sps_definition_out"
+ return 0
+ fi
+
sps_definition_tmp=$(mktemp -d \
"${TMPDIR:-/tmp}/.sps-definition.XXXXXX" 2>/dev/null) || {
sps_warn 'cannot create package definition hash workspace'
diff --git a/lib/mkiso/initrd-init b/lib/mkiso/initrd-init
index e97babf..9cb94d2 100755
--- a/lib/mkiso/initrd-init
+++ b/lib/mkiso/initrd-init
@@ -75,6 +75,7 @@ done
[ -n "$found" ] || rescue "mkiso: cannot find the SPS Linux live filesystem on a CD/USB."
+say "SPS: found live image on $found"
say "SPS: mounting live filesystem..."
"$bb" mount -t squashfs -o ro /mnt/iso/live/filesystem.squashfs /mnt/ro ||
rescue "mkiso: squashfs mount failed"
diff --git a/lib/recipe.awk b/lib/recipe.awk
index 6ae3952..f529eaa 100644
--- a/lib/recipe.awk
+++ b/lib/recipe.awk
@@ -1,5 +1,7 @@
-# Parse one recipe into a repository index record. Build command fields are
-# recognized here, but never executed.
+# Parse one or more recipes into repository index records. Build command
+# fields are recognized here, but never executed. src update feeds a whole
+# repository in one invocation; src check and the test parser still pass a
+# single file.
function recipe_trim(value) {
sub(/^[ \t]+/, "", value)
@@ -14,7 +16,13 @@ function recipe_stderr(message, command) {
}
function recipe_error(message) {
- recipe_stderr("recipe: " FILENAME ":" FNR ": " message)
+ recipe_where = FILENAME
+ recipe_at = FNR
+ if (recipe_file != "")
+ recipe_where = recipe_file
+ if (recipe_emitting)
+ recipe_at = recipe_last_fnr
+ recipe_stderr("recipe: " recipe_where ":" recipe_at ": " message)
recipe_failed = 1
}
@@ -95,64 +103,21 @@ function recipe_valid_dependency_list(label, value, count, items, i, item,
return 1
}
-function recipe_process(line, key, value, separator) {
- line = recipe_trim(line)
- if (line == "" || line ~ /^#/)
- return
-
- separator = match(line, /[ \t]/)
- if (!separator) {
- recipe_error("expected 'key value' record")
- return
- }
- key = substr(line, 1, RSTART - 1)
- value = recipe_trim(substr(line, RSTART + RLENGTH))
- if (value == "") {
- recipe_error("empty value for '" key "'")
- return
- }
-
- if (key == "name" || key == "version" || key == "release" ||
- key == "arch" || key == "description") {
- recipe_set_once(key, value)
- } else if (key == "depend" || key == "builddep" || key == "optional" ||
- key == "conflict") {
- recipe_append_dependency(key, value)
- } else if (key == "source") {
- recipe_source_count++
- } else if (key == "hash") {
- recipe_hash_count++
- recipe_hash[recipe_hash_count] = value
- } else if (key == "install") {
- recipe_install_count++
- } else if (key == "prepare" || key == "configure" || key == "build") {
- # mkpkg executes phase records later; indexing only validates their syntax.
- } else {
- recipe_error("unknown field '" key "'")
- }
-}
-
-{
- recipe_line = $0
- sub(/\r$/, "", recipe_line)
- if (recipe_continuing)
- recipe_logical = recipe_logical recipe_line
- else
- recipe_logical = recipe_line
-
- if (recipe_logical ~ /\\[ \t]*$/) {
- sub(/\\[ \t]*$/, "", recipe_logical)
- recipe_logical = recipe_logical " "
- recipe_continuing = 1
- next
- }
-
+function recipe_reset() {
+ delete recipe_seen
+ delete recipe_value
+ delete recipe_hash
+ recipe_source_count = 0
+ recipe_hash_count = 0
+ recipe_install_count = 0
recipe_continuing = 0
- recipe_process(recipe_logical)
recipe_logical = ""
+ recipe_failed = 0
+ recipe_emitting = 0
}
-END {
+function recipe_finish() {
+ recipe_emitting = 1
if (recipe_continuing)
recipe_error("unterminated line continuation")
@@ -226,8 +191,11 @@ END {
if (length(recipe_sha256) != 64 || recipe_sha256 !~ /^[[:xdigit:]]+$/)
recipe_error("invalid recipe SHA-256")
- if (recipe_failed)
- exit 2
+ recipe_emitting = 0
+ if (recipe_failed) {
+ recipe_any_failed = 1
+ return
+ }
OFS = "\t"
print recipe_value["name"], recipe_value["version"],
@@ -237,3 +205,109 @@ END {
recipe_path, recipe_value["description"], tolower(recipe_sha256),
recipe_value["conflict"]
}
+
+function recipe_process(line, key, value, separator) {
+ line = recipe_trim(line)
+ if (line == "" || line ~ /^#/)
+ return
+
+ separator = match(line, /[ \t]/)
+ if (!separator) {
+ recipe_error("expected 'key value' record")
+ return
+ }
+ key = substr(line, 1, RSTART - 1)
+ value = recipe_trim(substr(line, RSTART + RLENGTH))
+ if (value == "") {
+ recipe_error("empty value for '" key "'")
+ return
+ }
+
+ if (key == "name" || key == "version" || key == "release" ||
+ key == "arch" || key == "description") {
+ recipe_set_once(key, value)
+ } else if (key == "depend" || key == "builddep" || key == "optional" ||
+ key == "conflict") {
+ recipe_append_dependency(key, value)
+ } else if (key == "source") {
+ recipe_source_count++
+ } else if (key == "hash") {
+ recipe_hash_count++
+ recipe_hash[recipe_hash_count] = value
+ } else if (key == "install") {
+ recipe_install_count++
+ } else if (key == "prepare" || key == "configure" || key == "build") {
+ # mkpkg executes phase records later; indexing only validates their syntax.
+ } else {
+ recipe_error("unknown field '" key "'")
+ }
+}
+
+BEGIN {
+ if (recipe_digest_map != "") {
+ while ((getline recipe_dline < recipe_digest_map) > 0) {
+ recipe_n = split(recipe_dline, recipe_df, "\t")
+ if (recipe_n >= 2)
+ recipe_digest[recipe_df[1]] = recipe_df[2]
+ }
+ close(recipe_digest_map)
+ }
+ if (recipe_file_list != "") {
+ while ((getline recipe_listed < recipe_file_list) > 0) {
+ if (recipe_listed == "")
+ continue
+ ARGV[ARGC++] = recipe_listed
+ }
+ close(recipe_file_list)
+ if (ARGC == 1) {
+ recipe_skip_end = 1
+ exit 0
+ }
+ }
+}
+
+FNR == 1 {
+ if (recipe_active) {
+ recipe_finish()
+ if (recipe_any_failed)
+ exit 2
+ }
+ recipe_reset()
+ recipe_file = FILENAME
+ recipe_last_fnr = FNR
+ if (recipe_digest_map != "") {
+ recipe_path = FILENAME
+ recipe_sha256 = recipe_digest[FILENAME]
+ }
+ recipe_active = 1
+}
+
+{
+ recipe_last_fnr = FNR
+ recipe_line = $0
+ sub(/\r$/, "", recipe_line)
+ if (recipe_continuing)
+ recipe_logical = recipe_logical recipe_line
+ else
+ recipe_logical = recipe_line
+
+ if (recipe_logical ~ /\\[ \t]*$/) {
+ sub(/\\[ \t]*$/, "", recipe_logical)
+ recipe_logical = recipe_logical " "
+ recipe_continuing = 1
+ next
+ }
+
+ recipe_continuing = 0
+ recipe_process(recipe_logical)
+ recipe_logical = ""
+}
+
+END {
+ if (recipe_skip_end)
+ exit 0
+ if (!recipe_any_failed && recipe_active)
+ recipe_finish()
+ if (recipe_any_failed)
+ exit 2
+}
diff --git a/man/mkiso.1 b/man/mkiso.1
index f20ebf9..4bec931 100644
--- a/man/mkiso.1
+++ b/man/mkiso.1
@@ -31,6 +31,10 @@ only with its library closure, including the real
file behind the soname symlink.
The squashfs uses zstd so the live root mounts quickly; xz made QEMU
sit on the loop0 probe while a multi-gigabyte Plasma image decompressed.
+Compression level 3 keeps ISO creation short without changing the boot
+path. Directory copies use
+.B cp --reflink=auto
+when the host supports it.
.B --layout
writes the live root only, for tests and inspection.
.PP
diff --git a/man/setup.1 b/man/setup.1
index d86c7e4..54d7abc 100644
--- a/man/setup.1
+++ b/man/setup.1
@@ -46,6 +46,19 @@ selects PID 1;
will not install both.
Live images prefer systemd. If the build host has no systemd binaries,
busybox init runs instead and respawns gettys on tty1 and ttyS0.
+.PP
+On a live ISO,
+.B setup
+copies the bundled
+.I /usr/src/sps/core
+and
+.I /usr/src/sps/extra
+recipe trees into the target and indexes those
+.B dir
+repositories. That does not need a network.
+.B src update
+only clones GitHub when those trees are missing, which is why a slim
+image fails offline.
.SH PROFILES
.TP
.B minimal
@@ -154,6 +167,7 @@ Skip
.B src update
before
.B sget install .
+Live images still copy bundled recipe trees into the target.
.TP
.B --non-interactive
Never start
diff --git a/man/src.1 b/man/src.1
index 7e5ca88..668b5cf 100644
--- a/man/src.1
+++ b/man/src.1
@@ -19,10 +19,14 @@ checkouts together with configured local directory repositories.
.SH COMMANDS
.TP
.B update
-Clone absent Git repositories, fetch existing clean Git repositories,
+Clone absent Git repositories with
+.B --depth 1
+(later fetches stay full), fetch existing clean Git repositories,
fast-forward safe tracking branches, scan configured trees, validate recipes
and the selected dependency graph, and atomically replace the local package
-index. Invalid input leaves the previous index in place.
+index. Invalid input leaves the previous index in place. Recipe-only packages
+are hashed without a temporary directory; a repository is parsed in one AWK
+process rather than one process per recipe.
.TP
.BI check " recipe ..."
Validate each recipe with the same parser used by
@@ -72,6 +76,7 @@ refuses tracked or untracked local changes before fetching. It never runs
or an implicit checkout. A clean tracking branch is fast-forwarded only when
safe. Detached heads and branches without an origin upstream remain pinned;
locally ahead branches remain unchanged; divergent branches are refused.
+Git prompts are disabled so a missing network fails instead of hanging.
.SH FILES
.TP
.B /etc/sps/sps.conf
diff --git a/tests/test_definition_digest.sh b/tests/test_definition_digest.sh
index 726fea2..a3e5248 100644
--- a/tests/test_definition_digest.sh
+++ b/tests/test_definition_digest.sh
@@ -99,6 +99,28 @@ newline=${newline%X}
printf '%s\n' bad >"$tmp/copy/files/bad${newline}name"
expect_invalid_definition newline
grep -F 'tab or newline' "$tmp/newline.err" >/dev/null ||
- fail 'newline-containing path rejection was not explained'
+ fail 'newline-containing path rejection was not explained'
+
+# Recipe-only packages must hash the same two-line manifest the slow path
+# would write, without creating a workspace directory.
+mkdir -p "$tmp/recipe-only"
+printf '%s\n' 'name recipe-only' 'version 1.0' 'release 1' \
+ 'install true' >"$tmp/recipe-only/recipe"
+only_digest=$(sps_definition_sha256 "$tmp/recipe-only/recipe") ||
+ fail 'could not hash a recipe-only package'
+recipe_hash=$(sps_sha256 "$tmp/recipe-only/recipe") ||
+ fail 'could not hash recipe-only recipe file'
+recipe_hash=$(printf '%s\n' "$recipe_hash" | LC_ALL=C tr 'ABCDEF' 'abcdef')
+manifest_digest=$(printf 'sps-package-definition\t1\nfile\t-\t%s\trecipe\n' \
+ "$recipe_hash" | sps_sha256_stream) ||
+ fail 'could not hash recipe-only manifest'
+manifest_digest=$(printf '%s\n' "$manifest_digest" |
+ LC_ALL=C tr 'ABCDEF' 'abcdef')
+[ "$only_digest" = "$manifest_digest" ] ||
+ fail 'recipe-only digest does not match the two-line manifest'
+printf '%s\n' 'name recipe-only' 'version 1.1' 'release 1' \
+ 'install true' >"$tmp/recipe-only/recipe"
+[ "$(sps_definition_sha256 "$tmp/recipe-only/recipe")" != "$only_digest" ] ||
+ fail 'recipe-only digest ignored recipe content'
printf '%s\n' 'package definition digest tests passed'
diff --git a/tests/test_mkiso.sh b/tests/test_mkiso.sh
index 0a2a37f..165986b 100755
--- a/tests/test_mkiso.sh
+++ b/tests/test_mkiso.sh
@@ -93,6 +93,10 @@ grep -q dbus-run-session "$layout/etc/sps/live-plasma" ||
grep -q C.UTF-8 "$layout/etc/sps/live-plasma" ||
fail 'live-plasma must set a UTF-8 locale'
grep -q -- '-comp zstd' "$mkiso" || fail 'mkiso must write a zstd squashfs'
+grep -q -- '-Xcompression-level 3' "$mkiso" ||
+ fail 'mkiso squashfs compression should stay fast to build'
+grep -q 'found live image' "$project_dir/lib/mkiso/initrd-init" ||
+ fail 'initrd must say when the ISO is found'
grep -q 'mounting live filesystem' "$project_dir/lib/mkiso/initrd-init" ||
fail 'initrd must print before squashfs mount'
if [ -x /usr/bin/dialog ]; then
diff --git a/tests/test_recipe_contract.sh b/tests/test_recipe_contract.sh
index bdf0b2f..579ec96 100755
--- a/tests/test_recipe_contract.sh
+++ b/tests/test_recipe_contract.sh
@@ -231,3 +231,62 @@ expect_builder_failure bad-hook "unsupported recipe hook 'not-a-hook'" \
"$tmp/bad-hook/recipe"
printf '%s\n' 'test_recipe_contract: ok'
+
+# One AWK process can index a whole repository.
+mkdir -p "$tmp/batch-a" "$tmp/batch-b"
+cat >"$tmp/batch-a/recipe" <<'EOF_BATCH_A'
+name batch-a
+version 1.0
+release 1
+arch any
+install true
+EOF_BATCH_A
+cat >"$tmp/batch-b/recipe" <<'EOF_BATCH_B'
+name batch-b
+version 2.0
+release 1
+arch any
+depend batch-a
+install true
+EOF_BATCH_B
+printf '%s\n' "$tmp/batch-a/recipe" "$tmp/batch-b/recipe" >"$tmp/batch.list"
+{
+ printf '%s\t%s\n' "$tmp/batch-a/recipe" "$recipe_digest"
+ printf '%s\t%s\n' "$tmp/batch-b/recipe" "$recipe_digest"
+} >"$tmp/batch.digests"
+batch_out=$(awk -v recipe_repo=test -v recipe_priority=10 \
+ -v recipe_digest_map="$tmp/batch.digests" \
+ -v recipe_file_list="$tmp/batch.list" \
+ -v recipe_default_arch=testarch \
+ -f "$recipe_parser") || fail 'batch recipe parse failed'
+printf '%s\n' "$batch_out" | awk -F '\t' '
+ $1 == "batch-a" { a = 1 }
+ $1 == "batch-b" && $5 ~ /batch-a/ { b = 1 }
+ END { exit !(a && b) }
+' || fail 'batch recipe parse dropped a package'
+
+mkdir -p "$tmp/batch-bad"
+cat >"$tmp/batch-bad/recipe" <<'EOF_BATCH_BAD'
+name batch-bad
+version 1.0
+release 1
+arch any
+EOF_BATCH_BAD
+printf '%s\n' "$tmp/batch-a/recipe" "$tmp/batch-bad/recipe" \
+ >"$tmp/batch-fail.list"
+{
+ printf '%s\t%s\n' "$tmp/batch-a/recipe" "$recipe_digest"
+ printf '%s\t%s\n' "$tmp/batch-bad/recipe" "$recipe_digest"
+} >"$tmp/batch-fail.digests"
+set +e
+awk -v recipe_repo=test -v recipe_priority=10 \
+ -v recipe_digest_map="$tmp/batch-fail.digests" \
+ -v recipe_file_list="$tmp/batch-fail.list" \
+ -v recipe_default_arch=testarch \
+ -f "$recipe_parser" >"$tmp/batch-fail.out" 2>"$tmp/batch-fail.err"
+batch_st=$?
+set -e
+[ "$batch_st" -eq 2 ] ||
+ fail "batch parse of a bad recipe returned $batch_st instead of 2"
+grep -F "missing required 'install' command" "$tmp/batch-fail.err" >/dev/null ||
+ fail 'batch parse did not report the bad recipe'
diff --git a/tests/test_setup.sh b/tests/test_setup.sh
index c05e992..95c469d 100755
--- a/tests/test_setup.sh
+++ b/tests/test_setup.sh
@@ -107,10 +107,12 @@ cache=$tmp/cache
build=$tmp/build
repo=$tmp/repo
mkdir -p "$root" "$db" "$cache" "$build" "$repo/hello" "$repo/extra-pkg"
+mkdir -p "$tmp/nolive-fixture"
: >"$tmp/sps.conf"
printf 'dir test %s 10\n' "$repo" >"$tmp/repos.conf"
export SPS_ROOT=$root SPS_DB=$db SPS_CACHE=$cache SPS_BUILD=$build
export SPS_CONFIG=$tmp/sps.conf SPS_REPOS_CONFIG=$tmp/repos.conf
+export SPS_LIVE_ROOT=$tmp/nolive-fixture
cat >"$repo/hello/recipe" <<'EOF'
name hello
@@ -148,3 +150,79 @@ grep -q '^tester:x:1000:1000:Test User:/home/tester:/bin/sh$' \
[ -f "$root/root/sps-fstab.txt" ] || fail 'fstab notes missing'
grep -q 'grub-install' "$root/root/sps-bootloader.txt" ||
fail 'bootloader notes omitted grub-install warning'
+
+printf '%s\n' 'setup fixture tests passed'
+
+# Bundled live recipes: copy into the target and index locally (no Git clone).
+live=$tmp/live
+mkdir -p "$live/usr/src/sps/core/hello" "$live/usr/src/sps/extra/placeholder"
+cat >"$live/usr/src/sps/core/hello/recipe" <<'EOF'
+name hello
+version 1.0
+release 1
+arch any
+description bundled hello
+install mkdir -p "$PKG/usr/share/sps-test"
+install printf '%s\n' bundled >"$PKG/usr/share/sps-test/hello"
+EOF
+cat >"$live/usr/src/sps/extra/placeholder/recipe" <<'EOF'
+name placeholder
+version 1.0
+release 1
+arch any
+description bundled extra placeholder
+install true
+EOF
+
+root_seed=$tmp/root-seed
+db_seed=$tmp/db-seed
+cache_seed=$tmp/cache-seed
+build_seed=$tmp/build-seed
+mkdir -p "$root_seed" "$db_seed" "$cache_seed" "$build_seed"
+(
+ unset SPS_ROOT SPS_DB SPS_CACHE SPS_BUILD SPS_CONFIG SPS_REPOS_CONFIG
+ export SPS_LIVE_ROOT=$live
+ export SPS_SETUP_DIR=$project_dir/tests/setup-fixture
+ export SPS_LIBDIR=$project_dir/lib
+ export PATH=$project_dir/bin:$PATH
+ "$setup" --non-interactive --target "$root_seed" --profile tiny \
+ --hostname seeded --timezone UTC --keymap us --user tester \
+ --user-gecos 'Seed User'
+) || fail 'bundled-recipe setup failed'
+
+[ "$(cat "$root_seed/usr/share/sps-test/hello")" = bundled ] ||
+ fail 'bundled hello was not installed'
+[ -f "$root_seed/usr/src/sps/core/hello/recipe" ] ||
+ fail 'live recipe tree was not copied into the target'
+grep -qx 'dir core /usr/src/sps/core 100' "$root_seed/etc/sps/repos.conf" ||
+ fail 'installed repos.conf must use /usr/src/sps paths'
+grep -qx 'dir extra /usr/src/sps/extra 80' "$root_seed/etc/sps/repos.conf" ||
+ fail 'installed repos.conf must keep bundled extra as a dir repo'
+if grep -q 'git ' "$root_seed/etc/sps/repos.conf"; then
+ fail 'bundled live trees must not fall back to git'
+fi
+
+# No bundled trees: repos.conf is git clones, and --no-update must not
+# require the network.
+nolive=$tmp/nolive
+mkdir -p "$nolive"
+root_git=$tmp/root-git
+mkdir -p "$root_git"
+(
+ unset SPS_ROOT SPS_DB SPS_CACHE SPS_BUILD SPS_CONFIG SPS_REPOS_CONFIG
+ export SPS_LIVE_ROOT=$nolive
+ export SPS_SETUP_DIR=$project_dir/tests/setup-fixture
+ export SPS_LIBDIR=$project_dir/lib
+ export PATH=$project_dir/bin:$PATH
+ export SPS_ROOT=$root SPS_DB=$db SPS_CACHE=$cache SPS_BUILD=$build
+ export SPS_CONFIG=$tmp/sps.conf SPS_REPOS_CONFIG=$tmp/repos.conf
+ "$setup" --non-interactive --no-update --target "$root_git" \
+ --profile tiny --hostname gitfallback --timezone UTC \
+ --keymap us --user tester
+) || fail 'git-fallback --no-update setup failed'
+grep -q 'git core https://github.com/RobertFlexx/sps-core.git' \
+ "$root_git/etc/sps/repos.conf" ||
+ fail 'empty live root should write git core'
+grep -q 'git extra https://github.com/RobertFlexx/sps-extra.git' \
+ "$root_git/etc/sps/repos.conf" ||
+ fail 'empty live root should write git extra'