Subject: Re: bug#61884: add an option to du that allows to control which file types are counted

From: Pádraig Brady <[email-masked]>
Date: Mon, 13 Mar 2023 15:26:14 +0000

On 05/03/2023 01:00, Paul Eggert wrote:
> On 2023-03-04 15:33, Christoph Anton Mitterer wrote:
> 
>> But would symlinks (i.e. their length) count for it?
> 
> Sure, because you can read symlinks by using readlink, and that gives
> you their lengths.
> 
> Come to think of it, POSIX specifies st_size only for regular files and
> symlinks among the files you'll find in a directory. So du --apparent
> should count st_size only for these file types; it should ignore st_size
> for other file types unless we know somehow that those sizes make sense
> (which for directories is problematic for the reasons you mention).
> 
> 
>> What about hardlinked files, would they count once or n times?
> 
> That's an independent axis and is handled by -l. Hard links are not a
> file type.
> 
> 
>>                 b      block (buffered) special
>>                 c      character (unbuffered) special
>>                 d      directory
>>                 p      named pipe (FIFO)
>>                 f      regular file
>>                 l      symbolic link
>>                 s      socket
>>                 D      door (Solaris)
> 
> I expect Coreutils's already-existing usable_st_function should tell us
> which types have usable st_size. This will exclude directories, which
> should be the right thing for your use case.
> 
> 
> So I installed the attached patch to fix du --apparent to count sizes
> only when st_size is well-defined. This should address your use case so
> I'm boldly closing the bug report.

The attached adjusts the du/threshold test to pass
by avoiding testing --apparent with dirs

cheers,
Pádraig

From 775a212b6cbf635cd266e473aeaa5eea18ab930c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?P=C3=A1draig=20Brady?= <[email-masked]>
Date: Mon, 13 Mar 2023 15:11:27 +0000
Subject: [PATCH] tests: adjust du test for recent apparent size change

* tests/du/threshold.sh: Directories are assumed to be
of size 0 with --apparent since commit v9.1-187-g110bcd283
so remove --apparent cases from this test.
---
 tests/du/threshold.sh | 105 ++----------------------------------------
 1 file changed, 3 insertions(+), 102 deletions(-)

diff --git a/tests/du/threshold.sh b/tests/du/threshold.sh
index b438d29c9..c112f08a1 100755
--- a/tests/du/threshold.sh
+++ b/tests/du/threshold.sh
@@ -34,9 +34,9 @@ B1=$(stat --format="%B * %b" a/b/1 | xargs expr)
 B2=$(stat --format="%B * %b" a/b/2 | xargs expr)
 B3=$(stat --format="%B * %b" a/b/3 | xargs expr)
 
-Sa=$(stat --format=%s a    )
-Sb=$(stat --format=%s a/b  )
-Sc=$(stat --format=%s a/c  )
+Sa=0 # du always assumes st_size=0 for dirs
+Sb=0
+Sc=0
 S0=$(stat --format=%s a/b/0)
 S1=$(stat --format=%s a/b/1)
 S2=$(stat --format=%s a/b/2)
@@ -46,13 +46,11 @@ Bb0123=$(expr $Bb + $B0 + $B1 + $B2 + $B3)
 Sb0123=$(expr $Sb + $S0 + $S1 + $S2 + $S3)
 
 Bab0123=$(expr $Ba + $Bc + $Bb0123)
-Sab0123=$(expr $Sa + $Sc + $Sb0123)
 
 # Sanity checks
 test $Ba -gt 4 || skip_ "block size of a directory is smaller than 4 bytes"
 test $Bc -gt 4 || skip_ "block size of an empty directory is smaller than 4 \
 bytes"
-test $Sa -gt 4 || skip_ "apparent size of a directory is smaller than 4 bytes"
 test $B1 -gt 4 || skip_ "block size of small file smaller than 4 bytes"
 test $S3 -eq 3 || framework_failure_
 test $S2 -eq 2 || framework_failure_
@@ -62,7 +60,6 @@ test $B0 -eq 0 || skip_ "block size of an empty file unequal Zero"
 # block size of a/b/1 == a/b/2
 test $B1 -eq $B2 || framework_failure_
 # a is bigger than a/b.
-test $Sab0123 -gt $Sb0123 || framework_failure_
 test $Bab0123 -gt $Bb0123 || framework_failure_
 # a/b is bigger than empty a/c.
 test $Sb0123 -gt $Sc || framework_failure_
@@ -138,15 +135,6 @@ dutest ()
 
 # Check numbers around the total size of the main directory 'a'.
 # One byte greater than 'a'.
-s=$(expr $Sab0123 + 1)  # apparent size
-dutest "--app       -t $s"  ''                                  || fail=1
-dutest "--app -a    -t $s"  ''                                  || fail=1
-dutest "--app    -S -t $s"  ''                                  || fail=1
-dutest "--app -a -S -t $s"  ''                                  || fail=1
-dutest "--app       -t -$s" 'a a/b a/c'                         || fail=1
-dutest "--app -a    -t -$s" 'a a/b a/b/0 a/b/1 a/b/2 a/b/3 a/c' || fail=1
-dutest "--app    -S -t -$s" 'a a/b a/c'                         || fail=1
-dutest "--app -a -S -t -$s" 'a a/b a/b/0 a/b/1 a/b/2 a/b/3 a/c' || fail=1
 s=$(expr $Bab0123 + 1)  # block size
 dutest "            -t $s"  ''                                  || fail=1
 dutest "      -a    -t $s"  ''                                  || fail=1
@@ -158,15 +146,6 @@ dutest "         -S -t -$s" 'a a/b a/c'                         || fail=1
 dutest "      -a -S -t -$s" 'a a/b a/b/0 a/b/1 a/b/2 a/b/3 a/c' || fail=1
 
 # Exactly the size of 'a'.
-s=$Sab0123  # apparent size
-dutest "--app       --th=$s"  'a'                                 || fail=1
-dutest "--app -a    --th=$s"  'a'                                 || fail=1
-dutest "--app    -S --th=$s"  ''                                  || fail=1
-dutest "--app -a -S --th=$s"  ''                                  || fail=1
-dutest "--app       --th=-$s" 'a a/b a/c'                         || fail=1
-dutest "--app -a    --th=-$s" 'a a/b a/b/0 a/b/1 a/b/2 a/b/3 a/c' || fail=1
-dutest "--app    -S --th=-$s" 'a a/b a/c'                         || fail=1
-dutest "--app -a -S --th=-$s" 'a a/b a/b/0 a/b/1 a/b/2 a/b/3 a/c' || fail=1
 s=$Bab0123  # block size
 dutest "            --th=$s"  'a'                                 || fail=1
 dutest "      -a    --th=$s"  'a'                                 || fail=1
@@ -178,15 +157,6 @@ dutest "         -S --th=-$s" 'a a/b a/c'                         || fail=1
 dutest "      -a -S --th=-$s" 'a a/b a/b/0 a/b/1 a/b/2 a/b/3 a/c' || fail=1
 
 # One byte smaller than 'a'.
-s=$(expr $Sab0123 - 1)  # apparent size
-dutest "--app       --th=$s"  'a'                                 || fail=1
-dutest "--app -a    --th=$s"  'a'                                 || fail=1
-dutest "--app    -S --th=$s"  ''                                  || fail=1
-dutest "--app -a -S --th=$s"  ''                                  || fail=1
-dutest "--app       --th=-$s" 'a/b a/c'                           || fail=1
-dutest "--app -a    --th=-$s" 'a/b a/b/0 a/b/1 a/b/2 a/b/3 a/c'   || fail=1
-dutest "--app    -S --th=-$s" 'a a/b a/c'                         || fail=1
-dutest "--app -a -S --th=-$s" 'a a/b a/b/0 a/b/1 a/b/2 a/b/3 a/c' || fail=1
 s=$(expr $Bab0123 - 1)  # block size
 dutest "            --th=$s"  'a'                                 || fail=1
 dutest "      -a    --th=$s"  'a'                                 || fail=1
@@ -200,15 +170,6 @@ dutest "      -a -S --th=-$s" 'a a/b a/b/0 a/b/1 a/b/2 a/b/3 a/c' || fail=1
 
 # Check numbers around the total size of the sub directory 'a/b'.
 # One byte greater than 'a/b'.
-s=$(expr $Sb0123 + 1)  # apparent size
-dutest "--app       --th=$s"  'a'                                 || fail=1
-dutest "--app -a    --th=$s"  'a'                                 || fail=1
-dutest "--app    -S --th=$s"  ''                                  || fail=1
-dutest "--app -a -S --th=$s"  ''                                  || fail=1
-dutest "--app       --th=-$s" 'a/b a/c'                           || fail=1
-dutest "--app -a    --th=-$s" 'a/b a/b/0 a/b/1 a/b/2 a/b/3 a/c'   || fail=1
-dutest "--app    -S --th=-$s" 'a a/b a/c'                         || fail=1
-dutest "--app -a -S --th=-$s" 'a a/b a/b/0 a/b/1 a/b/2 a/b/3 a/c' || fail=1
 s=$(expr $Bb0123 + 1)  # block size
 dutest "            --th=$s"  'a'                                  || fail=1
 dutest "      -a    --th=$s"  'a'                                  || fail=1
@@ -220,15 +181,6 @@ dutest "         -S --th=-$s" 'a a/b a/c'                          || fail=1
 dutest "      -a -S --th=-$s" 'a a/b a/b/0 a/b/1 a/b/2 a/b/3 a/c'  || fail=1
 
 # Exactly the size of 'a/b'.
-s=$Sb0123  # apparent size
-dutest "--app       --th=$s"  'a a/b'                              || fail=1
-dutest "--app -a    --th=$s"  'a a/b'                              || fail=1
-dutest "--app    -S --th=$s"  'a/b'                                || fail=1
-dutest "--app -a -S --th=$s"  'a/b'                                || fail=1
-dutest "--app       --th=-$s" 'a/b a/c'                            || fail=1
-dutest "--app -a    --th=-$s" 'a/b a/b/0 a/b/1 a/b/2 a/b/3 a/c'    || fail=1
-dutest "--app    -S --th=-$s" 'a a/b a/c'                          || fail=1
-dutest "--app -a -S --th=-$s" 'a a/b a/b/0 a/b/1 a/b/2 a/b/3 a/c'  || fail=1
 s=$Bb0123  # block size
 dutest "            --th=$s"  'a a/b'                              || fail=1
 dutest "      -a    --th=$s"  'a a/b'                              || fail=1
@@ -240,15 +192,6 @@ dutest "         -S --th=-$s" 'a a/b a/c'                          || fail=1
 dutest "      -a -S --th=-$s" 'a a/b a/b/0 a/b/1 a/b/2 a/b/3 a/c'  || fail=1
 
 # One byte smaller than 'a/b'.
-s=$(expr $Sb0123 - 1)  # apparent size
-dutest "--app       --th=$s"  'a a/b'                              || fail=1
-dutest "--app -a    --th=$s"  'a a/b'                              || fail=1
-dutest "--app    -S --th=$s"  'a/b'                                || fail=1
-dutest "--app -a -S --th=$s"  'a/b'                                || fail=1
-dutest "--app       --th=-$s" 'a/c'                                || fail=1
-dutest "--app -a    --th=-$s" 'a/b/0 a/b/1 a/b/2 a/b/3 a/c'        || fail=1
-dutest "--app    -S --th=-$s" 'a a/c'                              || fail=1
-dutest "--app -a -S --th=-$s" 'a a/b/0 a/b/1 a/b/2 a/b/3 a/c'      || fail=1
 s=$(expr $Bb0123 - 1)  # block size
 dutest "            --th=$s"  'a a/b'                              || fail=1
 dutest "      -a    --th=$s"  'a a/b'                              || fail=1
@@ -262,15 +205,6 @@ dutest "      -a -S --th=-$s" 'a a/b/0 a/b/1 a/b/2 a/b/3 a/c'      || fail=1
 
 # Check numbers around the total size of the files a/b/[0123]'.
 echo One byte greater than 'a/b/3'.
-s=$(expr $S3 + 1)  # apparent size
-dutest "--app       --th=$s"  'a a/b a/c'                          || fail=1
-dutest "--app -a    --th=$s"  'a a/b a/c'                          || fail=1
-dutest "--app    -S --th=$s"  'a a/b a/c'                          || fail=1
-dutest "--app -a -S --th=$s"  'a a/b a/c'                          || fail=1
-dutest "--app       --th=-$s" ''                                   || fail=1
-dutest "--app -a    --th=-$s" 'a/b/0 a/b/1 a/b/2 a/b/3'            || fail=1
-dutest "--app    -S --th=-$s" ''                                   || fail=1
-dutest "--app -a -S --th=-$s" 'a/b/0 a/b/1 a/b/2 a/b/3'            || fail=1
 s=$(expr $B3 + 1)  # block size
 dutest "            --th=$s"  'a a/b'                              || fail=1
 dutest "      -a    --th=$s"  'a a/b'                              || fail=1
@@ -283,15 +217,6 @@ dutest "      -a -S --th=-$s" 'a a/b/0 a/b/1 a/b/2 a/b/3 a/c'      || fail=1
 
 # Exactly the size of 'a/b/3'.
 echo Exactly the size of 'a/b/3'.
-s=$S3  # apparent size
-dutest "--app       --th=$s"  'a a/b a/c'                          || fail=1
-dutest "--app -a    --th=$s"  'a a/b a/b/3 a/c'                    || fail=1
-dutest "--app    -S --th=$s"  'a a/b a/c'                          || fail=1
-dutest "--app -a -S --th=$s"  'a a/b a/b/3 a/c'                    || fail=1
-dutest "--app       --th=-$s" ''                                   || fail=1
-dutest "--app -a    --th=-$s" 'a/b/0 a/b/1 a/b/2 a/b/3'            || fail=1
-dutest "--app    -S --th=-$s" ''                                   || fail=1
-dutest "--app -a -S --th=-$s" 'a/b/0 a/b/1 a/b/2 a/b/3'            || fail=1
 s=$B3  # block size
 dutest "            --th=$s"  'a a/b a/c'                          || fail=1
 dutest "      -a    --th=$s"  'a a/b a/b/1 a/b/2 a/b/3 a/c'        || fail=1
@@ -304,15 +229,6 @@ dutest "      -a -S --th=-$s" 'a a/b/0 a/b/1 a/b/2 a/b/3 a/c'      || fail=1
 
 # Exactly the size of 'a/b/2'.
 echo Exactly the size of 'a/b/2'.
-s=$S2  # apparent size
-dutest "--app       --th=$s"  'a a/b a/c'                          || fail=1
-dutest "--app -a    --th=$s"  'a a/b a/b/2 a/b/3 a/c'              || fail=1
-dutest "--app    -S --th=$s"  'a a/b a/c'                          || fail=1
-dutest "--app -a -S --th=$s"  'a a/b a/b/2 a/b/3 a/c'              || fail=1
-dutest "--app       --th=-$s" ''                                   || fail=1
-dutest "--app -a    --th=-$s" 'a/b/0 a/b/1 a/b/2'                  || fail=1
-dutest "--app    -S --th=-$s" ''                                   || fail=1
-dutest "--app -a -S --th=-$s" 'a/b/0 a/b/1 a/b/2'                  || fail=1
 s=$B2  # block size
 dutest "            --th=$s"  'a a/b a/c'                          || fail=1
 dutest "      -a    --th=$s"  'a a/b a/b/1 a/b/2 a/b/3 a/c'        || fail=1
@@ -325,15 +241,6 @@ dutest "      -a -S --th=-$s" 'a a/b/0 a/b/1 a/b/2 a/b/3 a/c'      || fail=1
 
 # Exactly the size of 'a/b/1'.
 echo Exactly the size of 'a/b/1'.
-s=$S1  # apparent size
-dutest "--app       --th=$s"  'a a/b a/c'                          || fail=1
-dutest "--app -a    --th=$s"  'a a/b a/b/1 a/b/2 a/b/3 a/c'        || fail=1
-dutest "--app    -S --th=$s"  'a a/b a/c'                          || fail=1
-dutest "--app -a -S --th=$s"  'a a/b a/b/1 a/b/2 a/b/3 a/c'        || fail=1
-dutest "--app       --th=-$s" ''                                   || fail=1
-dutest "--app -a    --th=-$s" 'a/b/0 a/b/1'                        || fail=1
-dutest "--app    -S --th=-$s" ''                                   || fail=1
-dutest "--app -a -S --th=-$s" 'a/b/0 a/b/1'                        || fail=1
 s=$B1  # block size
 dutest "            --th=$s"  'a a/b a/c'                          || fail=1
 dutest "      -a    --th=$s"  'a a/b a/b/1 a/b/2 a/b/3 a/c'        || fail=1
@@ -346,12 +253,6 @@ dutest "      -a -S --th=-$s" 'a a/b/0 a/b/1 a/b/2 a/b/3 a/c'      || fail=1
 
 # Exactly the size of 'a/b/0'.
 echo Exactly the size of 'a/b/0'.
-s=$S0  # apparent size
-dutest "--app       --th=$s"  'a a/b a/c'                          || fail=1
-dutest "--app -a    --th=$s"  'a a/b a/b/0 a/b/1 a/b/2 a/b/3 a/c'  || fail=1
-dutest "--app    -S --th=$s"  'a a/b a/c'                          || fail=1
-dutest "--app -a -S --th=$s"  'a a/b a/b/0 a/b/1 a/b/2 a/b/3 a/c'  || fail=1
-# (maximum tests (-0) not possible).
 s=$B0  # block size
 dutest "            --th=$s"  'a a/b a/c'                          || fail=1
 dutest "      -a    --th=$s"  'a a/b a/b/0 a/b/1 a/b/2 a/b/3 a/c'  || fail=1
-- 
2.26.2