SD.Next btrfs deduplication example

in case SD.Next linux server got btrfs partition we might enjoy deduplication of the files and when we use a large number of models that deduplication may save some space

btrfs filesystem du --summarize --human-readable /docker/sdnext/
compsize -x /docker/sdnext/
jdupes --one-file-system --dedupe --recurse --size /docker/sdnext/

Execution example

root@server6:~ # btrfs filesystem du --summarize --human-readable /docker/sdnext/huggingface-volume
     Total   Exclusive  Set shared  Filename
 399.51GiB   399.51GiB       0.00B  /docker/sdnext/huggingface-volume

 root@server6:~ # compsize -x /docker/sdnext/huggingface-volume
Processed 526 files, 1860252 regular extents (1860252 refs), 48 inline.
Type       Perc     Disk Usage   Uncompressed Referenced
TOTAL       89%      357G         399G         399G
none       100%      175G         175G         175G
zlib        81%      181G         223G         223G


root@server6:~ # jdupes --one-file-system --dedupe --recurse --size /docker/sdnext/huggingface-volume

root@server6:~ # btrfs filesystem du --summarize --human-readable /docker/sdnext/huggingface-volume
     Total   Exclusive  Set shared  Filename
 399.51GiB   294.70GiB    36.56GiB  /docker/sdnext/huggingface-volume

 root@server6:~ # compsize -x /docker/sdnext/huggingface-volume
Processed 526 files, 1516941 regular extents (1861709 refs), 48 inline.
Type       Perc     Disk Usage   Uncompressed Referenced
TOTAL       89%      297G         333G         399G
none       100%      151G         151G         175G
zlib        80%      146G         182G         223G

Example run on only one folder. Result might be better on FS level. But just as an example

root@server6:~ # jdupes --one-file-system --dedupe --recurse --size /docker/sdnext/huggingface-volume
Scanning: 353 files, 527 items (in 2 specified)
  [SRC] /docker/sdnext/huggingface-volume/hub/models--HiDream-ai--HiDream-E1-Full/blobs/f1ecf4952c2076dc9ef09f54814a2d4b3f14bfa9
  ====> /docker/sdnext/huggingface-volume/hub/models--HiDream-ai--HiDream-I1-Dev/blobs/f1ecf4952c2076dc9ef09f54814a2d4b3f14bfa9
  ====> /docker/sdnext/huggingface-volume/hub/models--HiDream-ai--HiDream-I1-Fast/blobs/f1ecf4952c2076dc9ef09f54814a2d4b3f14bfa9
  ====> /docker/sdnext/huggingface-volume/hub/models--HiDream-ai--HiDream-I1-Full/blobs/f1ecf4952c2076dc9ef09f54814a2d4b3f14bfa9

  [SRC] /docker/sdnext/huggingface-volume/hub/models--HiDream-ai--HiDream-I1-Dev/blobs/cf6c4d78b3b4f575a8b296ee5fccf6ab4e14c686
  ====> /docker/sdnext/huggingface-volume/hub/models--HiDream-ai--HiDream-I1-Fast/blobs/cf6c4d78b3b4f575a8b296ee5fccf6ab4e14c686
  ====> /docker/sdnext/huggingface-volume/hub/models--HiDream-ai--HiDream-I1-Full/blobs/cf6c4d78b3b4f575a8b296ee5fccf6ab4e14c686

  [SRC] /docker/sdnext/huggingface-volume/hub/models--HiDream-ai--HiDream-I1-Dev/blobs/f99e44fb320546035ca153638f1f2276e0cd2e04
  ====> /docker/sdnext/huggingface-volume/hub/models--HiDream-ai--HiDream-I1-Fast/blobs/f99e44fb320546035ca153638f1f2276e0cd2e04
  ====> /docker/sdnext/huggingface-volume/hub/models--HiDream-ai--HiDream-I1-Full/blobs/f99e44fb320546035ca153638f1f2276e0cd2e04

  [SRC] /docker/sdnext/huggingface-volume/hub/models--Wan-AI--Wan2.1-T2V-1.3B-Diffusers/blobs/481c7b2b39771c44df6dd8d13ee12ed072d731b4a650bd092885d4d52db229ad
  ====> /docker/sdnext/huggingface-volume/hub/models--Wan-AI--Wan2.1-T2V-14B-Diffusers/blobs/481c7b2b39771c44df6dd8d13ee12ed072d731b4a650bd092885d4d52db229ad

  [SRC] /docker/sdnext/huggingface-volume/hub/models--Wan-AI--Wan2.1-T2V-1.3B-Diffusers/blobs/c0ef3a140898e228a3520c9adec60743d2e8e5b3d229651bb37f1a3921919f99
  ====> /docker/sdnext/huggingface-volume/hub/models--Wan-AI--Wan2.1-T2V-14B-Diffusers/blobs/c0ef3a140898e228a3520c9adec60743d2e8e5b3d229651bb37f1a3921919f99

  [SRC] /docker/sdnext/huggingface-volume/hub/models--Wan-AI--Wan2.1-T2V-1.3B-Diffusers/blobs/a451792c739c05bca4606190cc2dd16731411bac03b4cf6aacc5767321f857c9
  ====> /docker/sdnext/huggingface-volume/hub/models--Wan-AI--Wan2.1-T2V-14B-Diffusers/blobs/a451792c739c05bca4606190cc2dd16731411bac03b4cf6aacc5767321f857c9

  [SRC] /docker/sdnext/huggingface-volume/hub/models--Wan-AI--Wan2.1-T2V-1.3B-Diffusers/blobs/f93148bcc04052a169e1e49bfcf6125df6cf9bf243cb9c627da75266cf8e35c3
  ====> /docker/sdnext/huggingface-volume/hub/models--Wan-AI--Wan2.1-T2V-14B-Diffusers/blobs/f93148bcc04052a169e1e49bfcf6125df6cf9bf243cb9c627da75266cf8e35c3

  [SRC] /docker/sdnext/huggingface-volume/hub/models--Wan-AI--Wan2.1-T2V-1.3B-Diffusers/blobs/7e76e18d224531b8197a46231cb53daf7f2f6ca707130252becf933026ac4eea
  ====> /docker/sdnext/huggingface-volume/hub/models--Wan-AI--Wan2.1-T2V-14B-Diffusers/blobs/7e76e18d224531b8197a46231cb53daf7f2f6ca707130252becf933026ac4eea

  [SRC] /docker/sdnext/huggingface-volume/hub/models--Wan-AI--Wan2.1-T2V-1.3B-Diffusers/blobs/60ece61b46ecb3e6a5b705ea304bc97535317c2a
  ====> /docker/sdnext/huggingface-volume/hub/models--Wan-AI--Wan2.1-T2V-14B-Diffusers/blobs/60ece61b46ecb3e6a5b705ea304bc97535317c2a

  [SRC] /docker/sdnext/huggingface-volume/hub/models--Wan-AI--Wan2.1-T2V-1.3B-Diffusers/blobs/2fd01c57dddd8ae386d518c69c087c8ba8c73804
  ====> /docker/sdnext/huggingface-volume/hub/models--Wan-AI--Wan2.1-T2V-14B-Diffusers/blobs/2fd01c57dddd8ae386d518c69c087c8ba8c73804

  [SRC] /docker/sdnext/huggingface-volume/hub/models--Freepik--F-Lite/blobs/ec87bffd1923e8b2774a6d240c922a41f6143081d52cf83b8fe39e9d838c893e
  ====> /docker/sdnext/huggingface-volume/hub/models--HiDream-ai--HiDream-I1-Dev/blobs/ec87bffd1923e8b2774a6d240c922a41f6143081d52cf83b8fe39e9d838c893e
  ====> /docker/sdnext/huggingface-volume/hub/models--HiDream-ai--HiDream-I1-Fast/blobs/ec87bffd1923e8b2774a6d240c922a41f6143081d52cf83b8fe39e9d838c893e
  ====> /docker/sdnext/huggingface-volume/hub/models--HiDream-ai--HiDream-I1-Full/blobs/ec87bffd1923e8b2774a6d240c922a41f6143081d52cf83b8fe39e9d838c893e
  ====> /docker/sdnext/huggingface-volume/hub/models--ostris--Flex.2-preview/blobs/ec87bffd1923e8b2774a6d240c922a41f6143081d52cf83b8fe39e9d838c893e

  [SRC] /docker/sdnext/huggingface-volume/hub/models--Freepik--F-Lite/blobs/a5640855b301fcdbceddfa90ae8066cd9414aff020552a201a255ecf2059da00
  ====> /docker/sdnext/huggingface-volume/hub/models--HiDream-ai--HiDream-I1-Dev/blobs/a5640855b301fcdbceddfa90ae8066cd9414aff020552a201a255ecf2059da00
  ====> /docker/sdnext/huggingface-volume/hub/models--HiDream-ai--HiDream-I1-Fast/blobs/a5640855b301fcdbceddfa90ae8066cd9414aff020552a201a255ecf2059da00
  ====> /docker/sdnext/huggingface-volume/hub/models--HiDream-ai--HiDream-I1-Full/blobs/a5640855b301fcdbceddfa90ae8066cd9414aff020552a201a255ecf2059da00
  ====> /docker/sdnext/huggingface-volume/hub/models--ostris--Flex.2-preview/blobs/a5640855b301fcdbceddfa90ae8066cd9414aff020552a201a255ecf2059da00

  [SRC] /docker/sdnext/huggingface-volume/hub/models--Freepik--F-Lite/blobs/c8728bc3dca59d2616a2a594cbac3ddb9eb77d5b
  ====> /docker/sdnext/huggingface-volume/hub/models--HiDream-ai--HiDream-I1-Dev/blobs/c8728bc3dca59d2616a2a594cbac3ddb9eb77d5b
  ====> /docker/sdnext/huggingface-volume/hub/models--HiDream-ai--HiDream-I1-Fast/blobs/c8728bc3dca59d2616a2a594cbac3ddb9eb77d5b
  ====> /docker/sdnext/huggingface-volume/hub/models--HiDream-ai--HiDream-I1-Full/blobs/c8728bc3dca59d2616a2a594cbac3ddb9eb77d5b
  ====> /docker/sdnext/huggingface-volume/hub/models--ostris--Flex.2-preview/blobs/c8728bc3dca59d2616a2a594cbac3ddb9eb77d5b

  [SRC] /docker/sdnext/huggingface-volume/hub/models--nvidia--Cosmos-Predict2-2B-Text2Image/blobs/30dea2006604226b944972536b9fc1ae40e3ef39b844140641cc450af5490158
  ====> /docker/sdnext/huggingface-volume/hub/models--nvidia--Cosmos-Predict2-14B-Text2Image/blobs/30dea2006604226b944972536b9fc1ae40e3ef39b844140641cc450af5490158

  [SRC] /docker/sdnext/huggingface-volume/hub/models--nvidia--Cosmos-Predict2-2B-Text2Image/blobs/d90c625c136567ef62dc13d7fde0d54393a2b08e7400b3594d38c6a9ba12f897
  ====> /docker/sdnext/huggingface-volume/hub/models--nvidia--Cosmos-Predict2-14B-Text2Image/blobs/d90c625c136567ef62dc13d7fde0d54393a2b08e7400b3594d38c6a9ba12f897

  [SRC] /docker/sdnext/huggingface-volume/hub/models--nvidia--Cosmos-Predict2-2B-Text2Image/blobs/371ddccda7716967086fd98ef7e2f8e14f96f6ae
  ====> /docker/sdnext/huggingface-volume/hub/models--nvidia--Cosmos-Predict2-14B-Text2Image/blobs/371ddccda7716967086fd98ef7e2f8e14f96f6ae

  [SRC] /docker/sdnext/huggingface-volume/hub/models--nvidia--Cosmos-Predict2-2B-Text2Image/blobs/6c85dc465ff1a4392f21aeefb7ab6ea7b9b990a9
  ====> /docker/sdnext/huggingface-volume/hub/models--nvidia--Cosmos-Predict2-14B-Text2Image/blobs/6c85dc465ff1a4392f21aeefb7ab6ea7b9b990a9

on /docker/sdnext level it looks like

  [SRC] /docker/sdnext/huggingface-volume/hub/models--Freepik--F-Lite/blobs/ec87bffd1923e8b2774a6d240c922a41f6143081d52cf83b8fe39e9d838c893e
  ====> /docker/sdnext/huggingface-volume/hub/models--HiDream-ai--HiDream-I1-Dev/blobs/ec87bffd1923e8b2774a6d240c922a41f6143081d52cf83b8fe39e9d838c893e
  ====> /docker/sdnext/huggingface-volume/hub/models--HiDream-ai--HiDream-I1-Fast/blobs/ec87bffd1923e8b2774a6d240c922a41f6143081d52cf83b8fe39e9d838c893e
  ====> /docker/sdnext/huggingface-volume/hub/models--HiDream-ai--HiDream-I1-Full/blobs/ec87bffd1923e8b2774a6d240c922a41f6143081d52cf83b8fe39e9d838c893e
  ====> /docker/sdnext/huggingface-volume/hub/models--ostris--Flex.2-preview/blobs/ec87bffd1923e8b2774a6d240c922a41f6143081d52cf83b8fe39e9d838c893e
  ====> /docker/sdnext/mnt-volume/models/Diffusers/models--black-forest-labs--FLUX.1-dev/blobs/ec87bffd1923e8b2774a6d240c922a41f6143081d52cf83b8fe39e9d838c893e
  ====> /docker/sdnext/mnt-volume/models/Diffusers/models--black-forest-labs--FLUX.1-Kontext-dev/blobs/ec87bffd1923e8b2774a6d240c922a41f6143081d52cf83b8fe39e9d838c893e
  ====> /docker/sdnext/mnt-volume/models/Diffusers/models--black-forest-labs--FLUX.1-schnell/blobs/ec87bffd1923e8b2774a6d240c922a41f6143081d52cf83b8fe39e9d838c893e
  ====> /docker/sdnext/mnt-volume/models/Diffusers/models--lodestones--Chroma/blobs/ec87bffd1923e8b2774a6d240c922a41f6143081d52cf83b8fe39e9d838c893e
  ====> /docker/sdnext/mnt-volume/models/Diffusers/models--ostris--Flex.2-preview/blobs/ec87bffd1923e8b2774a6d240c922a41f6143081d52cf83b8fe39e9d838c893e
  ====> /docker/sdnext/mnt-volume/models/Diffusers/models--shuttleai--shuttle-3.1-aesthetic/blobs/ec87bffd1923e8b2774a6d240c922a41f6143081d52cf83b8fe39e9d838c893e

  [SRC] /docker/sdnext/huggingface-volume/hub/models--Freepik--F-Lite/blobs/a5640855b301fcdbceddfa90ae8066cd9414aff020552a201a255ecf2059da00
  ====> /docker/sdnext/huggingface-volume/hub/models--HiDream-ai--HiDream-I1-Dev/blobs/a5640855b301fcdbceddfa90ae8066cd9414aff020552a201a255ecf2059da00
  ====> /docker/sdnext/huggingface-volume/hub/models--HiDream-ai--HiDream-I1-Fast/blobs/a5640855b301fcdbceddfa90ae8066cd9414aff020552a201a255ecf2059da00
  ====> /docker/sdnext/huggingface-volume/hub/models--HiDream-ai--HiDream-I1-Full/blobs/a5640855b301fcdbceddfa90ae8066cd9414aff020552a201a255ecf2059da00
  ====> /docker/sdnext/huggingface-volume/hub/models--ostris--Flex.2-preview/blobs/a5640855b301fcdbceddfa90ae8066cd9414aff020552a201a255ecf2059da00
  ====> /docker/sdnext/mnt-volume/models/Diffusers/models--black-forest-labs--FLUX.1-dev/blobs/a5640855b301fcdbceddfa90ae8066cd9414aff020552a201a255ecf2059da00
  ====> /docker/sdnext/mnt-volume/models/Diffusers/models--black-forest-labs--FLUX.1-Kontext-dev/blobs/a5640855b301fcdbceddfa90ae8066cd9414aff020552a201a255ecf2059da00
  ====> /docker/sdnext/mnt-volume/models/Diffusers/models--black-forest-labs--FLUX.1-schnell/blobs/a5640855b301fcdbceddfa90ae8066cd9414aff020552a201a255ecf2059da00
  ====> /docker/sdnext/mnt-volume/models/Diffusers/models--lodestones--Chroma/blobs/a5640855b301fcdbceddfa90ae8066cd9414aff020552a201a255ecf2059da00
  ====> /docker/sdnext/mnt-volume/models/Diffusers/models--ostris--Flex.2-preview/blobs/a5640855b301fcdbceddfa90ae8066cd9414aff020552a201a255ecf2059da00
  ====> /docker/sdnext/mnt-volume/models/Diffusers/models--shuttleai--shuttle-3.1-aesthetic/blobs/a5640855b301fcdbceddfa90ae8066cd9414aff020552a201a255ecf2059da00
...
Deduplication done (6565 files processed)