#!/usr/bin/perl # # fdupe tool - finding duplicate files # # $Id: fdupe,v 1.7 2011/10/14 20:11:21 root Exp root $ # # Source code Copyright (c) 1998,2011 Bernhard Schneider. # May be used only for non-commercial purposes with # appropriate acknowledgement of copyright. # # FILE :        fdupe # DESCRIPTION : script finds duplicate Files. # AUTHOR:       Bernhard Schneider <bernhard@neaptide.org> # hints, crrections & ideas are welcome # # usage: fdupe.pl <path> <path> ... #        find / -xdev | fdupe.pl # # how to select and remove duplicates: #   redirect output to >file, edit the file and mark lines you #   wish to move/delete with a preceding dash (-) #   Use following script to delete marked files: #   #!/usr/bin/perl -n #   chomp; unlink if s/^-//; # # history: # 12.05.99 - goto statment replaced with next # 14.05.99 - minor changes # 18.05.99 - removed confusing 'for $y' #            included hash-search # 20.05.99 - minor changes # 02.03.00 - some functions rewritten, optimized for speed # 10.01.01 - hint-fix by Ozzie |ozric at kyuzz.org| # 05.03.02 - fixed hangups by reading block/char-Devices # 08.09.11 - skips checking of hard links # 14.10.11 - accept file names from stdin # #use strict; # uncomment for debugging $|=1; local (*F1,*F2); my %farray = (); my $statF1; # ------------------------------ # traverse directories sub scan ($) {     my ($dir) = $_[0];     opendir (DIR, $dir) or die "($dir) $!:$@";     map {           (-d) ? scan ($_) : push @{$farray{-s $_}},$_              unless (-l or -S  or -p or -c or -b);     } map "$dir/$_", grep !/^\.\.?$/, readdir (DIR); closedir (DIR); } # ------------------------------ # get chunk of bytes from a file sub getchunk ($$) {   my ($fsize,$pfname) = @_;   my $chunksize = 32;   my ($nread,$buff);   return undef unless open(F1,$$pfname);   $statF1 = [(stat  F1)[3,1]];   binmode F1;   $nread = read (F1,$buff,$chunksize);   ($nread == $chunksize || $nread == $fsize) ? "$buff" : undef; }  # ------------------------------ # compare two files sub mycmp ($) {   my ($fptr) = $_[0];   my ($buffa, $buffb);   my ($nread1,$nread2);   my $statF2;   my ($buffsize) = 16*1024;   return -1 unless (open(F2,"<$$fptr"));   $statF2 = [(stat  F2)[3,1]];   return 0    if ($statF2->[0] > 1 && $statF1->[1] == $statF2->[1]);   binmode F2;   seek (F1,0,0);   do {  $nread1 = read (F1,$buffa,$buffsize);      $nread2 = read (F2,$buffb,$buffsize);      if (($nread1 != $nread2) || ($buffa cmp $buffb)) {          return -1;         }   } while ($nread1);   return 0; } # ------------------------------ print "collecting files and sizes ...\n"; if (-t STDIN) {  $ARGV[0] = '.' unless $ARGV[0]; # use wd if no arguments given  map scan $_, @ARGV; } else {   while (<STDIN>)  {   s癧\r\n]$鞍g;   push @{$farray{-s $_}},$_    unless (-l or -S  or -p or -c or -b);  } } print "now comparing ...\n"; for my $fsize (reverse sort {$a <=> $b} keys %farray) {   my ($i,$fptr,$fref,$pnum,%dupes,%index,$chunk);   # skip files with unique file size   next if $#{$farray{$fsize}} == 0;   $pnum  = 0;   %dupes = %index = ();   nx:   for (my $nx=0;$nx<=$#{$farray{$fsize}};$nx++) # $nx now 1..count of files   {                                             # with the same size  $fptr = \$farray{$fsize}[$nx];          # ref to the first file     $chunk = getchunk $fsize,$fptr;     if ($pnum) {    for $i (@{$index{$chunk}}) {          $fref = ${$dupes{$i}}[0];       unless (mycmp $fref) {             # found duplicate, collecting          push @{$dupes{$i}},$fptr;    next nx;       }    }     }     # nothing found, collecting     push @{$dupes{$pnum}},$fptr;     push @{$index{$chunk}}, $pnum++;   }   # show found dupes for actual size   for $i (keys %dupes) {     $#{$dupes{$i}} || next;     print "\n size: $fsize\n\n";     for (@{$dupes{$i}}) {         print $$_,"\n";     }   } } close F1; close F2;