PROBLEM: cvs i puchni

[Michal Moskal]

On Mon, Mar 17, 2003 at 05:19:38PM +0100, Andrzej Krzysztofowicz wrote:
> > > 2. Jak sobie radzic przy problemach z fs-em ?
> > 
> > Nie rozumiem? Jakich problemach? Że za dużo plików w katalogu? Tak jak
> > napisałem jest ich około 256 razy mniej w katalogu.
> 
> Cos sie syplo, wcielo katalog. Zostaly pliki #nnnnnnnn w /lost+found ...
> Nie mow, ze z backupu ... ;)
> 
> > Właśnie piszę skrypt co to będzie dociągał (na distfiles) + support w
> > builderze i skrypcik do upgrejdu (żeby w specu md5 poprawiał) i powinno
> > być ok.
> 
> Co do samego pomyslu, to jestem za. Szczegoly mozna dopracowac.

Załączam co zrobiłem do tej pory, jutro to przetestuje i commitnę.

-- 
: Michal Moskal ::::: malekith/at/pld-linux.org :  GCS {C,UL}++++$ a? !tv
: PLD Linux ::::::: Wroclaw University, CS Dept :  {E-,w}-- {b++,e}>+++ h
-------------- następna część ---------
Index: builder
===================================================================
RCS file: /cvsroot/SPECS/builder,v
retrieving revision 1.135
diff -u -r1.135 builder
--- builder	16 Mar 2003 19:08:25 -0000	1.135
+++ builder	17 Mar 2003 17:36:21 -0000
@@ -302,6 +302,28 @@
     echo "$url"
 }
 
+src_no ()
+{
+    cd $SPECS_DIR
+    $RPMBUILD -bp  $BCOND --define 'prep %dump' $SPECFILE 2>&1 | \
+    	grep "SOURCEURL[0-9]*[ 	]*$1""[ 	]*$" | \
+	sed -e 's/.*SOURCEURL\([0-9][0-9]*\).*/\1/' | \
+	xargs
+}
+
+src_md5 ()
+{
+    no=$(src_no "$1")
+    [ -z "$no" ] && return
+    cd $SPECS_DIR
+    grep -i "#[ 	]*Source$no-md5[ 	]*:" $SPECFILE | sed -e 's/.*://' | xargs
+}
+
+distfiles_url ()
+{
+    echo "ftp://distfiles.pld.org.pl/src/$(src_md5 "$1" | sed -e 's|^\(.\)\(.\)|\1/\2/&|')"
+}
+
 get_files()
 {
     GET_FILES="$@"
@@ -334,7 +356,17 @@
 		    echo "Warning: no URL given for $i"
 		fi
 
-		if [ -z "$NOCVS" ]|| [ `echo $i | grep -vE '(ftp|http|https)://'` ]; then
+		if [ -n "$(src_md5 "$i")" ] ; then
+		    target=$(nourl "$i")
+		    url=$(distfiles_url "$i")
+		    if [ -z "$NOMIRRORS" ] ; then 
+			url="`find_mirror "$url"`"
+		    fi
+		    ${GETURI} -O "$target" "$url" || \
+			if [ `echo $url | grep -E 'ftp://'` ]; then 
+			  ${GETURI2} -O "$target" "$url"
+			fi
+		elif [ -z "$NOCVS" ]|| [ `echo $i | grep -vE '(ftp|http|https)://'` ]; then
 		    result=1
         	    retries_counter=0
 	            while [ "$result" != "0" -a "$retries_counter" -le "$CVS_RETRIES" ]; do
-------------- następna część ---------
#!/usr/bin/perl -w

$ftpdir = "/shm/ftp";

sub localfile ($)
{
  my $md5 = shift;
  $md5 =~ /^(.)(.)/ or die;
  return "$ftpdir/$1/$2/$md5";
}

SPEC: foreach (@ARGV) {
  $spec = $_;
  open(IN, "< $spec") or die "can't open $spec";
  $spec =~ m|([^/]+)\.spec$| or die;
  $spec_base = $1;

  # first extract src-number -> md5 mapping
  %src = ();
  $cnt = 0;
  $cvs_version = "";
  while (<IN>) {
    /\$Revision:\s*([\d\.]+)/ and $cvs_version = $1;
    /^\s*(#*)\s*source(\d+)-md5\s*:\s*([a-f0-9]{32})\s*$/i or next;
    $src{$2} = $3;
    $cnt++;
  }
  close(IN);

  $spec_id = "$spec_base-$cvs_version";

  # then remove files we already have from %src
  foreach $nr (keys %src) {
    $local = localfile($src{$nr}) . ".desc";
    if (-f $local) {
      open(DESC, "< $local") or die;
      $has = 0;
      while (<DESC>) {
        chomp;
        /^Used-by: $spec_id$/ and $has = 1;
      }
      close(DESC);
      if (!$has) {
        open(DESC, ">> $local") or die;
	print DESC "Used-by: $spec_id\n";
	close(DESC);
      }
      undef $src{$nr};
      $cnt--;
    }
  }

  # continue if we have no file in %src
  next SPEC if ($cnt == 0);

  %urls = ();
  
  open(RPM, "rpmbuild -bp --define 'prep \%dump' $spec 2>&1 |");
  $pkg_version = "";
  while (<RPM>) {
    /^[\s\d\-=:]+version\s+([^\s]*)\s*$/ and $pkg_version = $1;
    /^[\s\d\-=:]+SOURCEURL(\d+)\s+([^\s]*)\s*$/ or next;
    $urls{$1} = $2;
  }
  close(RPM);

  foreach $nr (keys %src) {
    defined $src{$nr} or next;
    $url = $urls{$nr};
    unless (defined $url) {
      print STDERR "$spec_id: no url for source #$nr\n";
      next;
    }
    $local = localfile($src{$nr});
    open(DESC, "> $local.desc") or die;
    print DESC "Fetched-for: $spec_base-$pkg_version\n";
    print DESC "Source-URL: $url\n";
    close(DESC);

    $err = system("wget -q -c -O $local.tmp $url");
    if ($err != 0) {
      print STDERR "$spec_id: failed to fetch $url\n";
      unlink("$local.desc");
      unlink("$local.tmp");
    } else {
      open(DESC, ">> $local.desc") or die;
      $md5 = `md5sum $local.tmp`;
      $md5 =~ /^([a-f0-9]{32})\s+/ or die;
      $md5 = $1;
      if ($md5 ne $src{$nr}) {
        print STDERR "$spec_id: md5 mismatch: $md5, while $src{$nr} expected for $url\n";
        unlink("$local.desc");
        unlink("$local.tmp");
      } else {
        print DESC "MD5: $md5\n";
	$size = -s "$local.tmp";
	$time = time;
	print DESC "Size: $size\nCurrent-time: $time\n";
        print DESC "Used-by: $spec_id\n";
	close(DESC);
        link("$local.tmp", "$local");
        unlink("$local.tmp");
      }
    }
  }
}