* apt-pkg/acquire-item.cc:

  - don't use ReadOnlyGzip mode for PDiffs as this mode doesn't work
    in combination with the AddFd methods of our hashclasses

Add also 2 testcases: one to test pdiffs in general and
one to test the handling of compressed indexes.

diff --git a/apt-pkg/acquire-item.cc b/apt-pkg/acquire-item.cc
index a289fb7..752bc6a 100644 (file)
--- a/apt-pkg/acquire-item.cc
+++ b/apt-pkg/acquire-item.cc
@@ -280,7 +280,7 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string IndexDiffFile)          /*{{{*/
       ss >> ServerSha1 >> size;
       unsigned long const ServerSize = atol(size.c_str());
 
-      FileFd fd(CurrentPackagesFile, FileFd::ReadOnlyGzip);
+      FileFd fd(CurrentPackagesFile, FileFd::ReadOnly);
       SHA1Summation SHA1;
       SHA1.AddFD(fd.Fd(), fd.Size());
       string const local_sha1 = SHA1.Result();
@@ -297,7 +297,7 @@ bool pkgAcqDiffIndex::ParseDiffIndex(string IndexDiffFile)          /*{{{*/
       else 
       {
         if(Debug)
-           std::clog << "SHA1-Current: " << ServerSha1 << std::endl;
+           std::clog << "SHA1-Current: " << ServerSha1 << " and we start at "<< fd.Name() << " " << fd.Size() << " " << local_sha1 << std::endl;
 
         // check the historie and see what patches we need
         string const history = Tags.FindS("SHA1-History");
@@ -511,7 +511,7 @@ bool pkgAcqIndexDiffs::QueueNextDiff()                                      /*{{{*/
    string FinalFile = _config->FindDir("Dir::State::lists");
    FinalFile += URItoFileName(RealURI);
 
-   FileFd fd(FinalFile, FileFd::ReadOnlyGzip);
+   FileFd fd(FinalFile, FileFd::ReadOnly);
    SHA1Summation SHA1;
    SHA1.AddFD(fd.Fd(), fd.Size());
    string local_sha1 = string(SHA1.Result());

diff --git a/debian/changelog b/debian/changelog
index c2c0b60..c05f943 100644 (file)
--- a/debian/changelog
+++ b/debian/changelog
@@ -19,8 +19,11 @@ apt (0.8.0~pre2) UNRELEASED; urgency=low
     - some http servers violate HTTP1.1 by not issuing a Reason-Phrase
       (or at least a space after the code) especially for 200, but lets
       be nice and ignore it as we don't need the reason in general
+  * apt-pkg/acquire-item.cc:
+    - don't use ReadOnlyGzip mode for PDiffs as this mode doesn't work
+      in combination with the AddFd methods of our hashclasses
 
- -- David Kalnischkies <kalnischkies@gmail.com>  Fri, 20 Aug 2010 23:00:54 +0200
+ -- David Kalnischkies <kalnischkies@gmail.com>  Sat, 21 Aug 2010 22:18:41 +0200
 
 apt (0.8.0~pre1) experimental; urgency=low
 

diff --git a/test/integration/Packages-pdiff-usage b/test/integration/Packages-pdiff-usage
new file mode 100644 (file)
index 0000000..d1530a9
--- /dev/null
+++ b/test/integration/Packages-pdiff-usage
@@ -0,0 +1,34 @@
+Package: apt
+Version: 0.7.25.3
+Architecture: i386
+Maintainer: APT Development Team <deity@lists.debian.org>
+Installed-Size: 5244
+Replaces: libapt-pkg-dev (<< 0.3.7), libapt-pkg-doc (<< 0.3.7)
+Provides: libapt-pkg-libc6.9-6-4.8
+Depends: libc6 (>= 2.3.4), libgcc1 (>= 1:4.1.1), libstdc++6 (>= 4.4.0), debian-archive-keyring
+Suggests: aptitude | synaptic | wajig, dpkg-dev, apt-doc, bzip2, lzma, python-apt
+Filename: pool/main/a/apt/apt_0.7.25.3_i386.deb
+Size: 1750610
+MD5sum: 311aadc67d1b72428b54c9b4e1f76671
+SHA1: 3c695e028f74d5c55226f9ef30000bbbd881088c
+SHA256: b46fd1546151c545fe4bfa56a5cc0e7811a9f68826326a529777fd660f28f050
+Description: Advanced front-end for dpkg
+ This is Debian's next generation front-end for the dpkg package manager.
+ It provides the apt-get utility and APT dselect method that provides a
+ simpler, safer way to install and upgrade packages.
+ .
+ APT features complete installation ordering, multiple source capability
+ and several other unique features, see the Users Guide in apt-doc.
+
+Package: oldstuff
+Version: 1.0
+Architecture: i386
+Maintainer: Joe Sixpack <joe@example.org>
+Installed-Size: 100
+Filename: pool/oldstuff_1.0_i386.deb
+Size: 100000
+MD5sum: 311aeeadf78324aaff54c9b4e1f76671
+SHA1: 3c695e028f74d5c544deeddaaa1242desa81088c
+SHA256: b46fd1546151c545fe4bfa56a5cc0e7deaef23e2da3e4f129727fd660f28f050
+Description: some cool but old stuff
+ This package will disappear in the next mirror update

diff --git a/test/integration/Packages-pdiff-usage-new b/test/integration/Packages-pdiff-usage-new
new file mode 100644 (file)
index 0000000..9157596
--- /dev/null
+++ b/test/integration/Packages-pdiff-usage-new
@@ -0,0 +1,37 @@
+Package: newstuff
+Version: 1.0
+Architecture: i386
+Maintainer: Joe Sixpack <joe@example.org>
+Installed-Size: 101
+Filename: pool/newstuff_1.0_i386.deb
+Size: 101100
+MD5sum: 311aeeadf78324aaff1ceaf3e1f76671
+SHA1: 3c695e028f7a1ae324deeddaaa1242desa81088c
+SHA256: b46fd154615edefab321cc56a5cc0e7deaef23e2da3e4f129727fd660f28f050
+Description: some cool and shiny new stuff
+ This package will appear in the next mirror update
+
+Package: apt
+Priority: important
+Section: admin
+Installed-Size: 5672
+Maintainer: APT Development Team <deity@lists.debian.org>
+Architecture: i386
+Version: 0.8.0~pre1
+Replaces: manpages-pl (<< 20060617-3~)
+Provides: libapt-pkg4.10
+Depends: libc6 (>= 2.3.4), libgcc1 (>= 1:4.1.1), libstdc++6 (>= 4.4.0), zlib1g (>= 1:1.1.4), debian-archive-keyring, gnupg
+Suggests: aptitude | synaptic | wajig, dpkg-dev, apt-doc, bzip2, lzma, python-apt
+Conflicts: python-apt (<< 0.7.93.2~)
+Filename: pool/main/a/apt/apt_0.8.0~pre1_i386.deb
+Size: 2013046
+MD5sum: 6786ca6270c988f2c201716ededaedec
+SHA1: fe26559e745d4c2c977c27170938852041c9adff
+SHA256: a12f968467e1e3cec24191b72bfe84f7aeed3ce422e1a60bb4f1454f2b89b8ee
+Description: Advanced front-end for dpkg
+ This is Debian's next generation front-end for the dpkg package manager.
+ It provides the apt-get utility and APT dselect method that provides a
+ simpler, safer way to install and upgrade packages.
+ .
+ APT features complete installation ordering, multiple source capability
+ and several other unique features, see the Users Guide in apt-doc.

diff --git a/test/integration/framework b/test/integration/framework
index 82fca20..8efe473 100644 (file)
--- a/test/integration/framework
+++ b/test/integration/framework
@@ -80,7 +80,8 @@ setupenvironment() {
        BUILDDIRECTORY="${TESTDIR}/../../build/bin"
        test -x "${BUILDDIRECTORY}/apt-get" || msgdie "You need to build tree first"
        local OLDWORKINGDIRECTORY=$(pwd)
-#      trap "cd /; rm -rf $TMPWORKINGDIRECTORY; cd $OLDWORKINGDIRECTORY" 0 HUP INT QUIT ILL ABRT FPE SEGV PIPE TERM
+       CURRENTTRAP="cd /; rm -rf $TMPWORKINGDIRECTORY; cd $OLDWORKINGDIRECTORY"
+       trap "$CURRENTTRAP" 0 HUP INT QUIT ILL ABRT FPE SEGV PIPE TERM
        cd $TMPWORKINGDIRECTORY
        mkdir rootdir aptarchive keys
        cd rootdir
@@ -373,17 +374,44 @@ setupaptarchive() {
        else
                setupflataptarchive
        fi
-       msgninfo "\tSign archive with Joe Sixpack key… "
-       for RELEASE in $(find aptarchive/ -name Release); do
-               gpg --no-default-keyring --secret-keyring ./keys/joesixpack.sec --keyring ./keys/joesixpack.pub --default-key Joe -abs -o ${RELEASE}.gpg ${RELEASE}
-       done
-       msgdone "info"
+       signreleasefiles
        msgninfo "\tSync APT's cache with the archive… "
        aptget update -qq
        msgdone "info"
 }
 
-diff() {
+signreleasefiles() {
+       local SIGNER="${1:-Joe Sixpack}"
+       msgninfo "\tSign archive with $SIGNER key… "
+       local SECKEYS=""
+       for KEY in $(find keys/ -name '*.sec'); do
+               SECKEYS="$SECKEYS --secret-keyring $KEY"
+       done
+       local PUBKEYS=""
+       for KEY in $(find keys/ -name '*.pub'); do
+               PUBKEYS="$PUBKEYS --keyring $KEY"
+       done
+       for RELEASE in $(find aptarchive/ -name Release); do
+               gpg --yes --no-default-keyring $SECKEYS $PUBKEYS --default-key "$SIGNER" -abs -o ${RELEASE}.gpg ${RELEASE}
+       done
+       msgdone "info"
+}
+
+changetowebserver() {
+       if which weborf > /dev/null; then
+               weborf -xb aptarchive/ 2>&1 > /dev/null &
+               CURRENTTRAP="kill $(ps | grep weborf | sed -e 's#^[ ]*##' | cut -d' ' -f 1); $CURRENTTRAP"
+               trap "$CURRENTTRAP" 0 HUP INT QUIT ILL ABRT FPE SEGV PIPE TERM
+               local APTARCHIVE="file://$(readlink -f ./aptarchive)"
+               for LIST in $(find rootdir/etc/apt/sources.list.d/ -name 'apt-test-*.list'); do
+                       sed -i $LIST -e "s#$APTARCHIVE#http://localhost:8080/#"
+               done
+               return 0
+       fi
+       return 1
+}
+
+checkdiff() {
        local DIFFTEXT="$($(which diff) -u $* | sed -e '/^---/ d' -e '/^+++/ d' -e '/^@@/ d')"
        if [ -n "$DIFFTEXT" ]; then
                echo
@@ -399,9 +427,9 @@ testfileequal() {
        shift
        msgtest "Test for correctness of file" "$FILE"
        if [ -z "$*" ]; then
-               echo -n "" | diff $FILE - && msgpass || msgfail
+               echo -n "" | checkdiff $FILE - && msgpass || msgfail
        else
-               echo "$*" | diff $FILE - && msgpass || msgfail
+               echo "$*" | checkdiff $FILE - && msgpass || msgfail
        fi
 }
 
@@ -410,7 +438,7 @@ testequal() {
        echo "$1" > $COMPAREFILE
        shift
        msgtest "Test for equality of" "$*"
-       $* 2>&1 | diff $COMPAREFILE - && msgpass || msgfail
+       $* 2>&1 | checkdiff $COMPAREFILE - && msgpass || msgfail
        rm $COMPAREFILE
 }
 
@@ -423,10 +451,10 @@ testequalor2() {
        shift 2
        msgtest "Test for equality OR of" "$*"
        $* 2>&1 1> $COMPAREAGAINST
-       (diff $COMPAREFILE1 $COMPAREAGAINST 1> /dev/null ||
-               diff $COMPAREFILE2 $COMPAREAGAINST 1> /dev/null) && msgpass ||
-               ( echo "\n${CINFO}Diff against OR 1${CNORMAL}" "$(diff $COMPAREFILE1 $COMPAREAGAINST)" \
-                      "\n${CINFO}Diff against OR 2${CNORMAL}" "$(diff $COMPAREFILE2 $COMPAREAGAINST)" &&
+       (checkdiff $COMPAREFILE1 $COMPAREAGAINST 1> /dev/null ||
+               checkdiff $COMPAREFILE2 $COMPAREAGAINST 1> /dev/null) && msgpass ||
+               ( echo "\n${CINFO}Diff against OR 1${CNORMAL}" "$(checkdiff $COMPAREFILE1 $COMPAREAGAINST)" \
+                      "\n${CINFO}Diff against OR 2${CNORMAL}" "$(checkdiff $COMPAREFILE2 $COMPAREAGAINST)" &&
                  msgfail )
        rm $COMPAREFILE1 $COMPAREFILE2 $COMPAREAGAINST
 }
@@ -448,7 +476,7 @@ N: No packages found"
        local ARCH=$(dpkg-architecture -qDEB_HOST_ARCH_CPU)
        eval `apt-config shell ARCH APT::Architecture`
        echo "$VIRTUAL" | sed -e "s/:$ARCH//" -e 's/:all//' > $COMPAREFILE
-       aptcache show $PACKAGE 2>&1 | diff $COMPAREFILE - && msgpass || msgfail
+       aptcache show $PACKAGE 2>&1 | checkdiff $COMPAREFILE - && msgpass || msgfail
        rm $COMPAREFILE
 }
 

diff --git a/test/integration/test-compressed-indexes b/test/integration/test-compressed-indexes
new file mode 100755 (executable)
index 0000000..0b73f4c
--- /dev/null
+++ b/test/integration/test-compressed-indexes
@@ -0,0 +1,131 @@
+#!/bin/sh
+set -e
+
+local TESTDIR=$(readlink -f $(dirname $0))
+. $TESTDIR/framework
+
+setupenvironment
+configarchitecture "i386"
+
+buildsimplenativepackage "testpkg" "i386" "1.0"
+setupaptarchive
+
+local GOODSHOW="$(aptcache show testpkg)
+"
+local GOODPOLICY="$(aptcache policy testpkg)"
+local GOODSHOWSRC="$(aptcache showsrc testpkg)
+"
+
+test $(echo "$GOODSHOW" | grep -e '^Package: testpkg' -e '^Version: 1.0' -e '^Architecture: i386' | wc -l) -eq 3
+testequal "$GOODSHOW" aptcache show testpkg
+test $(echo "$GOODPOLICY" | grep -e '^testpkg:' -e '^  Candidate:' -e '^  Installed: (none)' -e '500 file:/' | wc -l) -eq 4
+testequal "$GOODPOLICY" aptcache policy testpkg
+test $(echo "$GOODSHOWSRC" | grep -e '^Package: testpkg' -e '^Format: 3.0 (native)' -e '^Files:' -e '^Checksums-Sha256:' | wc -l) -eq 4
+testequal "$GOODSHOWSRC" aptcache showsrc testpkg
+
+
+testrun() {
+       local F
+       if [ -e rootdir/var/lib/apt/lists/*localhost*Release ]; then
+               msgtest "Check if all index files are" "${1:-uncompressed}"
+               if [ "$1" = "compressed" ]; then
+                       ! test -e rootdir/var/lib/apt/lists/*_Packages || F=1
+                       ! test -e rootdir/var/lib/apt/lists/*_Sources || F=1
+                       test -e rootdir/var/lib/apt/lists/*_Packages.gz || F=1
+                       test -e rootdir/var/lib/apt/lists/*_Sources.gz || F=1
+               else
+                       test -e rootdir/var/lib/apt/lists/*_Packages || F=1
+                       test -e rootdir/var/lib/apt/lists/*_Sources || F=1
+                       ! test -e rootdir/var/lib/apt/lists/*_Packages.gz || F=1
+                       ! test -e rootdir/var/lib/apt/lists/*_Sources.gz || F=1
+               fi
+               if [ -n "$F" ]; then
+                       ls -laR rootdir/var/lib/apt/lists/
+                       msgfail
+               else
+                       msgpass
+               fi
+               msgtest "Check if package is downloadable"
+               aptget install -d testpkg -qq && msgpass || msgfail
+               msgtest "\tdeb file is present"; test -f rootdir/var/cache/apt/archives/testpkg_1.0_i386.deb && msgpass || msgfail
+               aptget clean
+               msgtest "\tdeb file is gone"; ! test -f rootdir/var/cache/apt/archives/testpkg_1.0_i386.deb && msgpass || msgfail
+       fi
+       rm rootdir/var/cache/apt/pkgcache.bin rootdir/var/cache/apt/srcpkgcache.bin
+       testequal "$GOODSHOW" aptcache show testpkg
+       testequal "$GOODSHOW" aptcache show testpkg
+       rm rootdir/var/cache/apt/pkgcache.bin rootdir/var/cache/apt/srcpkgcache.bin
+       testequal "$GOODPOLICY" aptcache policy testpkg
+       testequal "$GOODPOLICY" aptcache policy testpkg
+       rm rootdir/var/cache/apt/pkgcache.bin rootdir/var/cache/apt/srcpkgcache.bin
+       testequal "$GOODSHOWSRC" aptcache showsrc testpkg
+       testequal "$GOODSHOWSRC" aptcache showsrc testpkg
+       rm -f rootdir/var/cache/apt/pkgcache.bin rootdir/var/cache/apt/srcpkgcache.bin
+       rm -rf rootdir/var/cache/apt/archives
+       msgtest "Check if the source is aptgetable"
+       aptget source testpkg -qq 2> /dev/null > /dev/null && msgpass || msgfail
+       msgtest "\tdsc file is present"; test -f testpkg_1.0.dsc && msgpass || msgfail
+       msgtest "\tdirectory is present"; test -d testpkg-1.0 && msgpass || msgfail
+       rm -rf testpkg-1.0
+}
+
+msgmsg "File: Test with uncompressed indexes"
+testrun
+
+aptget update -qq -o Acquire::Pdiffs=1
+msgmsg "File: Test with uncompressed indexes (update unchanged with pdiffs)"
+testrun
+
+aptget update -qq -o Acquire::Pdiffs=0
+msgmsg "File: Test with uncompressed indexes (update unchanged without pdiffs)"
+testrun
+
+rm -rf rootdir/var/lib/apt/lists
+echo 'Acquire::CompressionTypes::Order:: "gz";
+Acquire::GzipIndexes "true";' > rootdir/etc/apt/apt.conf.d/02compressindex
+
+aptget update -qq
+msgmsg "File: Test with compressed indexes"
+testrun "compressed"
+
+aptget update -qq -o Acquire::Pdiffs=1
+msgmsg "File: Test with compressed indexes (update unchanged with pdiffs)"
+testrun "compressed"
+
+aptget update -qq -o Acquire::Pdiffs=0
+msgmsg "File: Test with compressed indexes (update unchanged without pdiffs)"
+testrun "compressed"
+
+rm rootdir/etc/apt/apt.conf.d/02compressindex
+changetowebserver
+aptget update -qq
+local GOODPOLICY="$(aptcache policy testpkg)"
+test $(echo "$GOODPOLICY" | grep -e '^testpkg:' -e '^  Candidate:' -e '^  Installed: (none)' -e '500 http://' | wc -l) -eq 4
+testequal "$GOODPOLICY" aptcache policy testpkg
+
+msgmsg "HTTP: Test with uncompressed indexes"
+testrun
+
+aptget update -qq -o Acquire::Pdiffs=1
+msgmsg "HTTP: Test with uncompressed indexes (update unchanged with pdiffs)"
+testrun
+
+aptget update -qq -o Acquire::Pdiffs=0
+msgmsg "HTTP: Test with uncompressed indexes (update unchanged without pdiffs)"
+testrun
+
+rm -rf rootdir/var/lib/apt/lists
+echo 'Acquire::CompressionTypes::Order:: "gz";
+Acquire::GzipIndexes "true";' > rootdir/etc/apt/apt.conf.d/02compressindex
+
+aptget update -qq
+msgmsg "HTTP: Test with compressed indexes"
+testrun "compressed"
+
+aptget update -qq -o Acquire::Pdiffs=1
+msgmsg "HTTP: Test with compressed indexes (update unchanged with pdiffs)"
+testrun "compressed"
+
+aptget update -qq -o Acquire::Pdiffs=0
+msgmsg "HTTP: Test with compressed indexes (update unchanged without pdiffs)"
+testrun "compressed"

diff --git a/test/integration/test-pdiff-usage b/test/integration/test-pdiff-usage
new file mode 100755 (executable)
index 0000000..85cbe90
--- /dev/null
+++ b/test/integration/test-pdiff-usage
@@ -0,0 +1,51 @@
+#!/bin/sh
+set -e
+
+local TESTDIR=$(readlink -f $(dirname $0))
+. $TESTDIR/framework
+
+setupenvironment
+configarchitecture "i386"
+
+buildaptarchive
+setupflataptarchive
+changetowebserver
+signreleasefiles
+aptget update -qq
+
+testnopackage newstuff
+PKGFILE="${TESTDIR}/$(echo "$(basename $0)" | sed 's#^test-#Packages-#')"
+testequal "$(cat ${PKGFILE})
+" aptcache show apt oldstuff
+
+cp ${PKGFILE}-new aptarchive/Packages
+cat aptarchive/Packages | gzip > aptarchive/Packages.gz
+cat aptarchive/Packages | bzip2 > aptarchive/Packages.bz2
+cat aptarchive/Packages | lzma > aptarchive/Packages.lzma
+rm -rf aptarchive/Packages.diff
+aptftparchive release aptarchive/ > aptarchive/Release
+mkdir -p aptarchive/Packages.diff
+PATCHFILE="aptarchive/Packages.diff/$(date +%Y-%m-%d-%H%M.%S)"
+diff -e ${PKGFILE} ${PKGFILE}-new > ${PATCHFILE} || true
+cat $PATCHFILE | gzip > ${PATCHFILE}.gz
+PATCHINDEX="aptarchive/Packages.diff/Index"
+echo "SHA1-Current: $(sha1sum ${PKGFILE}-new | cut -d' ' -f 1) $(stat -c%s ${PKGFILE}-new)
+SHA1-History:
+ 9f4148e06d7faa37062994ff10d0c842d7017513 33053002 2010-08-18-2013.28
+ $(sha1sum $PKGFILE | cut -d' ' -f 1) $(stat -c%s $PKGFILE) $(basename $PATCHFILE)
+SHA1-Patches:
+ 7651fc0ac57cd83d41c63195a9342e2db5650257 19722 2010-08-18-0814.28
+ $(sha1sum $PATCHFILE | cut -d' ' -f 1) $(stat -c%s $PATCHFILE) $(basename $PATCHFILE)" > $PATCHINDEX
+sed -i aptarchive/Release \
+ -e "/^MD5Sum:/ a\
+ \ $(md5sum $PATCHINDEX | cut -d' ' -f 1) $(stat -c%s $PATCHINDEX) Packages.diff/Index" \
+ -e "/^SHA1:/ a\
+ \ $(sha1sum $PATCHINDEX | cut -d' ' -f 1) $(stat -c%s $PATCHINDEX) Packages.diff/Index" \
+ -e "/^SHA256:/ a\
+ \ $(sha256sum $PATCHINDEX | cut -d' ' -f 1) $(stat -c%s $PATCHINDEX) Packages.diff/Index"
+signreleasefiles
+aptget update -qq
+
+testnopackage oldstuff
+testequal "$(cat ${PKGFILE}-new)
+" aptcache show apt newstuff