HCoop / ntk / apt.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
* ftparchive/writer.cc:
[ntk/apt.git] / ftparchive / writer.cc
CommitLineData
b2e465d6
AL		 1// -*- mode: cpp; mode: fold -*-
2// Description /*{{{*/
5200ec6f 3// $Id: writer.cc,v 1.14 2004/03/24 01:40:43 mdz Exp $
b2e465d6
AL		 4/* ######################################################################
5
6 Writer
7
8 The file writer classes. These write various types of output, sources,
9 packages and contents.
10
11 ##################################################################### */
12 /*}}}*/
13// Include Files /*{{{*/
b2e465d6
AL		 14#include "writer.h"
15
dc738e7a 16#include <apti18n.h>
b2e465d6
AL		 17#include <apt-pkg/strutl.h>
18#include <apt-pkg/error.h>
19#include <apt-pkg/configuration.h>
03bef784 20#include <apt-pkg/aptconfiguration.h>
b2e465d6 21#include <apt-pkg/md5.h>
8c4e1f97 22#include <apt-pkg/hashes.h>
b2e465d6
AL		 23#include <apt-pkg/deblistparser.h>
24
25#include <sys/types.h>
26#include <unistd.h>
98953965 27#include <ctime>
b2e465d6 28#include <ftw.h>
98953965 29#include <fnmatch.h>
8c58f506 30#include <iostream>
bf99a6d3 31#include <sstream>
4f333a8b 32#include <memory>
b2e465d6
AL		 33
34#include "cachedb.h"
35#include "apt-ftparchive.h"
36#include "multicompress.h"
37 /*}}}*/
8c58f506 38using namespace std;
b2e465d6
AL		 39FTWScanner *FTWScanner::Owner;
40
64177f17
AL		 41// SetTFRewriteData - Helper for setting rewrite lists /*{{{*/
42// ---------------------------------------------------------------------
43/* */
44inline void SetTFRewriteData(struct TFRewriteData &tfrd,
45 const char *tag,
46 const char *rewrite,
47 const char *newtag = 0)
48{
49 tfrd.Tag = tag;
50 tfrd.Rewrite = rewrite;
51 tfrd.NewTag = newtag;
52}
53 /*}}}*/
54
b2e465d6
AL		 55// FTWScanner::FTWScanner - Constructor /*{{{*/
56// ---------------------------------------------------------------------
57/* */
31981076 58FTWScanner::FTWScanner(string const &Arch): Arch(Arch)
b2e465d6
AL		 59{
60 ErrorPrinted = false;
61 NoLinkAct = !_config->FindB("APT::FTPArchive::DeLinkAct",true);
3c54407f
DK		 62
63 DoMD5 = _config->FindB("APT::FTPArchive::MD5",true);
64 DoSHA1 = _config->FindB("APT::FTPArchive::SHA1",true);
65 DoSHA256 = _config->FindB("APT::FTPArchive::SHA256",true);
9abccf4a 66 DoSHA512 = _config->FindB("APT::FTPArchive::SHA512",true);
b2e465d6
AL		 67}
68 /*}}}*/
69// FTWScanner::Scanner - FTW Scanner /*{{{*/
70// ---------------------------------------------------------------------
71/* This is the FTW scanner, it processes each directory element in the
72 directory tree. */
cde41ae8 73int FTWScanner::ScannerFTW(const char *File,const struct stat *sb,int Flag)
b2e465d6
AL		 74{
75 if (Flag == FTW_DNR)
76 {
77 Owner->NewLine(1);
dc738e7a 78 ioprintf(c1out, _("W: Unable to read directory %s\n"), File);
b2e465d6
AL		 79 }
80 if (Flag == FTW_NS)
81 {
82 Owner->NewLine(1);
dc738e7a 83 ioprintf(c1out, _("W: Unable to stat %s\n"), File);
b2e465d6
AL		 84 }
85 if (Flag != FTW_F)
86 return 0;
87
cde41ae8
MV		 88 return ScannerFile(File, true);
89}
90 /*}}}*/
91// FTWScanner::ScannerFile - File Scanner /*{{{*/
92// ---------------------------------------------------------------------
93/* */
9209ec47 94int FTWScanner::ScannerFile(const char *File, bool const &ReadLink)
cde41ae8 95{
98953965 96 const char *LastComponent = strrchr(File, '/');
ab3846c0
MV		 97 char *RealPath = NULL;
98
98953965
AL		 99 if (LastComponent == NULL)
100 LastComponent = File;
101 else
102 LastComponent++;
103
9209ec47 104 vector<string>::const_iterator I;
98953965
AL		 105 for(I = Owner->Patterns.begin(); I != Owner->Patterns.end(); ++I)
106 {
107 if (fnmatch((*I).c_str(), LastComponent, 0) == 0)
108 break;
109 }
110 if (I == Owner->Patterns.end())
b2e465d6
AL		 111 return 0;
112
113 /* Process it. If the file is a link then resolve it into an absolute
114 name.. This works best if the directory components the scanner are
115 given are not links themselves. */
116 char Jnk[2];
117 Owner->OriginalPath = File;
ab3846c0 118 if (ReadLink &&
cde41ae8 119 readlink(File,Jnk,sizeof(Jnk)) != -1 &&
ab3846c0
MV		 120 (RealPath = realpath(File,NULL)) != 0)
121 {
122 Owner->DoPackage(RealPath);
123 free(RealPath);
124 }
b2e465d6
AL		 125 else
126 Owner->DoPackage(File);
127
128 if (_error->empty() == false)
129 {
130 // Print any errors or warnings found
131 string Err;
132 bool SeenPath = false;
133 while (_error->empty() == false)
134 {
135 Owner->NewLine(1);
136
9209ec47 137 bool const Type = _error->PopMessage(Err);
b2e465d6 138 if (Type == true)
dc738e7a 139 cerr << _("E: ") << Err << endl;
b2e465d6 140 else
dc738e7a 141 cerr << _("W: ") << Err << endl;
b2e465d6
AL		 142
143 if (Err.find(File) != string::npos)
144 SeenPath = true;
145 }
146
147 if (SeenPath == false)
dc738e7a 148 cerr << _("E: Errors apply to file ") << "'" << File << "'" << endl;
b2e465d6
AL		 149 return 0;
150 }
151
152 return 0;
153}
154 /*}}}*/
155// FTWScanner::RecursiveScan - Just scan a directory tree /*{{{*/
156// ---------------------------------------------------------------------
157/* */
9209ec47 158bool FTWScanner::RecursiveScan(string const &Dir)
b2e465d6 159{
ab3846c0 160 char *RealPath = NULL;
b2e465d6
AL		 161 /* If noprefix is set then jam the scan root in, so we don't generate
162 link followed paths out of control */
163 if (InternalPrefix.empty() == true)
164 {
ab3846c0 165 if ((RealPath = realpath(Dir.c_str(),NULL)) == 0)
dc738e7a 166 return _error->Errno("realpath",_("Failed to resolve %s"),Dir.c_str());
ab3846c0
MV		 167 InternalPrefix = RealPath;
168 free(RealPath);
b2e465d6
AL		 169 }
170
171 // Do recursive directory searching
172 Owner = this;
9209ec47 173 int const Res = ftw(Dir.c_str(),ScannerFTW,30);
b2e465d6
AL		 174
175 // Error treewalking?
176 if (Res != 0)
177 {
178 if (_error->PendingError() == false)
dc738e7a 179 _error->Errno("ftw",_("Tree walking failed"));
b2e465d6
AL		 180 return false;
181 }
182
183 return true;
184}
185 /*}}}*/
186// FTWScanner::LoadFileList - Load the file list from a file /*{{{*/
187// ---------------------------------------------------------------------
188/* This is an alternative to using FTW to locate files, it reads the list
189 of files from another file. */
9209ec47 190bool FTWScanner::LoadFileList(string const &Dir, string const &File)
b2e465d6 191{
ab3846c0 192 char *RealPath = NULL;
b2e465d6
AL		 193 /* If noprefix is set then jam the scan root in, so we don't generate
194 link followed paths out of control */
195 if (InternalPrefix.empty() == true)
196 {
ab3846c0 197 if ((RealPath = realpath(Dir.c_str(),NULL)) == 0)
dc738e7a 198 return _error->Errno("realpath",_("Failed to resolve %s"),Dir.c_str());
b2e465d6 199 InternalPrefix = RealPath;
ab3846c0 200 free(RealPath);
b2e465d6
AL		 201 }
202
203 Owner = this;
204 FILE *List = fopen(File.c_str(),"r");
205 if (List == 0)
dc738e7a 206 return _error->Errno("fopen",_("Failed to open %s"),File.c_str());
b2e465d6
AL		 207
208 /* We are a tad tricky here.. We prefix the buffer with the directory
209 name, that way if we need a full path with just use line.. Sneaky and
210 fully evil. */
211 char Line[1000];
212 char *FileStart;
213 if (Dir.empty() == true || Dir.end()[-1] != '/')
214 FileStart = Line + snprintf(Line,sizeof(Line),"%s/",Dir.c_str());
215 else
216 FileStart = Line + snprintf(Line,sizeof(Line),"%s",Dir.c_str());
217 while (fgets(FileStart,sizeof(Line) - (FileStart - Line),List) != 0)
218 {
219 char *FileName = _strstrip(FileStart);
220 if (FileName[0] == 0)
221 continue;
222
223 if (FileName[0] != '/')
224 {
225 if (FileName != FileStart)
226 memmove(FileStart,FileName,strlen(FileStart));
227 FileName = Line;
228 }
229
cde41ae8 230#if 0
b2e465d6
AL		 231 struct stat St;
232 int Flag = FTW_F;
233 if (stat(FileName,&St) != 0)
234 Flag = FTW_NS;
cde41ae8 235#endif
b2e465d6 236
cde41ae8 237 if (ScannerFile(FileName, false) != 0)
b2e465d6
AL		 238 break;
239 }
240
241 fclose(List);
242 return true;
243}
244 /*}}}*/
245// FTWScanner::Delink - Delink symlinks /*{{{*/
246// ---------------------------------------------------------------------
247/* */
248bool FTWScanner::Delink(string &FileName,const char *OriginalPath,
249 unsigned long &DeLinkBytes,
9209ec47 250 off_t const &FileSize)
b2e465d6
AL		 251{
252 // See if this isn't an internaly prefix'd file name.
253 if (InternalPrefix.empty() == false &&
254 InternalPrefix.length() < FileName.length() &&
255 stringcmp(FileName.begin(),FileName.begin() + InternalPrefix.length(),
256 InternalPrefix.begin(),InternalPrefix.end()) != 0)
257 {
258 if (DeLinkLimit != 0 && DeLinkBytes/1024 < DeLinkLimit)
259 {
260 // Tidy up the display
261 if (DeLinkBytes == 0)
262 cout << endl;
263
264 NewLine(1);
dc738e7a 265 ioprintf(c1out, _(" DeLink %s [%s]\n"), (OriginalPath + InternalPrefix.length()),
cde41ae8 266 SizeToStr(FileSize).c_str());
dc738e7a 267 c1out << flush;
b2e465d6
AL		 268
269 if (NoLinkAct == false)
270 {
271 char OldLink[400];
272 if (readlink(OriginalPath,OldLink,sizeof(OldLink)) == -1)
dc738e7a 273 _error->Errno("readlink",_("Failed to readlink %s"),OriginalPath);
b2e465d6
AL		 274 else
275 {
276 if (unlink(OriginalPath) != 0)
dc738e7a 277 _error->Errno("unlink",_("Failed to unlink %s"),OriginalPath);
b2e465d6
AL		 278 else
279 {
280 if (link(FileName.c_str(),OriginalPath) != 0)
281 {
282 // Panic! Restore the symlink
283 symlink(OldLink,OriginalPath);
dc738e7a 284 return _error->Errno("link",_("*** Failed to link %s to %s"),
b2e465d6
AL		 285 FileName.c_str(),
286 OriginalPath);
287 }
288 }
289 }
290 }
291
cde41ae8 292 DeLinkBytes += FileSize;
b2e465d6 293 if (DeLinkBytes/1024 >= DeLinkLimit)
dc738e7a 294 ioprintf(c1out, _(" DeLink limit of %sB hit.\n"), SizeToStr(DeLinkBytes).c_str());
b2e465d6
AL		 295 }
296
297 FileName = OriginalPath;
298 }
299
300 return true;
301}
302 /*}}}*/
b2e465d6
AL		 303
304// PackagesWriter::PackagesWriter - Constructor /*{{{*/
305// ---------------------------------------------------------------------
306/* */
9209ec47 307PackagesWriter::PackagesWriter(string const &DB,string const &Overrides,string const &ExtOverrides,
31981076 308 string const &Arch) :
66905344 309 FTWScanner(Arch), Db(DB), Stats(Db.Stats), TransWriter(NULL)
b2e465d6
AL		 310{
311 Output = stdout;
31981076 312 SetExts(".deb .udeb");
b2e465d6 313 DeLinkLimit = 0;
3cb3fe76 314
b2e465d6 315 // Process the command line options
3c54407f
DK		 316 DoMD5 = _config->FindB("APT::FTPArchive::Packages::MD5",DoMD5);
317 DoSHA1 = _config->FindB("APT::FTPArchive::Packages::SHA1",DoSHA1);
318 DoSHA256 = _config->FindB("APT::FTPArchive::Packages::SHA256",DoSHA256);
12cd178d 319 DoSHA256 = _config->FindB("APT::FTPArchive::Packages::SHA512",true);
ff574e76 320 DoAlwaysStat = _config->FindB("APT::FTPArchive::AlwaysStat", false);
b2e465d6
AL		 321 DoContents = _config->FindB("APT::FTPArchive::Contents",true);
322 NoOverride = _config->FindB("APT::FTPArchive::NoOverrideMsg",false);
9c24493f 323 LongDescription = _config->FindB("APT::FTPArchive::LongDescription",true);
b2e465d6
AL		 324
325 if (Db.Loaded() == false)
326 DoContents = false;
66905344 327
b2e465d6
AL		 328 // Read the override file
329 if (Overrides.empty() == false && Over.ReadOverride(Overrides) == false)
330 return;
331 else
332 NoOverride = true;
64177f17
AL		 333
334 if (ExtOverrides.empty() == false)
335 Over.ReadExtraOverride(ExtOverrides);
336
b2e465d6
AL		 337 _error->DumpErrors();
338}
98953965
AL		 339 /*}}}*/
340// FTWScanner::SetExts - Set extensions to support /*{{{*/
341// ---------------------------------------------------------------------
342/* */
9209ec47 343bool FTWScanner::SetExts(string const &Vals)
98953965
AL		 344{
345 ClearPatterns();
346 string::size_type Start = 0;
d6689735 347 while (Start <= Vals.length()-1)
98953965 348 {
31981076
DK		 349 string::size_type const Space = Vals.find(' ',Start);
350 string::size_type const Length = ((Space == string::npos) ? Vals.length() : Space) - Start;
351 if ( Arch.empty() == false )
98953965 352 {
31981076
DK		 353 AddPattern(string("*_") + Arch + Vals.substr(Start, Length));
354 AddPattern(string("*_all") + Vals.substr(Start, Length));
98953965 355 }
d6689735 356 else
31981076
DK		 357 AddPattern(string("*") + Vals.substr(Start, Length));
358
d6689735 359 Start += Length + 1;
98953965
AL		 360 }
361
362 return true;
363}
364
b2e465d6
AL		 365 /*}}}*/
366// PackagesWriter::DoPackage - Process a single package /*{{{*/
367// ---------------------------------------------------------------------
368/* This method takes a package and gets its control information and
cde41ae8
MV		 369 MD5, SHA1 and SHA256 then writes out a control record with the proper fields
370 rewritten and the path/size/hash appended. */
b2e465d6
AL		 371bool PackagesWriter::DoPackage(string FileName)
372{
b2e465d6 373 // Pull all the data we need form the DB
9a961efc 374 if (Db.GetFileInfo(FileName, true, DoContents, true, DoMD5, DoSHA1, DoSHA256, DoSHA512, DoAlwaysStat)
cde41ae8
MV		 375 == false)
376 {
b2e465d6 377 return false;
cde41ae8 378 }
b2e465d6 379
cde41ae8
MV		 380 off_t FileSize = Db.GetFileSize();
381 if (Delink(FileName,OriginalPath,Stats.DeLinkBytes,FileSize) == false)
b2e465d6
AL		 382 return false;
383
384 // Lookup the overide information
385 pkgTagSection &Tags = Db.Control.Section;
386 string Package = Tags.FindS("Package");
0b41e0e7
MV		 387 string Architecture;
388 // if we generate a Packages file for a given arch, we use it to
389 // look for overrides. if we run in "simple" mode without the
390 // "Architecures" variable in the config we use the architecure value
391 // from the deb file
392 if(Arch != "")
393 Architecture = Arch;
394 else
395 Architecture = Tags.FindS("Architecture");
396 auto_ptr<Override::Item> OverItem(Over.GetItem(Package,Architecture));
b2e465d6
AL		 397
398 if (Package.empty() == true)
dc738e7a 399 return _error->Error(_("Archive had no package field"));
0b41e0e7 400
b2e465d6 401 // If we need to do any rewriting of the header do it now..
0b41e0e7 402 if (OverItem.get() == 0)
b2e465d6
AL		 403 {
404 if (NoOverride == false)
405 {
406 NewLine(1);
dc738e7a 407 ioprintf(c1out, _(" %s has no override entry\n"), Package.c_str());
b2e465d6
AL		 408 }
409
0b41e0e7
MV		 410 OverItem = auto_ptr<Override::Item>(new Override::Item);
411 OverItem->FieldOverride["Section"] = Tags.FindS("Section");
412 OverItem->Priority = Tags.FindS("Priority");
b2e465d6
AL		 413 }
414
415 char Size[40];
cde41ae8 416 sprintf(Size,"%lu", (unsigned long) FileSize);
b2e465d6
AL		 417
418 // Strip the DirStrip prefix from the FileName and add the PathPrefix
419 string NewFileName;
420 if (DirStrip.empty() == false &&
421 FileName.length() > DirStrip.length() &&
422 stringcmp(FileName.begin(),FileName.begin() + DirStrip.length(),
423 DirStrip.begin(),DirStrip.end()) == 0)
424 NewFileName = string(FileName.begin() + DirStrip.length(),FileName.end());
425 else
426 NewFileName = FileName;
427 if (PathPrefix.empty() == false)
428 NewFileName = flCombine(PathPrefix,NewFileName);
9c24493f
DK		 429
430 /* Configuration says we don't want to include the long Description
431 in the package file - instead we want to ship a separated file */
432 string desc;
433 if (LongDescription == false) {
434 desc = Tags.FindS("Description").append("\n");
435 OverItem->FieldOverride["Description"] = desc.substr(0, desc.find('\n')).c_str();
436 }
437
b2e465d6 438 // This lists all the changes to the fields we are going to make.
64177f17 439 // (7 hardcoded + maintainer + suggests + end marker)
9c24493f 440 TFRewriteData Changes[6+2+OverItem->FieldOverride.size()+1+1];
64177f17 441
b2e465d6 442 unsigned int End = 0;
64177f17 443 SetTFRewriteData(Changes[End++], "Size", Size);
3c54407f
DK		 444 if (DoMD5 == true)
445 SetTFRewriteData(Changes[End++], "MD5sum", Db.MD5Res.c_str());
446 if (DoSHA1 == true)
447 SetTFRewriteData(Changes[End++], "SHA1", Db.SHA1Res.c_str());
448 if (DoSHA256 == true)
449 SetTFRewriteData(Changes[End++], "SHA256", Db.SHA256Res.c_str());
12cd178d
MV		 450 if (DoSHA512 == true)
451 SetTFRewriteData(Changes[End++], "SHA512", Db.SHA512Res.c_str());
64177f17
AL		 452 SetTFRewriteData(Changes[End++], "Filename", NewFileName.c_str());
453 SetTFRewriteData(Changes[End++], "Priority", OverItem->Priority.c_str());
454 SetTFRewriteData(Changes[End++], "Status", 0);
455 SetTFRewriteData(Changes[End++], "Optional", 0);
456
9c24493f
DK		 457 string DescriptionMd5;
458 if (LongDescription == false) {
459 MD5Summation descmd5;
460 descmd5.Add(desc.c_str());
461 DescriptionMd5 = descmd5.Result().Value();
462 SetTFRewriteData(Changes[End++], "Description-md5", DescriptionMd5.c_str());
66905344
DK		 463 if (TransWriter != NULL)
464 TransWriter->DoPackage(Package, desc, DescriptionMd5);
9c24493f
DK		 465 }
466
b2e465d6
AL		 467 // Rewrite the maintainer field if necessary
468 bool MaintFailed;
469 string NewMaint = OverItem->SwapMaint(Tags.FindS("Maintainer"),MaintFailed);
470 if (MaintFailed == true)
471 {
472 if (NoOverride == false)
473 {
474 NewLine(1);
dc738e7a
AL		 475 ioprintf(c1out, _(" %s maintainer is %s not %s\n"),
476 Package.c_str(), Tags.FindS("Maintainer").c_str(), OverItem->OldMaint.c_str());
b2e465d6
AL		 477 }
478 }
479
480 if (NewMaint.empty() == false)
64177f17 481 SetTFRewriteData(Changes[End++], "Maintainer", NewMaint.c_str());
b2e465d6
AL		 482
483 /* Get rid of the Optional tag. This is an ugly, ugly, ugly hack that
c6474fb6 484 dpkg-scanpackages does. Well sort of. dpkg-scanpackages just does renaming
b2e465d6
AL		 485 but dpkg does this append bit. So we do the append bit, at least that way the
486 status file and package file will remain similar. There are other transforms
487 but optional is the only legacy one still in use for some lazy reason. */
488 string OptionalStr = Tags.FindS("Optional");
489 if (OptionalStr.empty() == false)
490 {
491 if (Tags.FindS("Suggests").empty() == false)
492 OptionalStr = Tags.FindS("Suggests") + ", " + OptionalStr;
64177f17 493 SetTFRewriteData(Changes[End++], "Suggests", OptionalStr.c_str());
b2e465d6 494 }
64177f17 495
9209ec47 496 for (map<string,string>::const_iterator I = OverItem->FieldOverride.begin();
64177f17
AL		 497 I != OverItem->FieldOverride.end(); I++)
498 SetTFRewriteData(Changes[End++],I->first.c_str(),I->second.c_str());
499
500 SetTFRewriteData(Changes[End++], 0, 0);
501
b2e465d6
AL		 502 // Rewrite and store the fields.
503 if (TFRewrite(Output,Tags,TFRewritePackageOrder,Changes) == false)
504 return false;
505 fprintf(Output,"\n");
506
507 return Db.Finish();
508}
509 /*}}}*/
510
66905344
DK		 511// TranslationWriter::TranslationWriter - Constructor /*{{{*/
512// ---------------------------------------------------------------------
513/* Create a Translation-Master file for this Packages file */
34f1d96c
DK		 514TranslationWriter::TranslationWriter(string const &File, string const &TransCompress,
515 mode_t const &Permissions) : Output(NULL),
66905344
DK		 516 RefCounter(0)
517{
518 if (File.empty() == true)
519 return;
520
34f1d96c
DK		 521 Comp = new MultiCompress(File, TransCompress, Permissions);
522 Output = Comp->Input;
66905344
DK		 523}
524 /*}}}*/
525// TranslationWriter::DoPackage - Process a single package /*{{{*/
526// ---------------------------------------------------------------------
527/* Create a Translation-Master file for this Packages file */
528bool TranslationWriter::DoPackage(string const &Pkg, string const &Desc,
529 string const &MD5)
530{
531 if (Output == NULL)
532 return true;
533
534 // Different archs can include different versions and therefore
535 // different descriptions - so we need to check for both name and md5.
536 string const Record = Pkg + ":" + MD5;
537
538 if (Included.find(Record) != Included.end())
539 return true;
540
541 fprintf(Output, "Package: %s\nDescription-md5: %s\nDescription-en: %s\n",
542 Pkg.c_str(), MD5.c_str(), Desc.c_str());
543
544 Included.insert(Record);
545 return true;
546}
547 /*}}}*/
548// TranslationWriter::~TranslationWriter - Destructor /*{{{*/
549// ---------------------------------------------------------------------
550/* */
551TranslationWriter::~TranslationWriter()
552{
34f1d96c
DK		 553 if (Comp == NULL)
554 return;
555
556 delete Comp;
66905344
DK		 557}
558 /*}}}*/
559
b2e465d6
AL		 560// SourcesWriter::SourcesWriter - Constructor /*{{{*/
561// ---------------------------------------------------------------------
562/* */
9209ec47
DK		 563SourcesWriter::SourcesWriter(string const &BOverrides,string const &SOverrides,
564 string const &ExtOverrides)
b2e465d6
AL		 565{
566 Output = stdout;
98953965 567 AddPattern("*.dsc");
b2e465d6
AL		 568 DeLinkLimit = 0;
569 Buffer = 0;
570 BufSize = 0;
571
572 // Process the command line options
3c54407f
DK		 573 DoMD5 = _config->FindB("APT::FTPArchive::Sources::MD5",DoMD5);
574 DoSHA1 = _config->FindB("APT::FTPArchive::Sources::SHA1",DoSHA1);
575 DoSHA256 = _config->FindB("APT::FTPArchive::Sources::SHA256",DoSHA256);
b2e465d6
AL		 576 NoOverride = _config->FindB("APT::FTPArchive::NoOverrideMsg",false);
577
578 // Read the override file
579 if (BOverrides.empty() == false && BOver.ReadOverride(BOverrides) == false)
580 return;
581 else
582 NoOverride = true;
64177f17 583
cde41ae8
MV		 584 // WTF?? The logic above: if we can't read binary overrides, don't even try
585 // reading source overrides. if we can read binary overrides, then say there
586 // are no overrides. THIS MAKES NO SENSE! -- ajt@d.o, 2006/02/28
587
64177f17
AL		 588 if (ExtOverrides.empty() == false)
589 SOver.ReadExtraOverride(ExtOverrides);
b2e465d6 590
64177f17
AL		 591 if (SOverrides.empty() == false && FileExists(SOverrides) == true)
592 SOver.ReadOverride(SOverrides,true);
b2e465d6
AL		 593}
594 /*}}}*/
595// SourcesWriter::DoPackage - Process a single package /*{{{*/
596// ---------------------------------------------------------------------
597/* */
598bool SourcesWriter::DoPackage(string FileName)
599{
600 // Open the archive
601 FileFd F(FileName,FileFd::ReadOnly);
602 if (_error->PendingError() == true)
603 return false;
604
605 // Stat the file for later
606 struct stat St;
607 if (fstat(F.Fd(),&St) != 0)
608 return _error->Errno("fstat","Failed to stat %s",FileName.c_str());
609
610 if (St.st_size > 128*1024)
611 return _error->Error("DSC file '%s' is too large!",FileName.c_str());
612
613 if (BufSize < (unsigned)St.st_size+1)
614 {
615 BufSize = St.st_size+1;
616 Buffer = (char *)realloc(Buffer,St.st_size+1);
617 }
618
619 if (F.Read(Buffer,St.st_size) == false)
620 return false;
621
622 // Hash the file
623 char *Start = Buffer;
624 char *BlkEnd = Buffer + St.st_size;
f99da908 625
3c54407f 626 MD5Summation MD5;
f99da908
DK		 627 SHA1Summation SHA1;
628 SHA256Summation SHA256;
12cd178d 629 SHA256Summation SHA512;
3c54407f
DK		 630
631 if (DoMD5 == true)
632 MD5.Add((unsigned char *)Start,BlkEnd - Start);
633 if (DoSHA1 == true)
634 SHA1.Add((unsigned char *)Start,BlkEnd - Start);
635 if (DoSHA256 == true)
636 SHA256.Add((unsigned char *)Start,BlkEnd - Start);
12cd178d
MV		 637 if (DoSHA512 == true)
638 SHA512.Add((unsigned char *)Start,BlkEnd - Start);
f99da908 639
b2e465d6
AL		 640 // Add an extra \n to the end, just in case
641 *BlkEnd++ = '\n';
642
643 /* Remove the PGP trailer. Some .dsc's have this without a blank line
644 before */
645 const char *Key = "-----BEGIN PGP SIGNATURE-----";
646 for (char *MsgEnd = Start; MsgEnd < BlkEnd - strlen(Key) -1; MsgEnd++)
647 {
648 if (*MsgEnd == '\n' && strncmp(MsgEnd+1,Key,strlen(Key)) == 0)
649 {
650 MsgEnd[1] = '\n';
651 break;
652 }
653 }
654
655 /* Read records until we locate the Source record. This neatly skips the
656 GPG header (which is RFC822 formed) without any trouble. */
657 pkgTagSection Tags;
658 do
659 {
660 unsigned Pos;
661 if (Tags.Scan(Start,BlkEnd - Start) == false)
662 return _error->Error("Could not find a record in the DSC '%s'",FileName.c_str());
663 if (Tags.Find("Source",Pos) == true)
664 break;
665 Start += Tags.size();
666 }
667 while (1);
668 Tags.Trim();
669
670 // Lookup the overide information, finding first the best priority.
671 string BestPrio;
b2e465d6 672 string Bins = Tags.FindS("Binary");
5200ec6f 673 char Buffer[Bins.length() + 1];
0b41e0e7 674 auto_ptr<Override::Item> OverItem(0);
5200ec6f 675 if (Bins.empty() == false)
b2e465d6
AL		 676 {
677 strcpy(Buffer,Bins.c_str());
678
679 // Ignore too-long errors.
680 char *BinList[400];
681 TokSplitString(',',Buffer,BinList,sizeof(BinList)/sizeof(BinList[0]));
682
683 // Look at all the binaries
684 unsigned char BestPrioV = pkgCache::State::Extra;
685 for (unsigned I = 0; BinList[I] != 0; I++)
686 {
0b41e0e7
MV		 687 auto_ptr<Override::Item> Itm(BOver.GetItem(BinList[I]));
688 if (Itm.get() == 0)
b2e465d6 689 continue;
b2e465d6
AL		 690
691 unsigned char NewPrioV = debListParser::GetPrio(Itm->Priority);
692 if (NewPrioV < BestPrioV || BestPrio.empty() == true)
693 {
694 BestPrioV = NewPrioV;
695 BestPrio = Itm->Priority;
696 }
7524e348
MV		 697
698 if (OverItem.get() == 0)
699 OverItem = Itm;
b2e465d6
AL		 700 }
701 }
702
703 // If we need to do any rewriting of the header do it now..
0b41e0e7 704 if (OverItem.get() == 0)
b2e465d6
AL		 705 {
706 if (NoOverride == false)
707 {
708 NewLine(1);
dc738e7a 709 ioprintf(c1out, _(" %s has no override entry\n"), Tags.FindS("Source").c_str());
b2e465d6
AL		 710 }
711
0b41e0e7 712 OverItem = auto_ptr<Override::Item>(new Override::Item);
b2e465d6
AL		 713 }
714
0b41e0e7 715 auto_ptr<Override::Item> SOverItem(SOver.GetItem(Tags.FindS("Source")));
cde41ae8 716 // const auto_ptr<Override::Item> autoSOverItem(SOverItem);
0b41e0e7 717 if (SOverItem.get() == 0)
b2e465d6 718 {
cde41ae8 719 ioprintf(c1out, _(" %s has no source override entry\n"), Tags.FindS("Source").c_str());
0b41e0e7
MV		 720 SOverItem = auto_ptr<Override::Item>(BOver.GetItem(Tags.FindS("Source")));
721 if (SOverItem.get() == 0)
722 {
cde41ae8 723 ioprintf(c1out, _(" %s has no binary override entry either\n"), Tags.FindS("Source").c_str());
0b41e0e7
MV		 724 SOverItem = auto_ptr<Override::Item>(new Override::Item);
725 *SOverItem = *OverItem;
726 }
b2e465d6
AL		 727 }
728
729 // Add the dsc to the files hash list
f99da908 730 string const strippedName = flNotDir(FileName);
bf99a6d3 731 std::ostringstream ostreamFiles;
3c54407f 732 if (DoMD5 == true && Tags.Exists("Files"))
2a19ec29
MV		 733 ostreamFiles << "\n " << string(MD5.Result()) << " " << St.st_size << " "
734 << strippedName << "\n " << Tags.FindS("Files");
bf99a6d3
DK		 735 string const Files = ostreamFiles.str();
736
737 std::ostringstream ostreamSha1;
3c54407f 738 if (DoSHA1 == true && Tags.Exists("Checksums-Sha1"))
2a19ec29
MV		 739 ostreamSha1 << "\n " << string(SHA1.Result()) << " " << St.st_size << " "
740 << strippedName << "\n " << Tags.FindS("Checksums-Sha1");
bf99a6d3
DK		 741 string const ChecksumsSha1 = ostreamSha1.str();
742
743 std::ostringstream ostreamSha256;
3c54407f 744 if (DoSHA256 == true && Tags.Exists("Checksums-Sha256"))
2a19ec29
MV		 745 ostreamSha256 << "\n " << string(SHA256.Result()) << " " << St.st_size << " "
746 << strippedName << "\n " << Tags.FindS("Checksums-Sha256");
bf99a6d3 747 string const ChecksumsSha256 = ostreamSha256.str();
f99da908 748
9a961efc
MV		 749 std::ostringstream ostreamSha512;
750 if (Tags.Exists("Checksums-Sha512"))
751 ostreamSha512 << "\n " << string(SHA512.Result()) << " " << St.st_size << " "
752 << strippedName << "\n " << Tags.FindS("Checksums-Sha512");
753 string const ChecksumsSha512 = ostreamSha512.str();
754
b2e465d6
AL		 755 // Strip the DirStrip prefix from the FileName and add the PathPrefix
756 string NewFileName;
757 if (DirStrip.empty() == false &&
758 FileName.length() > DirStrip.length() &&
8c58f506 759 stringcmp(DirStrip,OriginalPath,OriginalPath + DirStrip.length()) == 0)
b2e465d6
AL		 760 NewFileName = string(OriginalPath + DirStrip.length());
761 else
762 NewFileName = OriginalPath;
763 if (PathPrefix.empty() == false)
764 NewFileName = flCombine(PathPrefix,NewFileName);
171c45bc 765
b2e465d6
AL		 766 string Directory = flNotFile(OriginalPath);
767 string Package = Tags.FindS("Source");
171c45bc 768
b2e465d6
AL		 769 // Perform the delinking operation over all of the files
770 string ParseJnk;
bf99a6d3 771 const char *C = Files.c_str();
ab3846c0 772 char *RealPath = NULL;
b2e465d6
AL		 773 for (;isspace(*C); C++);
774 while (*C != 0)
775 {
776 // Parse each of the elements
777 if (ParseQuoteWord(C,ParseJnk) == false ||
778 ParseQuoteWord(C,ParseJnk) == false ||
779 ParseQuoteWord(C,ParseJnk) == false)
780 return _error->Error("Error parsing file record");
781
782 char Jnk[2];
783 string OriginalPath = Directory + ParseJnk;
ab3846c0
MV		 784 if (readlink(OriginalPath.c_str(),Jnk,sizeof(Jnk)) != -1 &&
785 (RealPath = realpath(OriginalPath.c_str(),NULL)) != 0)
b2e465d6
AL		 786 {
787 string RP = RealPath;
ab3846c0 788 free(RealPath);
cde41ae8 789 if (Delink(RP,OriginalPath.c_str(),Stats.DeLinkBytes,St.st_size) == false)
b2e465d6
AL		 790 return false;
791 }
792 }
793
794 Directory = flNotFile(NewFileName);
795 if (Directory.length() > 2)
796 Directory.erase(Directory.end()-1);
171c45bc 797
b2e465d6 798 // This lists all the changes to the fields we are going to make.
f99da908
DK		 799 // (5 hardcoded + checksums + maintainer + end marker)
800 TFRewriteData Changes[5+2+1+SOverItem->FieldOverride.size()+1];
64177f17 801
b2e465d6 802 unsigned int End = 0;
64177f17 803 SetTFRewriteData(Changes[End++],"Source",Package.c_str(),"Package");
3c54407f
DK		 804 if (Files.empty() == false)
805 SetTFRewriteData(Changes[End++],"Files",Files.c_str());
806 if (ChecksumsSha1.empty() == false)
807 SetTFRewriteData(Changes[End++],"Checksums-Sha1",ChecksumsSha1.c_str());
808 if (ChecksumsSha256.empty() == false)
809 SetTFRewriteData(Changes[End++],"Checksums-Sha256",ChecksumsSha256.c_str());
12cd178d
MV		 810 if (ChecksumsSha512.empty() == false)
811 SetTFRewriteData(Changes[End++],"Checksums-Sha512",ChecksumsSha512.c_str());
171c45bc
AL		 812 if (Directory != "./")
813 SetTFRewriteData(Changes[End++],"Directory",Directory.c_str());
64177f17
AL		 814 SetTFRewriteData(Changes[End++],"Priority",BestPrio.c_str());
815 SetTFRewriteData(Changes[End++],"Status",0);
b2e465d6
AL		 816
817 // Rewrite the maintainer field if necessary
818 bool MaintFailed;
819 string NewMaint = OverItem->SwapMaint(Tags.FindS("Maintainer"),MaintFailed);
820 if (MaintFailed == true)
821 {
822 if (NoOverride == false)
823 {
824 NewLine(1);
dc738e7a
AL		 825 ioprintf(c1out, _(" %s maintainer is %s not %s\n"), Package.c_str(),
826 Tags.FindS("Maintainer").c_str(), OverItem->OldMaint.c_str());
b2e465d6
AL		 827 }
828 }
829 if (NewMaint.empty() == false)
64177f17
AL		 830 SetTFRewriteData(Changes[End++], "Maintainer", NewMaint.c_str());
831
9209ec47 832 for (map<string,string>::const_iterator I = SOverItem->FieldOverride.begin();
64177f17
AL		 833 I != SOverItem->FieldOverride.end(); I++)
834 SetTFRewriteData(Changes[End++],I->first.c_str(),I->second.c_str());
835
836 SetTFRewriteData(Changes[End++], 0, 0);
b2e465d6
AL		 837
838 // Rewrite and store the fields.
839 if (TFRewrite(Output,Tags,TFRewriteSourceOrder,Changes) == false)
840 return false;
841 fprintf(Output,"\n");
842
843 Stats.Packages++;
844
845 return true;
846}
847 /*}}}*/
848
849// ContentsWriter::ContentsWriter - Constructor /*{{{*/
850// ---------------------------------------------------------------------
851/* */
31981076
DK		 852ContentsWriter::ContentsWriter(string const &DB, string const &Arch) :
853 FTWScanner(Arch), Db(DB), Stats(Db.Stats)
b2e465d6
AL		 854
855{
31981076 856 SetExts(".deb");
b2e465d6
AL		 857 Output = stdout;
858}
859 /*}}}*/
860// ContentsWriter::DoPackage - Process a single package /*{{{*/
861// ---------------------------------------------------------------------
862/* If Package is the empty string the control record will be parsed to
863 determine what the package name is. */
9209ec47 864bool ContentsWriter::DoPackage(string FileName, string Package)
b2e465d6 865{
ff574e76 866 if (!Db.GetFileInfo(FileName, Package.empty(), true, false, false, false, false, false))
cde41ae8 867 {
b2e465d6 868 return false;
cde41ae8 869 }
b2e465d6
AL		 870
871 // Parse the package name
872 if (Package.empty() == true)
873 {
b2e465d6
AL		 874 Package = Db.Control.Section.FindS("Package");
875 }
876
877 Db.Contents.Add(Gen,Package);
878
879 return Db.Finish();
880}
881 /*}}}*/
882// ContentsWriter::ReadFromPkgs - Read from a packages file /*{{{*/
883// ---------------------------------------------------------------------
884/* */
9209ec47 885bool ContentsWriter::ReadFromPkgs(string const &PkgFile,string const &PkgCompress)
b2e465d6
AL		 886{
887 MultiCompress Pkgs(PkgFile,PkgCompress,0,false);
888 if (_error->PendingError() == true)
889 return false;
890
891 // Open the package file
892 int CompFd = -1;
b3d44315 893 pid_t Proc = -1;
b2e465d6
AL		 894 if (Pkgs.OpenOld(CompFd,Proc) == false)
895 return false;
896
897 // No auto-close FD
898 FileFd Fd(CompFd,false);
899 pkgTagFile Tags(&Fd);
900 if (_error->PendingError() == true)
901 {
902 Pkgs.CloseOld(CompFd,Proc);
903 return false;
904 }
905
906 // Parse.
907 pkgTagSection Section;
908 while (Tags.Step(Section) == true)
909 {
910 string File = flCombine(Prefix,Section.FindS("FileName"));
911 string Package = Section.FindS("Section");
912 if (Package.empty() == false && Package.end()[-1] != '/')
913 {
914 Package += '/';
915 Package += Section.FindS("Package");
916 }
917 else
918 Package += Section.FindS("Package");
919
920 DoPackage(File,Package);
921 if (_error->empty() == false)
922 {
923 _error->Error("Errors apply to file '%s'",File.c_str());
924 _error->DumpErrors();
925 }
926 }
927
928 // Tidy the compressor
929 if (Pkgs.CloseOld(CompFd,Proc) == false)
930 return false;
931
932 return true;
933}
98953965 934
b2e465d6 935 /*}}}*/
98953965
AL		 936
937// ReleaseWriter::ReleaseWriter - Constructor /*{{{*/
938// ---------------------------------------------------------------------
939/* */
9209ec47 940ReleaseWriter::ReleaseWriter(string const &DB)
98953965 941{
3cb3fe76
DK		 942 if (_config->FindB("APT::FTPArchive::Release::Default-Patterns", true) == true)
943 {
944 AddPattern("Packages");
945 AddPattern("Packages.gz");
946 AddPattern("Packages.bz2");
947 AddPattern("Packages.lzma");
b7080ced 948 AddPattern("Packages.xz");
3cb3fe76
DK		 949 AddPattern("Sources");
950 AddPattern("Sources.gz");
951 AddPattern("Sources.bz2");
952 AddPattern("Sources.lzma");
b7080ced 953 AddPattern("Sources.xz");
3cb3fe76 954 AddPattern("Release");
8d16c617 955 AddPattern("Index");
3cb3fe76
DK		 956 AddPattern("md5sum.txt");
957 }
958 AddPatterns(_config->FindVector("APT::FTPArchive::Release::Patterns"));
187492a6 959
98953965 960 Output = stdout;
9209ec47 961 time_t const now = time(NULL);
cb12d0a6
DK		 962
963 setlocale(LC_TIME, "C");
964
98953965
AL		 965 char datestr[128];
966 if (strftime(datestr, sizeof(datestr), "%a, %d %b %Y %H:%M:%S UTC",
967 gmtime(&now)) == 0)
968 {
969 datestr[0] = '\0';
970 }
971
c99e48ec 972 time_t const validuntil = now + _config->FindI("APT::FTPArchive::Release::ValidTime", 0);
cdb623ed 973 char validstr[128];
c99e48ec
DK		 974 if (now == validuntil ||
975 strftime(validstr, sizeof(validstr), "%a, %d %b %Y %H:%M:%S UTC",
976 gmtime(&validuntil)) == 0)
977 {
cdb623ed 978 validstr[0] = '\0';
c99e48ec
DK		 979 }
980
cb12d0a6
DK		 981 setlocale(LC_TIME, "");
982
98953965
AL		 983 map<string,string> Fields;
984 Fields["Origin"] = "";
985 Fields["Label"] = "";
986 Fields["Suite"] = "";
987 Fields["Version"] = "";
988 Fields["Codename"] = "";
989 Fields["Date"] = datestr;
c99e48ec 990 Fields["Valid-Until"] = validstr;
98953965
AL		 991 Fields["Architectures"] = "";
992 Fields["Components"] = "";
993 Fields["Description"] = "";
994
995 for(map<string,string>::const_iterator I = Fields.begin();
996 I != Fields.end();
997 ++I)
998 {
999 string Config = string("APT::FTPArchive::Release::") + (*I).first;
1000 string Value = _config->Find(Config, (*I).second.c_str());
1001 if (Value == "")
1002 continue;
1003
1004 fprintf(Output, "%s: %s\n", (*I).first.c_str(), Value.c_str());
1005 }
3c54407f
DK		 1006
1007 DoMD5 = _config->FindB("APT::FTPArchive::Release::MD5",DoMD5);
1008 DoSHA1 = _config->FindB("APT::FTPArchive::Release::SHA1",DoSHA1);
1009 DoSHA256 = _config->FindB("APT::FTPArchive::Release::SHA256",DoSHA256);
98953965
AL		 1010}
1011 /*}}}*/
1012// ReleaseWriter::DoPackage - Process a single package /*{{{*/
1013// ---------------------------------------------------------------------
1014bool ReleaseWriter::DoPackage(string FileName)
1015{
1016 // Strip the DirStrip prefix from the FileName and add the PathPrefix
1017 string NewFileName;
1018 if (DirStrip.empty() == false &&
1019 FileName.length() > DirStrip.length() &&
1020 stringcmp(FileName.begin(),FileName.begin() + DirStrip.length(),
1021 DirStrip.begin(),DirStrip.end()) == 0)
c0eb6bc6 1022 {
98953965 1023 NewFileName = string(FileName.begin() + DirStrip.length(),FileName.end());
c0eb6bc6 1024 while (NewFileName[0] == '/')
9202409d 1025 NewFileName = string(NewFileName.begin() + 1,NewFileName.end());
c0eb6bc6 1026 }
98953965
AL		 1027 else
1028 NewFileName = FileName;
c0eb6bc6 1029
98953965
AL		 1030 if (PathPrefix.empty() == false)
1031 NewFileName = flCombine(PathPrefix,NewFileName);
1032
1033 FileFd fd(FileName, FileFd::ReadOnly);
1034
1035 if (!fd.IsOpen())
1036 {
1037 return false;
1038 }
1039
c0eb6bc6 1040 CheckSums[NewFileName].size = fd.Size();
f7291f62 1041
8c4e1f97
DK		 1042 Hashes hs;
1043 hs.AddFD(fd.Fd(), 0, DoMD5, DoSHA1, DoSHA256, DoSHA512);
3c54407f 1044 if (DoMD5 == true)
8c4e1f97 1045 CheckSums[NewFileName].MD5 = hs.MD5.Result();
3c54407f 1046 if (DoSHA1 == true)
8c4e1f97 1047 CheckSums[NewFileName].SHA1 = hs.SHA1.Result();
3c54407f 1048 if (DoSHA256 == true)
8c4e1f97 1049 CheckSums[NewFileName].SHA256 = hs.SHA256.Result();
9abccf4a 1050 if (DoSHA512 == true)
8c4e1f97 1051 CheckSums[NewFileName].SHA512 = hs.SHA512.Result();
98953965 1052 fd.Close();
8c4e1f97 1053
98953965
AL		 1054 return true;
1055}
f7291f62
AL		 1056
1057 /*}}}*/
1058// ReleaseWriter::Finish - Output the checksums /*{{{*/
1059// ---------------------------------------------------------------------
1060void ReleaseWriter::Finish()
1061{
3c54407f 1062 if (DoMD5 == true)
f7291f62 1063 {
3c54407f
DK		 1064 fprintf(Output, "MD5Sum:\n");
1065 for(map<string,struct CheckSum>::const_iterator I = CheckSums.begin();
1066 I != CheckSums.end(); ++I)
1067 {
1068 fprintf(Output, " %s %16ld %s\n",
1069 (*I).second.MD5.c_str(),
1070 (*I).second.size,
1071 (*I).first.c_str());
1072 }
f7291f62 1073 }
3c54407f 1074 if (DoSHA1 == true)
f7291f62 1075 {
3c54407f
DK		 1076 fprintf(Output, "SHA1:\n");
1077 for(map<string,struct CheckSum>::const_iterator I = CheckSums.begin();
1078 I != CheckSums.end(); ++I)
1079 {
1080 fprintf(Output, " %s %16ld %s\n",
1081 (*I).second.SHA1.c_str(),
1082 (*I).second.size,
1083 (*I).first.c_str());
1084 }
f7291f62 1085 }
3c54407f 1086 if (DoSHA256 == true)
cde41ae8 1087 {
3c54407f
DK		 1088 fprintf(Output, "SHA256:\n");
1089 for(map<string,struct CheckSum>::const_iterator I = CheckSums.begin();
1090 I != CheckSums.end(); ++I)
1091 {
1092 fprintf(Output, " %s %16ld %s\n",
1093 (*I).second.SHA256.c_str(),
1094 (*I).second.size,
1095 (*I).first.c_str());
1096 }
cde41ae8 1097 }
9a961efc
MV		 1098
1099 fprintf(Output, "SHA512:\n");
1100 for(map<string,struct CheckSum>::const_iterator I = CheckSums.begin();
1101 I != CheckSums.end();
1102 ++I)
1103 {
1104 fprintf(Output, " %s %32ld %s\n",
1105 (*I).second.SHA512.c_str(),
1106 (*I).second.size,
1107 (*I).first.c_str());
1108 }
1109
f7291f62 1110}
ntk's bugfix for apt