Reorder redirects/release
[darcs-mirror-apt-repository-merge.git] / apt-repository-merge
1 #!/usr/bin/python
2
3 from optparse import OptionParser
4
5 import gzip
6 import deb822
7 import urllib2
8 import os
9 import os.path
10 import md5
11 import re
12
13 class Config(deb822._multivalued):
14     _multivalued_fields = {
15             "Repos": [ "Base", "Dist", "Section" ]
16             }
17
18 parser = OptionParser()
19 parser.add_option("-c", "--config", dest="config",
20                   help="Configuration to use")
21 (options, args) = parser.parse_args()
22
23 config = Config(file(options.config))
24
25 rewrite_rules_by_dir = {}
26
27 def add_rewrite(path, base):
28     dir = os.path.dirname(path)
29     file = os.path.basename(path)
30     rewrite_rules_by_dir.setdefault(dir,[])
31     rewrite_rules_by_dir[dir].append(( file, base ))
32
33 release_files = []
34
35 for arch in config["arches"].split():
36     print "Merging architecture %s" % arch
37
38     output_packages = {}
39
40     for repo in config["Repos"]:
41         url = "%s/dists/%s/%s/binary-%s/Packages.gz" % (
42                 repo["Base"], repo["Dist"], repo["Section"], arch
43                 )
44         safe_base = filter (lambda c: c.isalnum(), repo["base"])
45         cache = "%s/%s_%s_%s_%s_Packages.gz" % (
46                 config["Output"], safe_base, repo["Dist"], repo["Section"], arch
47                 )
48
49         print "Getting %s" % url
50         file(cache,'w').write( urllib2.urlopen(url).read())
51
52         packages = deb822.Packages.iter_paragraphs(gzip.GzipFile(cache))
53
54         for package in packages:
55             package_name = package["Package"]
56             if package_name not in output_packages:
57                 # print "Taking %s from %s" % (package_name,repo["Base"])
58
59                 output_packages[package_name] = ( repo, package,)
60
61
62     print "Writing Packages file for %s" % arch
63     package_file = "dists/%s/%s/binary-%s/Packages" % (
64             config["Dist"], config["Section"], arch)
65     package_out_file = "%s/%s" % ( config["Output"], package_file)
66     if not os.path.isdir(os.path.dirname(package_out_file)):
67         os.makedirs(os.path.dirname(package_out_file))
68     package_out = file(package_out_file,'w')
69     for repo, package in output_packages.values():
70         package_out.write(str(package))
71         package_out.write("\n")
72         add_rewrite(package["Filename"], repo["Base"])
73     package_out.close()     
74     gzip.GzipFile(package_out_file + ".gz", 'w').write(
75             file(package_out_file,'r').read()
76         )
77     release_files.append(package_file)
78     release_files.append(package_file + ".gz")
79
80 if True:    # to match indentation with above
81     print "Merging sources"
82
83     output_packages = {}
84
85     for repo in config["Repos"]:
86         url = "%s/dists/%s/%s/source/Sources.gz" % (
87                 repo["Base"], repo["Dist"], repo["Section"]
88                 )
89         safe_base = filter (lambda c: c.isalnum(), repo["Base"])
90         cache = "%s/%s_%s_%s_Sources.gz" % (
91                 config["Output"],safe_base, repo["Dist"], repo["Section"]
92                 )
93
94         print "Getting %s" % url
95         file(cache,'w').write( urllib2.urlopen(url).read())
96
97         packages = deb822.Sources.iter_paragraphs(gzip.GzipFile(cache))
98
99         for package in packages:
100             package_name = package["Package"]
101             if package_name not in output_packages:
102                 # print "Taking %s from %s" % (package_name,repo["Base"])
103
104                 output_packages[package_name] = ( repo, package,)
105
106
107     print "Writing Sources file"
108     package_file = "dists/%s/%s/sources/Sources" % ( config["Dist"], config["Section"])
109     package_out_file = "%s/%s" % ( config["Output"], package_file)
110
111     if not os.path.isdir(os.path.dirname(package_out_file)):
112         os.makedirs(os.path.dirname(package_out_file))
113     package_out = file(package_out_file,'w')
114     for repo, package in output_packages.values():
115         package_out.write(str(package))
116         package_out.write("\n")
117         for pkgfile in package["Files"]:
118             package_name = package["Package"]
119             if package_name[:3] == "lib":
120                 c = package_name[:4]
121             else:
122                 c = package_name[0]
123             pkgpath = "pool/%s/%s/%s/%s" % (
124                 repo["Section"], c, package_name, pkgfile["name"] )
125             add_rewrite(pkgpath, repo["Base"])
126     package_out.close()     
127     gzip.GzipFile(package_out_file + ".gz", 'w').write(
128             file(package_out_file,'r').read()
129         )
130     release_files.append(package_file)
131     release_files.append(package_file + ".gz")
132
133 print "Writing README.txt"
134 readme = file("%s/README.txt" % config["Output"],'w')
135 readme.write("This is a virtual apt repository, created by merging\n")
136 readme.write("several proper repositories. This should only be used\n")
137 readme.write("if the programs in question, such as debootstrap, do not\n")
138 readme.write("support more than one repository. Otherwise, please use\n")
139 readme.write("the repositories directly.\n\n")
140 readme.write("This was created by merging these repositories, prefering\n")
141 readme.write("The first ones:\n")
142 for repo in config["Repos"]:
143     readme.write(" %s %s %s\n" % (repo["Base"], repo["Dist"], repo["Section"]) )
144
145 print "Writing Release file"
146 release_file = file("%s/dists/%s/Release" % (config["Output"], config["Dist"]),'w')
147 release = deb822.Release()
148 release["Origin"] = "Debian"
149 release["Label"] = "Debian"
150 release["Suite"] = config["Dist"]
151 release["Codename"] = config["Dist"]
152 release["Architectures"] = config["Arches"]
153 release["Components"] = config["Section"]
154 release["Description"] = "Merged apt repository"
155 release["MD5Sum"] = []
156 for file_name in release_files:
157     output_file_name = "%s/%s" % (config["Output"], file_name)
158     f = file(output_file_name, 'r')
159     release["MD5Sum"].append({
160             'name'  : file_name.split('/',2)[2],
161             'size'  : os.path.getsize(output_file_name),
162             'md5sum': md5.new(f.read()).hexdigest()
163         })
164 release_file.write(str(release))
165
166
167 print "Writing redirects"
168 for dir, rewrites in rewrite_rules_by_dir.items():
169     outdir = "%s/%s" % (config["Output"], dir)
170     if not os.path.isdir(outdir):
171         os.makedirs(outdir)
172     htaccess = file("%s/.htaccess" % outdir,'w')
173     htaccess.write("RewriteEngine On\n")
174     readme = file("%s/README.txt" % outdir,'w')
175     readme.write("This is a directory with the following redirects set up:\n\n")
176     for filename, base in rewrites:
177         filename_escape = re.sub('\+','\\+',filename)
178         htaccess.write("RewriteRule ^(%s)$ %s/%s/$1 [R]\n" %
179                 (filename_escape, base, dir) )
180         readme.write("%s -> %s/%s/%s\n" %
181                 (filename, base, dir, filename) )