Initial commit
authorJoachim Breitner <mail@joachim-breitner.de>
Sun, 10 Aug 2008 18:17:12 +0000 (18:17 +0000)
committerJoachim Breitner <mail@joachim-breitner.de>
Sun, 10 Aug 2008 18:17:12 +0000 (18:17 +0000)
apt-repository-merge [new file with mode: 0644]
test.conf [new file with mode: 0644]

diff --git a/apt-repository-merge b/apt-repository-merge
new file mode 100644 (file)
index 0000000..df258a7
--- /dev/null
@@ -0,0 +1,143 @@
+#!/usr/bin/python
+
+from optparse import OptionParser
+
+import gzip
+from debian_bundle import deb822
+import urllib2
+import os
+import os.path
+
+class Config(deb822._multivalued):
+    _multivalued_fields = {
+           "Repos": [ "Base", "Dist", "Section" ]
+           }
+
+parser = OptionParser()
+parser.add_option("-c", "--config", dest="config",
+                  help="Configuration to use")
+(options, args) = parser.parse_args()
+
+config = Config(file(options.config))
+
+rewrite_rules_by_dir = {}
+
+def add_rewrite(path, base):
+    dir = os.path.dirname(path)
+    file = os.path.basename(path)
+    rewrite_rules_by_dir.setdefault(dir,[])
+    rewrite_rules_by_dir[dir].append(( file, base ))
+
+for arch in config["arches"].split():
+    print "Merging architecture %s" % arch
+
+    output_packages = {}
+
+    for repo in config["Repos"]:
+       url = "%s/dists/%s/%s/binary-%s/Packages.gz" % (
+               repo["Base"], repo["Dist"], repo["Section"], arch
+               )
+       safe_base = filter (lambda c: c.isalnum(), repo["base"])
+       cache = "%s/%s_%s_%s_%s_Packages.gz" % (
+               config["Output"], safe_base, repo["Dist"], repo["Section"], arch
+               )
+
+       print "Getting %s" % url
+       file(cache,'w').write( urllib2.urlopen(url).read())
+
+       packages = deb822.Packages.iter_paragraphs(gzip.GzipFile(cache))
+
+       for package in packages:
+           package_name = package["Package"]
+           if package_name not in output_packages:
+               # print "Taking %s from %s" % (package_name,repo["Base"])
+
+               output_packages[package_name] = ( repo, package,)
+
+
+    package_out_file = "%s/dists/%s/%s/binary-%s/Packages" % (
+       config["Output"], config["Dist"], config["Section"], arch
+       )
+    if not os.path.isdir(os.path.dirname(package_out_file)):
+       os.makedirs(os.path.dirname(package_out_file))
+    package_out = file(package_out_file,'w')
+    for repo, package in output_packages.values():
+       package_out.write(str(package))
+       package_out.write("\n")
+       add_rewrite(package["Filename"], repo["Base"])
+
+if True:    # to match indentation with above
+    print "Merging sources"
+
+    output_packages = {}
+
+    for repo in config["Repos"]:
+       url = "%s/dists/%s/%s/source/Sources.gz" % (
+               repo["Base"], repo["Dist"], repo["Section"]
+               )
+       safe_base = filter (lambda c: c.isalnum(), repo["Base"])
+       cache = "%s/%s_%s_%s_Sources.gz" % (
+               config["Output"],safe_base, repo["Dist"], repo["Section"]
+               )
+
+       print "Getting %s" % url
+       file(cache,'w').write( urllib2.urlopen(url).read())
+       print "done."
+
+       packages = deb822.Sources.iter_paragraphs(gzip.GzipFile(cache))
+
+       for package in packages:
+           package_name = package["Package"]
+           if package_name not in output_packages:
+               # print "Taking %s from %s" % (package_name,repo["Base"])
+
+               output_packages[package_name] = ( repo, package,)
+
+
+    package_out_file = "%s/dists/%s/%s/sources/Sources" % (
+       config["Output"], config["Dist"], config["Section"]
+       )
+    if not os.path.isdir(os.path.dirname(package_out_file)):
+       os.makedirs(os.path.dirname(package_out_file))
+    package_out = file(package_out_file,'w')
+    for repo, package in output_packages.values():
+       package_out.write(str(package))
+       package_out.write("\n")
+       for pkgfile in package["Files"]:
+           package_name = package["Package"]
+           if package_name[:3] == "lib":
+               c = package_name[:4]
+           else:
+               c = package_name[0]
+           pkgpath = "pool/%s/%s/%s/%s" % (
+               repo["Section"], c, package_name, pkgfile["name"] )
+           add_rewrite(pkgpath, repo["Base"])
+
+print "Writing README.txt"
+readme = file("%s/README.txt" % config["Output"],'w')
+readme.write("This is a virtual apt repository, created by merging\n")
+readme.write("several proper repositories. This should only be used\n")
+readme.write("if the programs in question, such as debootstrap, do not\n")
+readme.write("support more than one repository. Otherwise, please use\n")
+readme.write("the repositories directly.\n\n")
+readme.write("This was created by merging these repositories, prefering\n")
+readme.write("The first ones:\n")
+for repo in config["Repos"]:
+    readme.write(" %s %s %s\n" % (repo["Base"], repo["Dist"], repo["Section"]) )
+
+
+print "Writing redirects"
+for dir, rewrites in rewrite_rules_by_dir.items():
+    outdir = "%s/%s" % (config["Output"], dir)
+    if not os.path.isdir(outdir):
+       os.makedirs(outdir)
+    htaccess = file("%s/.htaccess" % outdir,'w')
+    htaccess.write("RewriteEngine On\n")
+    readme = file("%s/README.txt" % outdir,'w')
+    readme.write("This is a directory with the following redirects set up:\n\n")
+    for filename, base in rewrites:
+       htaccess.write("RewriteRule (%s) %s/%s/$1 [R]\n" %
+               (filename, base, dir) )
+       readme.write("%s -> %s/%s/%s\n" %
+               (filename, base, dir, filename) )
+
diff --git a/test.conf b/test.conf
new file mode 100644 (file)
index 0000000..42ad191
--- /dev/null
+++ b/test.conf
@@ -0,0 +1,8 @@
+Repos:
+ http://pkg-fso.alioth.debian.org/debian unstable main
+ http://mirror/debian experimental main
+# http://mirror/debian/ sid main
+Arches: armel i386
+Dist: unstable+experimental+pkg-fso
+Section: main
+Output: output