diff --git a/README.rst b/README.rst index ec9da98..7736a51 100644 --- a/README.rst +++ b/README.rst @@ -16,7 +16,20 @@ as: :: - termux-apt-repo + termux-apt-repo [-h] [--use-hard-links] input output [dist] [comp] + + positional arguments: + input folder where .deb files are located + output folder with repository tree + dist name of distribution folder. deb files are put into + output/dists/distribution/component/binary-$ARCH/ + comp name of component folder. deb files are put into + output/dists/distribution/component/binary-$ARCH/ + + optional arguments: + -h, --help show this help message and exit + --use-hard-links use hard links instead of copying deb files. Will not work + on an android device When using outside Termux (the script should work on most Linux distributions), install with ``pip3 install termux-apt-repo``. @@ -54,8 +67,9 @@ containing the single line: :: - deb [trusted=yes] $REPO_URL termux extras + deb [trusted=yes] $REPO_URL $dist $comp -If the published ``$REPO_URL`` is https, users must first install the -``apt-transport-https`` package which is not preinstalled (likely to -come preinstalled in the future). +``[trusted=yes]`` is needed if the repo has not been signed with a gpg key. +To sign it, edit ``termux-apt-repo`` and change ``if False:`` to ``if True:`` near +end of script. The signing key then has to be imported by the user to make apt +trust it. diff --git a/termux-apt-repo b/termux-apt-repo index 5a8ca3c..8fe46b3 100755 --- a/termux-apt-repo +++ b/termux-apt-repo @@ -1,11 +1,11 @@ #!/usr/bin/env python3 -import datetime, hashlib, os, re, shutil, subprocess, sys +import datetime, hashlib, os, re, shutil, subprocess, sys, glob, argparse -DISTRIBUTION = 'termux' -COMPONENT = 'extras' +COMPONENTS = [] supported_arches = ['all', 'arm', 'i686', 'aarch64', 'x86_64'] encountered_arches = set() +hashes = ['md5', 'sha1', 'sha256', 'sha512'] def get_package_name(filename): # Expects the 'name_version_arch.deb' naming scheme. @@ -19,7 +19,7 @@ def control_file_contents(debfile): stderr=subprocess.DEVNULL ).strip() -def add_deb(deb_to_add_path): +def add_deb(deb_to_add_path, component, use_hard_links): deb_to_add_control_file = control_file_contents(deb_to_add_path) deb_to_add_pkg_name = re.search('Package: (.*)', deb_to_add_control_file).group(1) deb_to_add_pkg_version = re.search('Version: (.*)', deb_to_add_control_file).group(1) @@ -30,101 +30,127 @@ def add_deb(deb_to_add_path): encountered_arches.add(deb_arch) package_name = get_package_name(os.path.basename(deb_to_add_path)) - arch_dir_path = component_path + '/binary-' + deb_arch + arch_dir_path = os.path.join(distribution_path, component, 'binary-' + deb_arch) if not os.path.isdir(arch_dir_path): os.makedirs(arch_dir_path) # Add .deb file: print('Adding deb file: ' + os.path.basename(deb_to_add_path) + '...') - dest_deb_dir_path = component_path + '/binary-' + deb_arch + dest_deb_dir_path = os.path.join(distribution_path, component, 'binary-' + deb_arch) if not os.path.isdir(dest_deb_dir_path): os.makedirs(dest_deb_dir_path) - destination_deb_file = dest_deb_dir_path + '/' + os.path.basename(deb_to_add_path) - shutil.copy2(deb_to_add_path, destination_deb_file) - -if len(sys.argv) == 3: - input_path = sys.argv[1] - output_path = sys.argv[2] - distribution_path = output_path + '/dists/' + DISTRIBUTION - component_path = distribution_path + '/' + COMPONENT -else: - sys.exit('Usage: termux-apt-repo ') + destination_deb_file = os.path.join(dest_deb_dir_path, os.path.basename(deb_to_add_path)) + if not use_hard_links: + shutil.copy2(deb_to_add_path, destination_deb_file) + else: + os.link(deb_to_add_path, destination_deb_file) + +parser = argparse.ArgumentParser(description='Create a repository with deb files') +parser.add_argument('input', metavar='input', type=str, + help='folder where .deb files are located') +parser.add_argument('output', metavar='output', type=str, + help='folder with repository tree') +parser.add_argument('distribution', metavar='dist', type=str, nargs='?', default='termux', + help='name of distribution folder. deb files are put into output/dists/distribution/component/binary-$ARCH/') +parser.add_argument('component', metavar='comp', type=str, nargs='?', default='extras', + help='name of component folder. deb files are put into output/dists/distribution/component/binary-$ARCH/') +parser.add_argument('--use-hard-links', default=False, action='store_true', + help='use hard links instead of copying deb files. Will not work on an android device') + +args = parser.parse_args() +input_path = args.input +output_path = args.output +DISTRIBUTION = args.distribution +default_component = args.component +use_hard_links = args.use_hard_links +distribution_path = os.path.join(output_path, 'dists', DISTRIBUTION) if not os.path.isdir(input_path): sys.exit("'" + input_path + '" does not exist') -if os.path.exists(output_path): - shutil.rmtree(output_path) -os.makedirs(component_path) - -added_debs_count = 0 -for f in sorted(os.listdir(input_path)): - if not f.endswith('.deb'): continue - deb_path = os.path.join(input_path, f) - added_debs_count += 1 - add_deb(deb_path) - -if added_debs_count == 0: - sys.exit('Not .deb file found in ' + input_path) +debs_in_path = glob.glob(os.path.join(input_path, "*.deb")) +debs_in_path += glob.glob(os.path.join(input_path, "*/*.deb")) +if not debs_in_path: + sys.exit('No .deb file found in ' + input_path) +else: + for deb_path in sorted(debs_in_path): + component = os.path.dirname(os.path.relpath(deb_path, input_path)) + if not component: + component = default_component + if component not in COMPONENTS: + COMPONENTS.append(component) + if os.path.isdir(os.path.join(distribution_path, component)): + shutil.rmtree(os.path.join(distribution_path, component)) + os.makedirs(os.path.join(distribution_path, component)) + + add_deb(deb_path, component, use_hard_links) # See https://wiki.debian.org/RepositoryFormat#A.22Release.22_files for format: release_file_path = distribution_path + '/Release' -release_file = open(release_file_path, 'w') +release_file = open(release_file_path, 'w') print("Codename: termux", file=release_file) print("Version: 1", file=release_file) print("Architectures: " + ' '.join(encountered_arches), file=release_file) -print("Description: Extras repository", file=release_file) -print("Suite: termux", file=release_file) +print("Description: " + DISTRIBUTION + " repository", file=release_file) +print("Suite: " + DISTRIBUTION, file=release_file) print("Date: " + datetime.datetime.utcnow().strftime('%a, %d %b %Y %H:%M:%S UTC'), file=release_file) -print("SHA256:", file=release_file) - -# Create Packages files and add their info to Release file: -for arch in encountered_arches: - print('Creating package file for ' + arch + '...') - arch_dir_path = component_path + '/binary-' + arch - if not os.path.isdir(arch_dir_path): continue - packages_file_path = arch_dir_path + '/Packages' - packagesxz_file_path = packages_file_path + '.xz' - binary_path = 'binary-' + arch - with open(packages_file_path, 'w') as packages_file: - first = True - for f in sorted(os.listdir(arch_dir_path)): - if not f.endswith('.deb'): - continue - if first: - first = False - else: - print('', file=packages_file) - deb_to_read_path = os.path.join(arch_dir_path, f) - # Extract the control file from the .deb: - scanpackages_output = control_file_contents(deb_to_read_path) - package_name = re.search('Package: (.*)', scanpackages_output).group(1) - package_arch = re.search('Architecture: (.*)', scanpackages_output).group(1) - # Add these fields which dpkg-scanpackages would have done: - scanpackages_output += '\nFilename: dists/' + DISTRIBUTION + '/' + COMPONENT + '/' + binary_path + '/' + os.path.basename(deb_to_read_path) - scanpackages_output += '\nSize: ' + str(os.stat(deb_to_read_path).st_size) - scanpackages_output += '\nSHA256: ' + hashlib.sha256(open(deb_to_read_path, 'rb').read()).hexdigest() - print(scanpackages_output, file=packages_file) - - # Create Packages.xz - subprocess.check_call(['rm -f ' + packagesxz_file_path + '; xz -9 --keep ' + packages_file_path], shell=True, universal_newlines=True) - # Write info about Packages and Packages.xz to Release file: - print(' ' + hashlib.sha256(open(packages_file_path, 'rb').read()).hexdigest() - + ' ' - + str(os.stat(packages_file_path).st_size) - + ' ' + COMPONENT + '/' + binary_path + '/Packages' - , file=release_file) - print(' ' + hashlib.sha256(open(packagesxz_file_path, 'rb').read()).hexdigest() - + ' ' - + str(os.stat(packagesxz_file_path).st_size) - + ' ' + COMPONENT + '/' + binary_path + '/Packages.xz' - , file=release_file) +# Create Packages files: +for component in COMPONENTS: + for arch_dir_path in glob.glob(os.path.join(distribution_path, component, 'binary-*')): + arch = os.path.basename(arch_dir_path).split('-')[1] + print('Creating package file for ' + component + " and " + arch + '...') + packages_file_path = arch_dir_path + '/Packages' + packagesxz_file_path = packages_file_path + '.xz' + binary_path = 'binary-' + arch + with open(packages_file_path, 'w') as packages_file: + for deb_to_read_path in sorted(glob.glob(os.path.join(arch_dir_path, "*.deb"))): + # Extract the control file from the .deb: + scanpackages_output = control_file_contents(deb_to_read_path) + package_name = re.search('Package: (.*)', scanpackages_output).group(1) + package_arch = re.search('Architecture: (.*)', scanpackages_output).group(1) + # Add these fields which dpkg-scanpackages would have done: + scanpackages_output += '\nFilename: ' + os.path.join('dists', DISTRIBUTION, component, binary_path, os.path.basename(deb_to_read_path)) + scanpackages_output += '\nSize: ' + str(os.stat(deb_to_read_path).st_size) + for hash in hashes: + if hash == "md5": + hash_string = hash.upper()+'Sum' + else: + hash_string = hash.upper() + scanpackages_output += '\n'+hash_string + ': ' + getattr(hashlib, hash)(open(deb_to_read_path, 'rb').read()).hexdigest() + print(scanpackages_output, file=packages_file) + print('', file=packages_file) + # Create Packages.xz + subprocess.check_call(['rm -f ' + packagesxz_file_path + '; xz -9 --keep ' + packages_file_path], shell=True, universal_newlines=True) + +# Get components in output folder, we might have more folders than we are adding now +COMPONENTS = [d for d in os.listdir(distribution_path) if os.path.isdir(os.path.join(distribution_path, d))] +for hash in hashes: + if hash == 'md5': + hash_string = hash.upper()+'Sum' + else: + hash_string = hash.upper() + print(hash_string + ': ', file=release_file) + + for component in COMPONENTS: + for arch_dir_path in glob.glob(os.path.join(distribution_path, component, 'binary-*')): + packages_file_path = arch_dir_path + '/Packages' + packagesxz_file_path = packages_file_path + '.xz' + binary_path = os.path.basename(arch_dir_path) + # Write info about Packages and Packages.xz to Release file: + print(' '+' '.join([getattr(hashlib, hash)(open(packages_file_path, 'rb').read()).hexdigest(), + str(os.stat(packages_file_path).st_size), + os.path.join(component, binary_path, 'Packages')]) + , file=release_file) + print(' '+' '.join([getattr(hashlib, hash)(open(packagesxz_file_path, 'rb').read()).hexdigest(), + str(os.stat(packagesxz_file_path).st_size), + os.path.join(component, binary_path, 'Packages.xz')]) + , file=release_file) release_file.close() if False: print('Signing with gpg...') - subprocess.check_call(['gpg --pinentry-mode loopback --digest-algo SHA256 --clearsign -o ' + subprocess.check_call(['gpg --yes --pinentry-mode loopback --digest-algo SHA256 --clearsign -o ' + distribution_path + '/InRelease ' + distribution_path + '/Release'], shell=True, universal_newlines=True) @@ -135,6 +161,8 @@ print('Make the ' + output_path + ' directory accessible at $REPO_URL') print('') print('Users can then access the repo by adding a file at') print(' $PREFIX/etc/apt/sources.list.d') -print('containing the single line:') -print(' deb [trusted=yes] $REPO_URL termux extras') -print('If the $REPO_URL is https, users must first install the apt-transport-https package') +print('containing:') +for component in COMPONENTS: + print(' deb [trusted=yes] $REPO_URL '+DISTRIBUTION+' '+component) +print('') +print('[trusted=yes] is not needed if the repo has been signed with a gpg key')