From 695d165d6d595a1780b5c52ff6dd0a38b24f5005 Mon Sep 17 00:00:00 2001 From: Henrik Grimler Date: Fri, 31 Aug 2018 23:16:52 +0200 Subject: [PATCH 01/15] termux-apt-repo: use glob to shorten code a bit --- termux-apt-repo | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/termux-apt-repo b/termux-apt-repo index 5a8ca3c..8c2b181 100755 --- a/termux-apt-repo +++ b/termux-apt-repo @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -import datetime, hashlib, os, re, shutil, subprocess, sys +import datetime, hashlib, os, re, shutil, subprocess, sys, glob DISTRIBUTION = 'termux' COMPONENT = 'extras' @@ -57,15 +57,13 @@ if os.path.exists(output_path): shutil.rmtree(output_path) os.makedirs(component_path) -added_debs_count = 0 -for f in sorted(os.listdir(input_path)): - if not f.endswith('.deb'): continue - deb_path = os.path.join(input_path, f) - added_debs_count += 1 - add_deb(deb_path) - -if added_debs_count == 0: +debs_in_path = glob.glob(os.path.join(input_path, "*.deb")) +if not debs_in_path: sys.exit('Not .deb file found in ' + input_path) +else: + for f in sorted(debs_in_path): + deb_path = f + add_deb(deb_path) # See https://wiki.debian.org/RepositoryFormat#A.22Release.22_files for format: release_file_path = distribution_path + '/Release' @@ -88,14 +86,11 @@ for arch in encountered_arches: binary_path = 'binary-' + arch with open(packages_file_path, 'w') as packages_file: first = True - for f in sorted(os.listdir(arch_dir_path)): - if not f.endswith('.deb'): - continue + for deb_to_read_path in sorted(glob.glob(os.path.join(arch_dir_path, "*.deb"))): if first: first = False else: print('', file=packages_file) - deb_to_read_path = os.path.join(arch_dir_path, f) # Extract the control file from the .deb: scanpackages_output = control_file_contents(deb_to_read_path) package_name = re.search('Package: (.*)', scanpackages_output).group(1) From c3f308e31a5153e659789ac15b28a6c03a24590a Mon Sep 17 00:00:00 2001 From: Henrik Grimler Date: Fri, 31 Aug 2018 23:17:53 +0200 Subject: [PATCH 02/15] termux-apt-repo: refactor and use several hashes Use md5, sha1, sha256 and sha512 instead of just sha256 --- termux-apt-repo | 42 +++++++++++++++++++++++++++++------------- 1 file changed, 29 insertions(+), 13 deletions(-) diff --git a/termux-apt-repo b/termux-apt-repo index 8c2b181..f9f6eac 100755 --- a/termux-apt-repo +++ b/termux-apt-repo @@ -6,6 +6,7 @@ DISTRIBUTION = 'termux' COMPONENT = 'extras' supported_arches = ['all', 'arm', 'i686', 'aarch64', 'x86_64'] encountered_arches = set() +hashes = ['md5', 'sha1', 'sha256', 'sha512'] def get_package_name(filename): # Expects the 'name_version_arch.deb' naming scheme. @@ -74,7 +75,6 @@ print("Architectures: " + ' '.join(encountered_arches), file=release_file) print("Description: Extras repository", file=release_file) print("Suite: termux", file=release_file) print("Date: " + datetime.datetime.utcnow().strftime('%a, %d %b %Y %H:%M:%S UTC'), file=release_file) -print("SHA256:", file=release_file) # Create Packages files and add their info to Release file: for arch in encountered_arches: @@ -98,23 +98,39 @@ for arch in encountered_arches: # Add these fields which dpkg-scanpackages would have done: scanpackages_output += '\nFilename: dists/' + DISTRIBUTION + '/' + COMPONENT + '/' + binary_path + '/' + os.path.basename(deb_to_read_path) scanpackages_output += '\nSize: ' + str(os.stat(deb_to_read_path).st_size) - scanpackages_output += '\nSHA256: ' + hashlib.sha256(open(deb_to_read_path, 'rb').read()).hexdigest() + for hash in hashes: + if hash == "md5": + hash_string = hash.upper()+'Sum' + else: + hash_string = hash.upper() + scanpackages_output += '\n'+hash_string + ': ' + getattr(hashlib, hash)(open(deb_to_read_path, 'rb').read()).hexdigest() print(scanpackages_output, file=packages_file) # Create Packages.xz subprocess.check_call(['rm -f ' + packagesxz_file_path + '; xz -9 --keep ' + packages_file_path], shell=True, universal_newlines=True) - # Write info about Packages and Packages.xz to Release file: - print(' ' + hashlib.sha256(open(packages_file_path, 'rb').read()).hexdigest() - + ' ' - + str(os.stat(packages_file_path).st_size) - + ' ' + COMPONENT + '/' + binary_path + '/Packages' - , file=release_file) - print(' ' + hashlib.sha256(open(packagesxz_file_path, 'rb').read()).hexdigest() - + ' ' - + str(os.stat(packagesxz_file_path).st_size) - + ' ' + COMPONENT + '/' + binary_path + '/Packages.xz' - , file=release_file) +for hash in hashes: + if hash == 'md5': + hash_string = hash.upper()+'Sum' + else: + hash_string = hash.upper() + print(hash_string + ': ', file=release_file) + + for arch in encountered_arches: + arch_dir_path = component_path + '/binary-' + arch + if not os.path.isdir(arch_dir_path): continue + packages_file_path = arch_dir_path + '/Packages' + packagesxz_file_path = packages_file_path + '.xz' + binary_path = 'binary-' + arch + # Write info about Packages and Packages.xz to Release file: + print(' '+' '.join([getattr(hashlib, hash)(open(packages_file_path, 'rb').read()).hexdigest(), + str(os.stat(packages_file_path).st_size), + os.path.join(COMPONENT, binary_path, 'Packages')]) + , file=release_file) + print(' '+' '.join([getattr(hashlib, hash)(open(packagesxz_file_path, 'rb').read()).hexdigest(), + str(os.stat(packagesxz_file_path).st_size), + os.path.join(COMPONENT, binary_path, 'Packages.xz')]) + , file=release_file) release_file.close() if False: From 74c5fdb4de5d88a657ead39d58c88db479dccf68 Mon Sep 17 00:00:00 2001 From: Henrik Grimler Date: Sat, 13 Oct 2018 08:19:48 +0200 Subject: [PATCH 03/15] Add optional arguments DISTRIBUTION and COMPONENT Use os.path.join instead of path1+"/"+path2. rm output_path/dists/DISTRIBUTION/COMPONENT instead of output_path/*. --- termux-apt-repo | 38 ++++++++++++++++++++------------------ 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/termux-apt-repo b/termux-apt-repo index f9f6eac..796bdc5 100755 --- a/termux-apt-repo +++ b/termux-apt-repo @@ -31,55 +31,57 @@ def add_deb(deb_to_add_path): encountered_arches.add(deb_arch) package_name = get_package_name(os.path.basename(deb_to_add_path)) - arch_dir_path = component_path + '/binary-' + deb_arch + arch_dir_path = os.path.join(distribution_path, COMPONENT, 'binary-' + deb_arch) if not os.path.isdir(arch_dir_path): os.makedirs(arch_dir_path) # Add .deb file: print('Adding deb file: ' + os.path.basename(deb_to_add_path) + '...') - dest_deb_dir_path = component_path + '/binary-' + deb_arch + dest_deb_dir_path = os.path.join(distribution_path, COMPONENT, 'binary-' + deb_arch) if not os.path.isdir(dest_deb_dir_path): os.makedirs(dest_deb_dir_path) - destination_deb_file = dest_deb_dir_path + '/' + os.path.basename(deb_to_add_path) + destination_deb_file = os.path.join(dest_deb_dir_path, os.path.basename(deb_to_add_path)) shutil.copy2(deb_to_add_path, destination_deb_file) -if len(sys.argv) == 3: +if len(sys.argv) > 2: input_path = sys.argv[1] output_path = sys.argv[2] - distribution_path = output_path + '/dists/' + DISTRIBUTION - component_path = distribution_path + '/' + COMPONENT + if len(sys.argv) > 3: + DISTRIBUTION = sys.argv[3] + if len(sys.argv) > 4: + COMPONENT = sys.argv[4] + distribution_path = os.path.join(output_path, 'dists', DISTRIBUTION) else: - sys.exit('Usage: termux-apt-repo ') + sys.exit('Usage: termux-apt-repo [ ]\n and defaults to "termux" and "extras" if arguments are omitted') if not os.path.isdir(input_path): sys.exit("'" + input_path + '" does not exist') -if os.path.exists(output_path): - shutil.rmtree(output_path) -os.makedirs(component_path) +if os.path.exists(os.path.join(distribution_path, COMPONENT)): + shutil.rmtree(os.path.join(distribution_path, COMPONENT)) +os.makedirs(os.path.join(distribution_path, COMPONENT)) debs_in_path = glob.glob(os.path.join(input_path, "*.deb")) if not debs_in_path: sys.exit('Not .deb file found in ' + input_path) else: - for f in sorted(debs_in_path): - deb_path = f + for deb_path in sorted(debs_in_path): add_deb(deb_path) # See https://wiki.debian.org/RepositoryFormat#A.22Release.22_files for format: release_file_path = distribution_path + '/Release' -release_file = open(release_file_path, 'w') +release_file = open(release_file_path, 'w') print("Codename: termux", file=release_file) print("Version: 1", file=release_file) print("Architectures: " + ' '.join(encountered_arches), file=release_file) -print("Description: Extras repository", file=release_file) -print("Suite: termux", file=release_file) +print("Description: " + DISTRIBUTION + " repository", file=release_file) +print("Suite: " + DISTRIBUTION, file=release_file) print("Date: " + datetime.datetime.utcnow().strftime('%a, %d %b %Y %H:%M:%S UTC'), file=release_file) # Create Packages files and add their info to Release file: for arch in encountered_arches: print('Creating package file for ' + arch + '...') - arch_dir_path = component_path + '/binary-' + arch + arch_dir_path = os.path.join(distribution_path, COMPONENT, 'binary-' + arch) if not os.path.isdir(arch_dir_path): continue packages_file_path = arch_dir_path + '/Packages' packagesxz_file_path = packages_file_path + '.xz' @@ -96,7 +98,7 @@ for arch in encountered_arches: package_name = re.search('Package: (.*)', scanpackages_output).group(1) package_arch = re.search('Architecture: (.*)', scanpackages_output).group(1) # Add these fields which dpkg-scanpackages would have done: - scanpackages_output += '\nFilename: dists/' + DISTRIBUTION + '/' + COMPONENT + '/' + binary_path + '/' + os.path.basename(deb_to_read_path) + scanpackages_output += '\nFilename: ' + os.path.join('dists', DISTRIBUTION, COMPONENT, binary_path, os.path.basename(deb_to_read_path)) scanpackages_output += '\nSize: ' + str(os.stat(deb_to_read_path).st_size) for hash in hashes: if hash == "md5": @@ -117,7 +119,7 @@ for hash in hashes: print(hash_string + ': ', file=release_file) for arch in encountered_arches: - arch_dir_path = component_path + '/binary-' + arch + arch_dir_path = os.path.join(distribution_path, COMPONENT, 'binary-' + arch) if not os.path.isdir(arch_dir_path): continue packages_file_path = arch_dir_path + '/Packages' packagesxz_file_path = packages_file_path + '.xz' From 1fe634f7f41bf67f7873ec619b6b8c98657d44b3 Mon Sep 17 00:00:00 2001 From: Henrik Grimler Date: Sat, 13 Oct 2018 10:07:26 +0200 Subject: [PATCH 04/15] Fix typo --- termux-apt-repo | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/termux-apt-repo b/termux-apt-repo index 796bdc5..5eb5dad 100755 --- a/termux-apt-repo +++ b/termux-apt-repo @@ -63,7 +63,7 @@ os.makedirs(os.path.join(distribution_path, COMPONENT)) debs_in_path = glob.glob(os.path.join(input_path, "*.deb")) if not debs_in_path: - sys.exit('Not .deb file found in ' + input_path) + sys.exit('No .deb file found in ' + input_path) else: for deb_path in sorted(debs_in_path): add_deb(deb_path) From 705ea5c39fe32a9bf9fbcdd8f11ed98000abe093 Mon Sep 17 00:00:00 2001 From: Henrik Grimler Date: Sat, 13 Oct 2018 15:57:45 +0200 Subject: [PATCH 05/15] Allow for termux/stable & termux/debug & .. The script looks for .deb files in subfolders and takes the subfolder name as COMPONENT. .deb files located in the input_path directory (and not in a subfolder) are still put into the "extras" component folder. When updating the Release file, all component directories are taken into account, so it's possible to only update for example termux/debug and keeping termux/stable as is. --- termux-apt-repo | 114 +++++++++++++++++++++++++----------------------- 1 file changed, 59 insertions(+), 55 deletions(-) diff --git a/termux-apt-repo b/termux-apt-repo index 5eb5dad..47bdf16 100755 --- a/termux-apt-repo +++ b/termux-apt-repo @@ -3,7 +3,7 @@ import datetime, hashlib, os, re, shutil, subprocess, sys, glob DISTRIBUTION = 'termux' -COMPONENT = 'extras' +COMPONENTS = ['extras'] supported_arches = ['all', 'arm', 'i686', 'aarch64', 'x86_64'] encountered_arches = set() hashes = ['md5', 'sha1', 'sha256', 'sha512'] @@ -20,7 +20,7 @@ def control_file_contents(debfile): stderr=subprocess.DEVNULL ).strip() -def add_deb(deb_to_add_path): +def add_deb(deb_to_add_path, component): deb_to_add_control_file = control_file_contents(deb_to_add_path) deb_to_add_pkg_name = re.search('Package: (.*)', deb_to_add_control_file).group(1) deb_to_add_pkg_version = re.search('Version: (.*)', deb_to_add_control_file).group(1) @@ -31,14 +31,14 @@ def add_deb(deb_to_add_path): encountered_arches.add(deb_arch) package_name = get_package_name(os.path.basename(deb_to_add_path)) - arch_dir_path = os.path.join(distribution_path, COMPONENT, 'binary-' + deb_arch) + arch_dir_path = os.path.join(distribution_path, component, 'binary-' + deb_arch) if not os.path.isdir(arch_dir_path): os.makedirs(arch_dir_path) # Add .deb file: print('Adding deb file: ' + os.path.basename(deb_to_add_path) + '...') - dest_deb_dir_path = os.path.join(distribution_path, COMPONENT, 'binary-' + deb_arch) + dest_deb_dir_path = os.path.join(distribution_path, component, 'binary-' + deb_arch) if not os.path.isdir(dest_deb_dir_path): os.makedirs(dest_deb_dir_path) destination_deb_file = os.path.join(dest_deb_dir_path, os.path.basename(deb_to_add_path)) shutil.copy2(deb_to_add_path, destination_deb_file) @@ -49,7 +49,7 @@ if len(sys.argv) > 2: if len(sys.argv) > 3: DISTRIBUTION = sys.argv[3] if len(sys.argv) > 4: - COMPONENT = sys.argv[4] + COMPONENTS = [sys.argv[4]] distribution_path = os.path.join(output_path, 'dists', DISTRIBUTION) else: sys.exit('Usage: termux-apt-repo [ ]\n and defaults to "termux" and "extras" if arguments are omitted') @@ -57,16 +57,24 @@ else: if not os.path.isdir(input_path): sys.exit("'" + input_path + '" does not exist') -if os.path.exists(os.path.join(distribution_path, COMPONENT)): - shutil.rmtree(os.path.join(distribution_path, COMPONENT)) -os.makedirs(os.path.join(distribution_path, COMPONENT)) +for component in COMPONENTS: + if os.path.exists(os.path.join(distribution_path, component)): + shutil.rmtree(os.path.join(distribution_path, component)) + os.makedirs(os.path.join(distribution_path, component)) debs_in_path = glob.glob(os.path.join(input_path, "*.deb")) +debs_in_path += glob.glob(os.path.join(input_path, "*/*.deb")) if not debs_in_path: sys.exit('No .deb file found in ' + input_path) else: for deb_path in sorted(debs_in_path): - add_deb(deb_path) + component = os.path.dirname(os.path.relpath(deb_path, input_path)) + if not component: + component = COMPONENTS[0] + else: + COMPONENTS.append(component) + add_deb(deb_path, component) +COMPONENTS = list(set(COMPONENTS)) # See https://wiki.debian.org/RepositoryFormat#A.22Release.22_files for format: release_file_path = distribution_path + '/Release' @@ -78,39 +86,36 @@ print("Description: " + DISTRIBUTION + " repository", file=release_file) print("Suite: " + DISTRIBUTION, file=release_file) print("Date: " + datetime.datetime.utcnow().strftime('%a, %d %b %Y %H:%M:%S UTC'), file=release_file) -# Create Packages files and add their info to Release file: -for arch in encountered_arches: - print('Creating package file for ' + arch + '...') - arch_dir_path = os.path.join(distribution_path, COMPONENT, 'binary-' + arch) - if not os.path.isdir(arch_dir_path): continue - packages_file_path = arch_dir_path + '/Packages' - packagesxz_file_path = packages_file_path + '.xz' - binary_path = 'binary-' + arch - with open(packages_file_path, 'w') as packages_file: - first = True - for deb_to_read_path in sorted(glob.glob(os.path.join(arch_dir_path, "*.deb"))): - if first: - first = False - else: +# Create Packages files: +for component in COMPONENTS: + for arch_dir_path in glob.glob(os.path.join(distribution_path, component, 'binary-*')): + arch = os.path.basename(arch_dir_path).split('-')[1] + print('Creating package file for ' + component + " and " + arch + '...') + packages_file_path = arch_dir_path + '/Packages' + packagesxz_file_path = packages_file_path + '.xz' + binary_path = 'binary-' + arch + with open(packages_file_path, 'w') as packages_file: + for deb_to_read_path in sorted(glob.glob(os.path.join(arch_dir_path, "*.deb"))): + # Extract the control file from the .deb: + scanpackages_output = control_file_contents(deb_to_read_path) + package_name = re.search('Package: (.*)', scanpackages_output).group(1) + package_arch = re.search('Architecture: (.*)', scanpackages_output).group(1) + # Add these fields which dpkg-scanpackages would have done: + scanpackages_output += '\nFilename: ' + os.path.join('dists', DISTRIBUTION, component, binary_path, os.path.basename(deb_to_read_path)) + scanpackages_output += '\nSize: ' + str(os.stat(deb_to_read_path).st_size) + for hash in hashes: + if hash == "md5": + hash_string = hash.upper()+'Sum' + else: + hash_string = hash.upper() + scanpackages_output += '\n'+hash_string + ': ' + getattr(hashlib, hash)(open(deb_to_read_path, 'rb').read()).hexdigest() + print(scanpackages_output, file=packages_file) print('', file=packages_file) - # Extract the control file from the .deb: - scanpackages_output = control_file_contents(deb_to_read_path) - package_name = re.search('Package: (.*)', scanpackages_output).group(1) - package_arch = re.search('Architecture: (.*)', scanpackages_output).group(1) - # Add these fields which dpkg-scanpackages would have done: - scanpackages_output += '\nFilename: ' + os.path.join('dists', DISTRIBUTION, COMPONENT, binary_path, os.path.basename(deb_to_read_path)) - scanpackages_output += '\nSize: ' + str(os.stat(deb_to_read_path).st_size) - for hash in hashes: - if hash == "md5": - hash_string = hash.upper()+'Sum' - else: - hash_string = hash.upper() - scanpackages_output += '\n'+hash_string + ': ' + getattr(hashlib, hash)(open(deb_to_read_path, 'rb').read()).hexdigest() - print(scanpackages_output, file=packages_file) - - # Create Packages.xz - subprocess.check_call(['rm -f ' + packagesxz_file_path + '; xz -9 --keep ' + packages_file_path], shell=True, universal_newlines=True) + # Create Packages.xz + subprocess.check_call(['rm -f ' + packagesxz_file_path + '; xz -9 --keep ' + packages_file_path], shell=True, universal_newlines=True) +# Get components in output folder, we might have more folders than we are adding now +COMPONENTS = [d for d in os.listdir(distribution_path) if os.path.isdir(os.path.join(distribution_path, d))] for hash in hashes: if hash == 'md5': hash_string = hash.upper()+'Sum' @@ -118,21 +123,20 @@ for hash in hashes: hash_string = hash.upper() print(hash_string + ': ', file=release_file) - for arch in encountered_arches: - arch_dir_path = os.path.join(distribution_path, COMPONENT, 'binary-' + arch) - if not os.path.isdir(arch_dir_path): continue - packages_file_path = arch_dir_path + '/Packages' - packagesxz_file_path = packages_file_path + '.xz' - binary_path = 'binary-' + arch - # Write info about Packages and Packages.xz to Release file: - print(' '+' '.join([getattr(hashlib, hash)(open(packages_file_path, 'rb').read()).hexdigest(), - str(os.stat(packages_file_path).st_size), - os.path.join(COMPONENT, binary_path, 'Packages')]) - , file=release_file) - print(' '+' '.join([getattr(hashlib, hash)(open(packagesxz_file_path, 'rb').read()).hexdigest(), - str(os.stat(packagesxz_file_path).st_size), - os.path.join(COMPONENT, binary_path, 'Packages.xz')]) - , file=release_file) + for component in COMPONENTS: + for arch_dir_path in glob.glob(os.path.join(distribution_path, component, 'binary-*')): + packages_file_path = arch_dir_path + '/Packages' + packagesxz_file_path = packages_file_path + '.xz' + binary_path = os.path.basename(arch_dir_path) + # Write info about Packages and Packages.xz to Release file: + print(' '+' '.join([getattr(hashlib, hash)(open(packages_file_path, 'rb').read()).hexdigest(), + str(os.stat(packages_file_path).st_size), + os.path.join(component, binary_path, 'Packages')]) + , file=release_file) + print(' '+' '.join([getattr(hashlib, hash)(open(packagesxz_file_path, 'rb').read()).hexdigest(), + str(os.stat(packagesxz_file_path).st_size), + os.path.join(component, binary_path, 'Packages.xz')]) + , file=release_file) release_file.close() if False: From 38eaae13354ba35dc15a9b20c040210a71eb84ea Mon Sep 17 00:00:00 2001 From: Henrik Grimler Date: Sat, 13 Oct 2018 15:59:26 +0200 Subject: [PATCH 06/15] Update hints about how to make repo available. Don't use [trusted=yes] and remove suggestion to install apt-transport-https (no longer needed) --- termux-apt-repo | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/termux-apt-repo b/termux-apt-repo index 47bdf16..df7cef9 100755 --- a/termux-apt-repo +++ b/termux-apt-repo @@ -153,5 +153,4 @@ print('') print('Users can then access the repo by adding a file at') print(' $PREFIX/etc/apt/sources.list.d') print('containing the single line:') -print(' deb [trusted=yes] $REPO_URL termux extras') -print('If the $REPO_URL is https, users must first install the apt-transport-https package') +print(' deb $REPO_URL termux extras') From 319c92b621ed803f0f57894c5d867dcde3fb3d31 Mon Sep 17 00:00:00 2001 From: Henrik Grimler Date: Sat, 13 Oct 2018 16:05:13 +0200 Subject: [PATCH 07/15] Update user hints to show how multi-component repos are enabled --- termux-apt-repo | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/termux-apt-repo b/termux-apt-repo index df7cef9..36f0a5e 100755 --- a/termux-apt-repo +++ b/termux-apt-repo @@ -152,5 +152,6 @@ print('Make the ' + output_path + ' directory accessible at $REPO_URL') print('') print('Users can then access the repo by adding a file at') print(' $PREFIX/etc/apt/sources.list.d') -print('containing the single line:') -print(' deb $REPO_URL termux extras') +print('containing:') +for component in COMPONENTS: + print(' deb $REPO_URL '+DISTRIBUTION+' '+component) From acde7921adfa524c0927bce6c349627565f42b2d Mon Sep 17 00:00:00 2001 From: Henrik Grimler Date: Sat, 13 Oct 2018 16:12:13 +0200 Subject: [PATCH 08/15] Sign with option --yes to overwrite existing files They were previously deleted when output_folder was deleted --- termux-apt-repo | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/termux-apt-repo b/termux-apt-repo index 36f0a5e..dd5b431 100755 --- a/termux-apt-repo +++ b/termux-apt-repo @@ -141,7 +141,7 @@ release_file.close() if False: print('Signing with gpg...') - subprocess.check_call(['gpg --pinentry-mode loopback --digest-algo SHA256 --clearsign -o ' + subprocess.check_call(['gpg --yes --pinentry-mode loopback --digest-algo SHA256 --clearsign -o ' + distribution_path + '/InRelease ' + distribution_path + '/Release'], shell=True, universal_newlines=True) From bb021f2b0eb90fb7c04308e61c34e9b2b1099add Mon Sep 17 00:00:00 2001 From: Henrik Grimler Date: Sat, 13 Oct 2018 16:34:43 +0200 Subject: [PATCH 09/15] Add hint on when [trusted=yes] is needed --- termux-apt-repo | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/termux-apt-repo b/termux-apt-repo index dd5b431..643d492 100755 --- a/termux-apt-repo +++ b/termux-apt-repo @@ -154,4 +154,6 @@ print('Users can then access the repo by adding a file at') print(' $PREFIX/etc/apt/sources.list.d') print('containing:') for component in COMPONENTS: - print(' deb $REPO_URL '+DISTRIBUTION+' '+component) + print(' deb [trusted=yes] $REPO_URL '+DISTRIBUTION+' '+component) +print('') +print('[trusted=yes] is not needed if the repo has been signed with a gpg key') From 1b649f622e0e6b19a370574bed4f231127ddb222 Mon Sep 17 00:00:00 2001 From: Henrik Grimler Date: Sat, 13 Oct 2018 17:49:03 +0200 Subject: [PATCH 10/15] Use argparse to handle command line arguments --- termux-apt-repo | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/termux-apt-repo b/termux-apt-repo index 643d492..bab8d07 100755 --- a/termux-apt-repo +++ b/termux-apt-repo @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -import datetime, hashlib, os, re, shutil, subprocess, sys, glob +import datetime, hashlib, os, re, shutil, subprocess, sys, glob, argparse DISTRIBUTION = 'termux' COMPONENTS = ['extras'] @@ -43,16 +43,22 @@ def add_deb(deb_to_add_path, component): destination_deb_file = os.path.join(dest_deb_dir_path, os.path.basename(deb_to_add_path)) shutil.copy2(deb_to_add_path, destination_deb_file) -if len(sys.argv) > 2: - input_path = sys.argv[1] - output_path = sys.argv[2] - if len(sys.argv) > 3: - DISTRIBUTION = sys.argv[3] - if len(sys.argv) > 4: - COMPONENTS = [sys.argv[4]] - distribution_path = os.path.join(output_path, 'dists', DISTRIBUTION) -else: - sys.exit('Usage: termux-apt-repo [ ]\n and defaults to "termux" and "extras" if arguments are omitted') +parser = argparse.ArgumentParser(description='Create a repository with deb files') +parser.add_argument('input', metavar='input', type=str, + help='folder where .deb files are located') +parser.add_argument('output', metavar='output', type=str, + help='folder with repository tree') +parser.add_argument('distribution', metavar='dist', type=str, nargs='?', default='termux', + help='name of distribution folder. deb files are put into output/dists/distribution/component/binary-$ARCH/') +parser.add_argument('component', metavar='comp', type=str, nargs='?', default='extras', + help='name of component folder. deb files are put into output/dists/distribution/component/binary-$ARCH/') + +args = parser.parse_args() +input_path = args.input +output_path = args.output +DISTRIBUTION = args.distribution +COMPONENTS = [args.component] +distribution_path = os.path.join(output_path, 'dists', DISTRIBUTION) if not os.path.isdir(input_path): sys.exit("'" + input_path + '" does not exist') From 481cbe24c532121179d0deed872e8ac5100eabf8 Mon Sep 17 00:00:00 2001 From: Henrik Grimler Date: Sat, 13 Oct 2018 17:49:58 +0200 Subject: [PATCH 11/15] Add option --use-hard-links Uses hard links instead of copying deb files to output folder. Won't work in termux but can save some disk space on a normal computer. --- termux-apt-repo | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/termux-apt-repo b/termux-apt-repo index bab8d07..c07e4c6 100755 --- a/termux-apt-repo +++ b/termux-apt-repo @@ -20,7 +20,7 @@ def control_file_contents(debfile): stderr=subprocess.DEVNULL ).strip() -def add_deb(deb_to_add_path, component): +def add_deb(deb_to_add_path, component, use_hard_links): deb_to_add_control_file = control_file_contents(deb_to_add_path) deb_to_add_pkg_name = re.search('Package: (.*)', deb_to_add_control_file).group(1) deb_to_add_pkg_version = re.search('Version: (.*)', deb_to_add_control_file).group(1) @@ -41,7 +41,10 @@ def add_deb(deb_to_add_path, component): dest_deb_dir_path = os.path.join(distribution_path, component, 'binary-' + deb_arch) if not os.path.isdir(dest_deb_dir_path): os.makedirs(dest_deb_dir_path) destination_deb_file = os.path.join(dest_deb_dir_path, os.path.basename(deb_to_add_path)) - shutil.copy2(deb_to_add_path, destination_deb_file) + if not use_hard_links: + shutil.copy2(deb_to_add_path, destination_deb_file) + else: + os.link(deb_to_add_path, destination_deb_file) parser = argparse.ArgumentParser(description='Create a repository with deb files') parser.add_argument('input', metavar='input', type=str, @@ -52,12 +55,15 @@ parser.add_argument('distribution', metavar='dist', type=str, nargs='?', default help='name of distribution folder. deb files are put into output/dists/distribution/component/binary-$ARCH/') parser.add_argument('component', metavar='comp', type=str, nargs='?', default='extras', help='name of component folder. deb files are put into output/dists/distribution/component/binary-$ARCH/') +parser.add_argument('--use-hard-links', default=False, action='store_true', + help='use hard links instead of copying deb files. Will not work on an android device') args = parser.parse_args() input_path = args.input output_path = args.output DISTRIBUTION = args.distribution COMPONENTS = [args.component] +use_hard_links = args.use_hard_links distribution_path = os.path.join(output_path, 'dists', DISTRIBUTION) if not os.path.isdir(input_path): @@ -79,7 +85,7 @@ else: component = COMPONENTS[0] else: COMPONENTS.append(component) - add_deb(deb_path, component) + add_deb(deb_path, component, use_hard_links) COMPONENTS = list(set(COMPONENTS)) # See https://wiki.debian.org/RepositoryFormat#A.22Release.22_files for format: From 7814e0f2c21ebf0a68f360dfbf399f9ffcd6135d Mon Sep 17 00:00:00 2001 From: Henrik Grimler Date: Sat, 13 Oct 2018 17:59:39 +0200 Subject: [PATCH 12/15] Correctly remove old component folders before starting to add debs --- termux-apt-repo | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/termux-apt-repo b/termux-apt-repo index c07e4c6..99920e7 100755 --- a/termux-apt-repo +++ b/termux-apt-repo @@ -2,8 +2,7 @@ import datetime, hashlib, os, re, shutil, subprocess, sys, glob, argparse -DISTRIBUTION = 'termux' -COMPONENTS = ['extras'] +COMPONENTS = [] supported_arches = ['all', 'arm', 'i686', 'aarch64', 'x86_64'] encountered_arches = set() hashes = ['md5', 'sha1', 'sha256', 'sha512'] @@ -62,18 +61,13 @@ args = parser.parse_args() input_path = args.input output_path = args.output DISTRIBUTION = args.distribution -COMPONENTS = [args.component] +default_component = args.component use_hard_links = args.use_hard_links distribution_path = os.path.join(output_path, 'dists', DISTRIBUTION) if not os.path.isdir(input_path): sys.exit("'" + input_path + '" does not exist') -for component in COMPONENTS: - if os.path.exists(os.path.join(distribution_path, component)): - shutil.rmtree(os.path.join(distribution_path, component)) - os.makedirs(os.path.join(distribution_path, component)) - debs_in_path = glob.glob(os.path.join(input_path, "*.deb")) debs_in_path += glob.glob(os.path.join(input_path, "*/*.deb")) if not debs_in_path: @@ -82,11 +76,13 @@ else: for deb_path in sorted(debs_in_path): component = os.path.dirname(os.path.relpath(deb_path, input_path)) if not component: - component = COMPONENTS[0] - else: + component = default_component + if component not in COMPONENTS: COMPONENTS.append(component) + shutil.rmtree(os.path.join(distribution_path, component)) + os.makedirs(os.path.join(distribution_path, component)) + add_deb(deb_path, component, use_hard_links) -COMPONENTS = list(set(COMPONENTS)) # See https://wiki.debian.org/RepositoryFormat#A.22Release.22_files for format: release_file_path = distribution_path + '/Release' From a79d2f4128d8df4d40babb26d468e37a8aabac4d Mon Sep 17 00:00:00 2001 From: Henrik Grimler Date: Sat, 13 Oct 2018 18:23:13 +0200 Subject: [PATCH 13/15] Update README to reflect changes as well --- README.rst | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/README.rst b/README.rst index ec9da98..56d8599 100644 --- a/README.rst +++ b/README.rst @@ -16,7 +16,20 @@ as: :: - termux-apt-repo + termux-apt-repo [-h] [--use-hard-links] input output [dist] [comp] + + positional arguments: + input folder where .deb files are located + output folder with repository tree + dist name of distribution folder. deb files are put into + output/dists/distribution/component/binary-$ARCH/ + comp name of component folder. deb files are put into + output/dists/distribution/component/binary-$ARCH/ + + optional arguments: + -h, --help show this help message and exit + --use-hard-links use hard links instead of copying deb files. Will not work + on an android device When using outside Termux (the script should work on most Linux distributions), install with ``pip3 install termux-apt-repo``. @@ -54,8 +67,9 @@ containing the single line: :: - deb [trusted=yes] $REPO_URL termux extras + deb [trusted=yes] $REPO_URL $dist $comp -If the published ``$REPO_URL`` is https, users must first install the -``apt-transport-https`` package which is not preinstalled (likely to -come preinstalled in the future). +[trusted=yes] is needed if the repo has not been signed with a gpg key. +To sign it, edit `termux-apt-repo` and change `if False:` to `if True:` near +end of script. The signing key then has to be imported by the user to make apt +trust it. From d7ed0b3bb52539876f1ac3f925e816fe71247116 Mon Sep 17 00:00:00 2001 From: Henrik Grimler Date: Sat, 13 Oct 2018 18:25:30 +0200 Subject: [PATCH 14/15] Fix code highlightning in README changes --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 56d8599..7736a51 100644 --- a/README.rst +++ b/README.rst @@ -69,7 +69,7 @@ containing the single line: deb [trusted=yes] $REPO_URL $dist $comp -[trusted=yes] is needed if the repo has not been signed with a gpg key. -To sign it, edit `termux-apt-repo` and change `if False:` to `if True:` near +``[trusted=yes]`` is needed if the repo has not been signed with a gpg key. +To sign it, edit ``termux-apt-repo`` and change ``if False:`` to ``if True:`` near end of script. The signing key then has to be imported by the user to make apt trust it. From 41757a2487469e3a2f491fa4a8b5de240e80efb7 Mon Sep 17 00:00:00 2001 From: Henrik Grimler Date: Sun, 14 Oct 2018 19:48:46 +0200 Subject: [PATCH 15/15] Fix error if output_path/dists doesn't exist --- termux-apt-repo | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/termux-apt-repo b/termux-apt-repo index 99920e7..8fe46b3 100755 --- a/termux-apt-repo +++ b/termux-apt-repo @@ -79,7 +79,8 @@ else: component = default_component if component not in COMPONENTS: COMPONENTS.append(component) - shutil.rmtree(os.path.join(distribution_path, component)) + if os.path.isdir(os.path.join(distribution_path, component)): + shutil.rmtree(os.path.join(distribution_path, component)) os.makedirs(os.path.join(distribution_path, component)) add_deb(deb_path, component, use_hard_links)