downloader.rb: Use keyword arguments
This commit is contained in:
parent
f2c9eac887
commit
0d6506170e
Notes:
git
2025-01-16 07:14:37 +00:00
@ -1,6 +1,8 @@
|
|||||||
# Used by configure and make to download or update mirrored Ruby and GCC
|
# Used by configure and make to download or update mirrored Ruby and GCC
|
||||||
# files. This will use HTTPS if possible, falling back to HTTP.
|
# files. This will use HTTPS if possible, falling back to HTTP.
|
||||||
|
|
||||||
|
# -*- frozen-string-literal: true -*-
|
||||||
|
|
||||||
require 'fileutils'
|
require 'fileutils'
|
||||||
require 'open-uri'
|
require 'open-uri'
|
||||||
require 'pathname'
|
require 'pathname'
|
||||||
@ -59,30 +61,29 @@ class Downloader
|
|||||||
end
|
end
|
||||||
|
|
||||||
class GNU < self
|
class GNU < self
|
||||||
def self.download(name, *rest)
|
def self.download(name, *rest, **options)
|
||||||
if https?
|
if https?
|
||||||
begin
|
begin
|
||||||
super("https://cdn.jsdelivr.net/gh/gcc-mirror/gcc@master/#{name}", name, *rest)
|
super("https://cdn.jsdelivr.net/gh/gcc-mirror/gcc@master/#{name}", name, *rest, **options)
|
||||||
rescue => e
|
rescue => e
|
||||||
m1, m2 = e.message.split("\n", 2)
|
m1, m2 = e.message.split("\n", 2)
|
||||||
STDERR.puts "Download failed (#{m1}), try another URL\n#{m2}"
|
STDERR.puts "Download failed (#{m1}), try another URL\n#{m2}"
|
||||||
super("https://raw.githubusercontent.com/gcc-mirror/gcc/master/#{name}", name, *rest)
|
super("https://raw.githubusercontent.com/gcc-mirror/gcc/master/#{name}", name, *rest, **options)
|
||||||
end
|
end
|
||||||
else
|
else
|
||||||
super("https://repo.or.cz/official-gcc.git/blob_plain/HEAD:/#{name}", name, *rest)
|
super("https://repo.or.cz/official-gcc.git/blob_plain/HEAD:/#{name}", name, *rest, **options)
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
class RubyGems < self
|
class RubyGems < self
|
||||||
def self.download(name, dir = nil, since = true, options = {})
|
def self.download(name, dir = nil, since = true, **options)
|
||||||
require 'rubygems'
|
require 'rubygems'
|
||||||
options = options.dup
|
|
||||||
options[:ssl_ca_cert] = Dir.glob(File.expand_path("../lib/rubygems/ssl_certs/**/*.pem", File.dirname(__FILE__)))
|
options[:ssl_ca_cert] = Dir.glob(File.expand_path("../lib/rubygems/ssl_certs/**/*.pem", File.dirname(__FILE__)))
|
||||||
if Gem::Version.new(name[/-\K[^-]*(?=\.gem\z)/]).prerelease?
|
if Gem::Version.new(name[/-\K[^-]*(?=\.gem\z)/]).prerelease?
|
||||||
options[:ignore_http_client_errors] = true
|
options[:ignore_http_client_errors] = true
|
||||||
end
|
end
|
||||||
super("https://rubygems.org/downloads/#{name}", name, dir, since, options)
|
super("https://rubygems.org/downloads/#{name}", name, dir, since, **options)
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
@ -107,16 +108,13 @@ class Downloader
|
|||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
def self.download(name, dir = nil, since = true, options = {})
|
def self.download(name, dir = nil, since = true, unicode_beta: nil, **options)
|
||||||
options = options.dup
|
|
||||||
unicode_beta = options.delete(:unicode_beta)
|
|
||||||
name_dir_part = name.sub(/[^\/]+$/, '')
|
name_dir_part = name.sub(/[^\/]+$/, '')
|
||||||
if unicode_beta == 'YES'
|
if unicode_beta == 'YES'
|
||||||
if INDEX.size == 0
|
if INDEX.size == 0
|
||||||
index_options = options.dup
|
cache_save = false # TODO: make sure caching really doesn't work for index file
|
||||||
index_options[:cache_save] = false # TODO: make sure caching really doesn't work for index file
|
|
||||||
index_data = File.read(under(dir, "index.html")) rescue nil
|
index_data = File.read(under(dir, "index.html")) rescue nil
|
||||||
index_file = super(UNICODE_PUBLIC+name_dir_part, "#{name_dir_part}index.html", dir, true, index_options)
|
index_file = super(UNICODE_PUBLIC+name_dir_part, "#{name_dir_part}index.html", dir, true, cache_save: cache_save, **options)
|
||||||
INDEX[:index] = File.read(index_file)
|
INDEX[:index] = File.read(index_file)
|
||||||
since = true unless INDEX[:index] == index_data
|
since = true unless INDEX[:index] == index_data
|
||||||
end
|
end
|
||||||
@ -125,7 +123,7 @@ class Downloader
|
|||||||
beta_name = INDEX[:index][/#{Regexp.quote(file_base)}(-[0-9.]+d\d+)?\.txt/]
|
beta_name = INDEX[:index][/#{Regexp.quote(file_base)}(-[0-9.]+d\d+)?\.txt/]
|
||||||
# make sure we always check for new versions of files,
|
# make sure we always check for new versions of files,
|
||||||
# because they can easily change in the beta period
|
# because they can easily change in the beta period
|
||||||
super(UNICODE_PUBLIC+name_dir_part+beta_name, name, dir, since, options)
|
super(UNICODE_PUBLIC+name_dir_part+beta_name, name, dir, since, **options)
|
||||||
else
|
else
|
||||||
index_file = Pathname.new(under(dir, name_dir_part+'index.html'))
|
index_file = Pathname.new(under(dir, name_dir_part+'index.html'))
|
||||||
if index_file.exist? and name_dir_part !~ /^(12\.1\.0|emoji\/12\.0)/
|
if index_file.exist? and name_dir_part !~ /^(12\.1\.0|emoji\/12\.0)/
|
||||||
@ -133,7 +131,7 @@ class Downloader
|
|||||||
"Remove all files in this directory and in .downloaded-cache/ " +
|
"Remove all files in this directory and in .downloaded-cache/ " +
|
||||||
"because they may be leftovers from the beta period."
|
"because they may be leftovers from the beta period."
|
||||||
end
|
end
|
||||||
super(UNICODE_PUBLIC+name, name, dir, since, options)
|
super(UNICODE_PUBLIC+name, name, dir, since, **options)
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
@ -198,23 +196,20 @@ class Downloader
|
|||||||
# Example usage:
|
# Example usage:
|
||||||
# download 'http://www.unicode.org/Public/UCD/latest/ucd/UnicodeData.txt',
|
# download 'http://www.unicode.org/Public/UCD/latest/ucd/UnicodeData.txt',
|
||||||
# 'UnicodeData.txt', 'enc/unicode/data'
|
# 'UnicodeData.txt', 'enc/unicode/data'
|
||||||
def self.download(url, name, dir = nil, since = true, options = {})
|
def self.download(url, name, dir = nil, since = true,
|
||||||
options = options.dup
|
cache_save: ENV["CACHE_SAVE"] != "no", cache_dir: nil,
|
||||||
|
ignore_http_client_errors: nil,
|
||||||
|
dryrun: nil, verbose: false, **options)
|
||||||
url = URI(url)
|
url = URI(url)
|
||||||
dryrun = options.delete(:dryrun)
|
|
||||||
|
|
||||||
if name
|
if name
|
||||||
file = Pathname.new(under(dir, name))
|
file = Pathname.new(under(dir, name))
|
||||||
else
|
else
|
||||||
name = File.basename(url.path)
|
name = File.basename(url.path)
|
||||||
end
|
end
|
||||||
cache_save = options.delete(:cache_save) {
|
cache = cache_file(url, name, cache_dir)
|
||||||
ENV["CACHE_SAVE"] != "no"
|
|
||||||
}
|
|
||||||
cache = cache_file(url, name, options.delete(:cache_dir))
|
|
||||||
file ||= cache
|
file ||= cache
|
||||||
if since.nil? and file.exist?
|
if since.nil? and file.exist?
|
||||||
if $VERBOSE
|
if verbose
|
||||||
$stdout.puts "#{file} already exists"
|
$stdout.puts "#{file} already exists"
|
||||||
$stdout.flush
|
$stdout.flush
|
||||||
end
|
end
|
||||||
@ -227,7 +222,7 @@ class Downloader
|
|||||||
puts "Download #{url} into #{file}"
|
puts "Download #{url} into #{file}"
|
||||||
return
|
return
|
||||||
end
|
end
|
||||||
if link_cache(cache, file, name, $VERBOSE)
|
if link_cache(cache, file, name, verbose: verbose)
|
||||||
return file.to_path
|
return file.to_path
|
||||||
end
|
end
|
||||||
if !https? and URI::HTTPS === url
|
if !https? and URI::HTTPS === url
|
||||||
@ -235,12 +230,11 @@ class Downloader
|
|||||||
url.scheme = 'http'
|
url.scheme = 'http'
|
||||||
url = URI(url.to_s)
|
url = URI(url.to_s)
|
||||||
end
|
end
|
||||||
if $VERBOSE
|
if verbose
|
||||||
$stdout.print "downloading #{name} ... "
|
$stdout.print "downloading #{name} ... "
|
||||||
$stdout.flush
|
$stdout.flush
|
||||||
end
|
end
|
||||||
mtime = nil
|
mtime = nil
|
||||||
ignore_http_client_errors = options.delete(:ignore_http_client_errors)
|
|
||||||
options = options.merge(http_options(file, since.nil? ? true : since))
|
options = options.merge(http_options(file, since.nil? ? true : since))
|
||||||
begin
|
begin
|
||||||
data = with_retry(10) do
|
data = with_retry(10) do
|
||||||
@ -253,7 +247,7 @@ class Downloader
|
|||||||
rescue OpenURI::HTTPError => http_error
|
rescue OpenURI::HTTPError => http_error
|
||||||
case http_error.message
|
case http_error.message
|
||||||
when /^304 / # 304 Not Modified
|
when /^304 / # 304 Not Modified
|
||||||
if $VERBOSE
|
if verbose
|
||||||
$stdout.puts "#{name} not modified"
|
$stdout.puts "#{name} not modified"
|
||||||
$stdout.flush
|
$stdout.flush
|
||||||
end
|
end
|
||||||
@ -288,7 +282,7 @@ class Downloader
|
|||||||
if mtime
|
if mtime
|
||||||
dest.utime(mtime, mtime)
|
dest.utime(mtime, mtime)
|
||||||
end
|
end
|
||||||
if $VERBOSE
|
if verbose
|
||||||
$stdout.puts "done"
|
$stdout.puts "done"
|
||||||
$stdout.flush
|
$stdout.flush
|
||||||
end
|
end
|
||||||
@ -306,20 +300,24 @@ class Downloader
|
|||||||
dir ? File.join(dir, File.basename(name)) : name
|
dir ? File.join(dir, File.basename(name)) : name
|
||||||
end
|
end
|
||||||
|
|
||||||
|
def self.default_cache_dir
|
||||||
|
if cache_dir = ENV['CACHE_DIR']
|
||||||
|
return cache_dir unless cache_dir.empty?
|
||||||
|
end
|
||||||
|
".downloaded-cache"
|
||||||
|
end
|
||||||
|
|
||||||
def self.cache_file(url, name, cache_dir = nil)
|
def self.cache_file(url, name, cache_dir = nil)
|
||||||
case cache_dir
|
case cache_dir
|
||||||
when false
|
when false
|
||||||
return nil
|
return nil
|
||||||
when nil
|
when nil
|
||||||
cache_dir = ENV['CACHE_DIR']
|
cache_dir = default_cache_dir
|
||||||
if !cache_dir or cache_dir.empty?
|
|
||||||
cache_dir = ".downloaded-cache"
|
|
||||||
end
|
|
||||||
end
|
end
|
||||||
Pathname.new(cache_dir) + (name || File.basename(URI(url).path))
|
Pathname.new(cache_dir) + (name || File.basename(URI(url).path))
|
||||||
end
|
end
|
||||||
|
|
||||||
def self.link_cache(cache, file, name, verbose = false)
|
def self.link_cache(cache, file, name, verbose: false)
|
||||||
return false unless cache and cache.exist?
|
return false unless cache and cache.exist?
|
||||||
return true if cache.eql?(file)
|
return true if cache.eql?(file)
|
||||||
if /cygwin/ !~ RUBY_PLATFORM or /winsymlink:nativestrict/ =~ ENV['CYGWIN']
|
if /cygwin/ !~ RUBY_PLATFORM or /winsymlink:nativestrict/ =~ ENV['CYGWIN']
|
||||||
@ -460,7 +458,7 @@ if $0 == __FILE__
|
|||||||
end
|
end
|
||||||
ARGV.shift
|
ARGV.shift
|
||||||
end
|
end
|
||||||
$VERBOSE = true
|
options[:verbose] = true
|
||||||
if dl
|
if dl
|
||||||
args.each do |name|
|
args.each do |name|
|
||||||
dir = destdir
|
dir = destdir
|
||||||
@ -479,10 +477,10 @@ if $0 == __FILE__
|
|||||||
end
|
end
|
||||||
name = "#{prefix}/#{name}"
|
name = "#{prefix}/#{name}"
|
||||||
end
|
end
|
||||||
dl.download(name, dir, since, options)
|
dl.download(name, dir, since, **options)
|
||||||
end
|
end
|
||||||
else
|
else
|
||||||
abort "usage: #{$0} url name" unless args.size == 2
|
abort "usage: #{$0} url name" unless args.size == 2
|
||||||
Downloader.download(args[0], args[1], destdir, since, options)
|
Downloader.download(args[0], args[1], destdir, since, **options)
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
@ -43,7 +43,7 @@ class ExtLibs
|
|||||||
end
|
end
|
||||||
|
|
||||||
def do_download(url, cache_dir)
|
def do_download(url, cache_dir)
|
||||||
Downloader.download(url, nil, nil, nil, :cache_dir => cache_dir)
|
Downloader.download(url, nil, nil, nil, cache_dir: cache_dir)
|
||||||
end
|
end
|
||||||
|
|
||||||
def do_checksum(cache, chksums)
|
def do_checksum(cache, chksums)
|
||||||
|
Loading…
x
Reference in New Issue
Block a user