Download file using multiple parallel Range requests

download

#!/usr/bin/env ruby

require "bundler/inline"

gemfile do
  source "https://rubygems.org"
  gem "typhoeus"
end

require "uri"
require "typhoeus"

abort "Usage: ruby download.rb <url>" if ARGV.empty?
url = ARGV.first

response = Typhoeus.head(url)

if response.success?
  if content_length = response.headers["Content-Length"]
    total_size = content_length.to_i
  else
    abort "Content-Length header is missing, unable to determine file size."
  end
  if content_disposition = response.headers["Content-Disposition"]
    match = response.headers["Content-Disposition"].match(/filename="([^"]+)"/)
    filename = match[1] if match
  end
else
  abort "HEAD request failed: #{response.code}"
end

unless filename
  uri = URI.parse(url)
  filename = File.basename(uri.path)
end

num_parts = 4
part_size = (total_size / num_parts.to_f).ceil

ranges = Array.new(num_parts) do |i|
  start_byte = i * part_size
  end_byte = [start_byte + part_size - 1, total_size - 1].min
  "bytes=#{start_byte}-#{end_byte}"
end

hydra = Typhoeus::Hydra.new
parts = []

ranges.each_with_index do |range, index|
  request = Typhoeus::Request.new(url, headers: { "Range" => range })
  request.on_complete do |response|
    if response.success?
      file_name = "part_#{index}"
      File.write(file_name, response.body)
      parts << file_name
      puts response.headers["Content-Range"]
      puts "Downloaded part #{index}: #{response.code}"
    else
      puts "Failed to download part #{index}: #{response.code}"
    end
  end
  hydra.queue(request)
end

hydra.run

File.open(filename, "wb") do |output|
  parts.each do |part|
    File.open(part, "rb") do |input|
      IO.copy_stream(input, output)
    end
  end
end

parts.each { |part| File.delete(part) }

puts "Download complete: #{filename}"

@floehopper You should be able to do this pretty effectively with Async::HTTP too.

You should be able to do this pretty effectively with Async::HTTP too.

Presumably I'd still need a fair amount of custom code to do the HEAD request, work out what Range requests to make, and to reassemble the parts at the end...? Or does Async::HTTP have that built in?

It is not built in by default is it's quite a specific task, but there is an example here: https://github.com/socketry/async-http/blob/main/examples/download/chunked.rb

It is not built in by default is it's quite a specific task, but there is an example here: https://github.com/socketry/async-http/blob/main/examples/download/chunked.rb

Thanks 👍