[rubygems/rubygems] Reduce allocations when parsing compact index

This still allocates a ton (a string for each line, plus a bunch of
splits into arrays), but it helps a bit when Bundler has to go through
dependency resolution.

```
==> memprof.after.txt <==
Total allocated: 194.14 MB (2317172 objects)
Total retained:  60.81 MB (593164 objects)

==> memprof.before.txt <==
Total allocated: 211.97 MB (2404890 objects)
Total retained:  62.85 MB (640342 objects)
```

https://github.com/rubygems/rubygems/commit/c68b41b0e5
This commit is contained in:
Samuel Giddins 2023-09-18 13:56:20 -07:00 committed by git
parent bf71b0eda5
commit 2b6228be48
4 changed files with 23 additions and 10 deletions

View File

@ -32,11 +32,11 @@ module Bundler
name, versions_string, info_checksum = line.split(" ", 3)
info_checksums_by_name[name] = info_checksum || ""
versions_string.split(",").each do |version|
if version.start_with?("-")
version = version[1..-1].split("-", 2).unshift(name)
delete = version.delete_prefix!("-")
version = version.split("-", 2).unshift(name)
if delete
versions_by_name[name].delete(version)
else
version = version.split("-", 2).unshift(name)
versions_by_name[name] << version
end
end

View File

@ -6,12 +6,15 @@ module Bundler
GemParser = Gem::Resolver::APISet::GemParser
else
class GemParser
EMPTY_ARRAY = [].freeze
private_constant :EMPTY_ARRAY
def parse(line)
version_and_platform, rest = line.split(" ", 2)
version, platform = version_and_platform.split("-", 2)
dependencies, requirements = rest.split("|", 2).map {|s| s.split(",") } if rest
dependencies = dependencies ? dependencies.map {|d| parse_dependency(d) } : []
requirements = requirements ? requirements.map {|d| parse_dependency(d) } : []
dependencies, requirements = rest.split("|", 2).map! {|s| s.split(",") } if rest
dependencies = dependencies ? dependencies.map! {|d| parse_dependency(d) } : EMPTY_ARRAY
requirements = requirements ? requirements.map! {|d| parse_dependency(d) } : EMPTY_ARRAY
[version, platform, dependencies, requirements]
end
@ -20,6 +23,7 @@ module Bundler
def parse_dependency(string)
dependency = string.split(":")
dependency[-1] = dependency[-1].split("&") if dependency.size > 1
dependency[0] = -dependency[0]
dependency
end
end

View File

@ -95,7 +95,12 @@ module Bundler
# because we need to preserve \n line endings on windows when calculating
# the checksum
SharedHelpers.filesystem_access(path, :read) do
SharedHelpers.digest(:MD5).hexdigest(File.read(path))
File.open(path, "rb") do |f|
digest = SharedHelpers.digest(:MD5).new
buf = String.new(:capacity => 16_384, :encoding => Encoding::BINARY)
digest << buf while f.read(16_384, buf)
digest.hexdigest
end
end
end

View File

@ -1,12 +1,15 @@
# frozen_string_literal: true
class Gem::Resolver::APISet::GemParser
EMPTY_ARRAY = [].freeze
private_constant :EMPTY_ARRAY
def parse(line)
version_and_platform, rest = line.split(" ", 2)
version, platform = version_and_platform.split("-", 2)
dependencies, requirements = rest.split("|", 2).map {|s| s.split(",") } if rest
dependencies = dependencies ? dependencies.map {|d| parse_dependency(d) } : []
requirements = requirements ? requirements.map {|d| parse_dependency(d) } : []
dependencies, requirements = rest.split("|", 2).map! {|s| s.split(",") } if rest
dependencies = dependencies ? dependencies.map! {|d| parse_dependency(d) } : EMPTY_ARRAY
requirements = requirements ? requirements.map! {|d| parse_dependency(d) } : EMPTY_ARRAY
[version, platform, dependencies, requirements]
end
@ -15,6 +18,7 @@ class Gem::Resolver::APISet::GemParser
def parse_dependency(string)
dependency = string.split(":")
dependency[-1] = dependency[-1].split("&") if dependency.size > 1
dependency[0] = -dependency[0]
dependency
end
end