fix(deps): support multiple index URLs per line and optimize downgrade check

- Rewrite _split_index_url() to handle multiple --index-url /
  --extra-index-url options on a single requirements.txt line using
  regex-based parsing instead of single split.
- Cache installed_packages snapshot in collect_requirements() to avoid
  repeated subprocess calls during downgrade blacklist checks.
- Add unit tests for multi-URL lines and bare --index-url edge case.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Dr.Lt.Data 2026-02-28 08:46:42 +09:00
parent da7e6f4454
commit 3d9c9ca8de
2 changed files with 86 additions and 18 deletions

View File

@ -261,6 +261,13 @@ class UnifiedDepResolver:
sources: dict[str, list[str]] = defaultdict(list)
extra_index_urls: list[str] = []
# Snapshot installed packages once to avoid repeated subprocess calls.
# Skip when downgrade_blacklist is empty (the common default).
installed_snapshot = (
manager_util.get_installed_packages()
if self.downgrade_blacklist else {}
)
for pack_path in self.node_pack_paths:
# Exclude disabled node packs (directory-based mechanism).
if self._is_disabled_path(pack_path):
@ -287,9 +294,8 @@ class UnifiedDepResolver:
# 1. Separate --index-url / --extra-index-url handling
# (before path separator check, because URLs contain '/')
if '--index-url' in line or '--extra-index-url' in line:
pkg_spec, index_url = self._split_index_url(line)
if index_url:
extra_index_urls.append(index_url)
pkg_spec, index_urls = self._split_index_url(line)
extra_index_urls.extend(index_urls)
line = pkg_spec
if not line:
# Standalone option line (no package prefix)
@ -317,7 +323,8 @@ class UnifiedDepResolver:
continue
# 5. Downgrade blacklist check
if self._is_downgrade_blacklisted(pkg_name, pkg_spec):
if self._is_downgrade_blacklisted(pkg_name, pkg_spec,
installed_snapshot):
skipped.append((pkg_spec, "downgrade blacklisted"))
continue
@ -510,20 +517,53 @@ class UnifiedDepResolver:
return manager_util.robust_readlines(filepath)
@staticmethod
def _split_index_url(line: str) -> tuple[str, str | None]:
"""Split ``'package --index-url URL'`` → ``(package, URL)``.
def _split_index_url(line: str) -> tuple[str, list[str]]:
"""Split index-url options from a requirement line.
Also handles standalone ``--index-url URL`` and
``--extra-index-url URL`` lines (with no package prefix).
Handles lines with one or more ``--index-url`` / ``--extra-index-url``
options. Returns ``(package_spec, [url, ...])``.
Examples::
"torch --extra-index-url U1 --index-url U2"
("torch", ["U1", "U2"])
"--index-url URL"
("", ["URL"])
"""
# Handle --extra-index-url first (contains '--index-url' as substring)
for option in ('--extra-index-url', '--index-url'):
if option in line:
parts = line.split(option, 1)
pkg_spec = parts[0].strip()
url = parts[1].strip() if len(parts) == 2 else None
return pkg_spec, url
return line, None
urls: list[str] = []
remainder_tokens: list[str] = []
# Regex: match --extra-index-url or --index-url followed by its value
option_re = re.compile(
r'(--(?:extra-)?index-url)\s+(\S+)'
)
# Pattern for bare option flags without a URL value
bare_option_re = re.compile(r'^--(?:extra-)?index-url$')
last_end = 0
for m in option_re.finditer(line):
# Text before this option is part of the package spec
before = line[last_end:m.start()].strip()
if before:
remainder_tokens.append(before)
urls.append(m.group(2))
last_end = m.end()
# Trailing text after last option
trailing = line[last_end:].strip()
if trailing:
remainder_tokens.append(trailing)
# Strip any bare option flags that leaked into remainder tokens
# (e.g. "--index-url" with no URL value after it)
remainder_tokens = [
t for t in remainder_tokens if not bare_option_re.match(t)
]
pkg_spec = " ".join(remainder_tokens).strip()
return pkg_spec, urls
def _remap_package(self, pkg: str) -> str:
"""Apply ``pip_overrides`` remapping."""
@ -539,15 +579,19 @@ class UnifiedDepResolver:
name = re.split(r'[><=!~;\[@ ]', spec)[0].strip()
return name.lower().replace('-', '_')
def _is_downgrade_blacklisted(self, pkg_name: str, pkg_spec: str) -> bool:
def _is_downgrade_blacklisted(self, pkg_name: str, pkg_spec: str,
installed: dict) -> bool:
"""Reproduce the downgrade logic from ``is_blacklisted()``.
Uses ``manager_util.StrictVersion`` **not** ``packaging.version``.
Args:
installed: Pre-fetched snapshot from
``manager_util.get_installed_packages()``.
"""
if pkg_name not in self.downgrade_blacklist:
return False
installed = manager_util.get_installed_packages()
match = _VERSION_SPEC_PATTERN.search(pkg_spec)
if match is None:

View File

@ -354,6 +354,30 @@ class TestIndexUrlSeparation:
assert deps.requirements[0].name == "torch"
assert "https://download.pytorch.org/whl/cu121" in deps.extra_index_urls
def test_multiple_index_urls_on_single_line(self, tmp_path):
"""Multiple --extra-index-url / --index-url on the same line."""
p = _make_node_pack(
str(tmp_path), "pack_a",
"torch --extra-index-url https://url1.example.com "
"--index-url https://url2.example.com\n",
)
r = _resolver([p])
deps = r.collect_requirements()
assert len(deps.requirements) == 1
assert deps.requirements[0].name == "torch"
assert "https://url1.example.com" in deps.extra_index_urls
assert "https://url2.example.com" in deps.extra_index_urls
def test_bare_index_url_no_value(self, tmp_path):
"""Bare ``--index-url`` with no URL value must not become a package."""
p = _make_node_pack(str(tmp_path), "pack_a",
"--index-url\nnumpy>=1.20\n")
r = _resolver([p])
deps = r.collect_requirements()
assert len(deps.requirements) == 1
assert deps.requirements[0].name == "numpy"
assert deps.extra_index_urls == []
# ===========================================================================
# Downgrade blacklist