mirror of
https://github.com/pewdiepie-archdaemon/odysseus.git
synced 2026-09-10 18:22:20 +02:00
* fix(skill-importer): validate URL scheme and improve skills.sh handling * fix(skill-importer): enhance DNS resolution and SSRF protection in fetch URL handling * fix(url-safety): add allowed_dist parameter to check_outbound_url for flexible private blocking * test(skill-importer): add comprehensive tests for URL parsing and outbound checks * ensure newline at end of file in test_check_outbound_url_allows_public_ip * fix(skill-importer): improve TLS certificate handling in _get_checked function * fix(skill-importer): enhance _check_fetch_url to handle both hostnames and full URLs * fix(skill-importer): enhance parse_skill_source to support skills.sh URLs in path and netloc * fix(skill-importer): simplify skills.sh hostname check in parse_skill_source * fix(skill-importer): enhance parse_skill_source to identify skills.sh URLs in path and handle localhost/IP addresses * fix(skill-importer): enhance _resolve_and_check_url to validate all resolved IP addresses and prevent TOCTOU vulnerabilities * fix(skill-importer): enhance parse_skill_source to support schemeless GitHub and skills.sh URLs * fix(memory): resolve CodeQL URL sanitization warning and restore _check_fetch_url test alias * fix(memory): pin skill fetch sockets without rewriting URLs * fix(memory): reject unsupported skill wrapper hosts * refactor(url-safety): remove unused importer exception * test(memory): keep redirect regression hermetic * test(dns-rebinding): add test for _PinnedTransport to ensure connection to pinned IP * fix(skill-importer): enhance skills.sh support to extract GitHub links from page content * fix(skill-importer): improve URL scheme validation for GitHub and skills.sh links * fix(skills): reject unusable skill URLs instead of guessing Resolving a skills.sh link by scraping the first github.com URL out of the page body cannot work. Skill pages only ever link the repository root, never the skill's subdirectory, so every skill in a repo resolved to the same bundle: importing skills.sh/anthropics/skills/pdf walked the whole monorepo, saturated the 64-file cap, and installed algorithmic-art behind an ok:true response. Restore the redirect-target unwrap and fail with a message that says what to do instead. Also report the real reason a URL is rejected. The scheme check keyed off "://" appearing anywhere in the string, so a supplied-but-unusable URL came back as "URL is required", and a schemeless URL carrying "://" in its query was reported as an unsupported scheme. Key off the parsed scheme and let opaque schemes (mailto:, javascript:) and a schemeless host:port fall through to the host check. * test(skills): tighten the real-socket pinning regression The handler swallowed its own exceptions, so a failure inside it surfaced as a confusing assertion on the captured client address. Record the exception and assert on it, run the thread as a daemon, and close the listening socket from the test so a hang cannot outlive the run. Also drop the duplicate ipaddress import and the missing newline. * fix(skills): require exact GitHub skill URLs * test(skills): read complete pinned request headers --------- Co-authored-by: RaresKeY <158580472+RaresKeY@users.noreply.github.com> Co-authored-by: Léo <leograndcontact@gmail.com>
179 lines
4.7 KiB
Python
179 lines
4.7 KiB
Python
"""Skill URL importer — GitHub path parsing."""
|
|
import ipaddress
|
|
|
|
import pytest
|
|
|
|
from services.memory.skill_importer import (
|
|
ResolvedSource,
|
|
SkillImportError,
|
|
_assert_github_url,
|
|
_fetch_bytes,
|
|
_list_github_dir,
|
|
parse_skill_source,
|
|
)
|
|
|
|
|
|
def _allow_fetch(monkeypatch):
|
|
monkeypatch.setattr(
|
|
"services.memory.skill_importer._resolve_and_check_url",
|
|
lambda url: [ipaddress.ip_address("93.184.216.34")],
|
|
)
|
|
|
|
|
|
def test_parse_github_blob_skill_md():
|
|
src = parse_skill_source(
|
|
"https://github.com/anthropics/skills/blob/main/skills/pdf/SKILL.md"
|
|
)
|
|
assert src.owner == "anthropics"
|
|
assert src.repo == "skills"
|
|
assert src.ref == "main"
|
|
assert src.path.endswith("skills/pdf/SKILL.md")
|
|
|
|
|
|
def test_parse_github_tree_directory():
|
|
src = parse_skill_source(
|
|
"https://github.com/example/my-skills/tree/develop/caveman-skill"
|
|
)
|
|
assert src.owner == "example"
|
|
assert src.repo == "my-skills"
|
|
assert src.ref == "develop"
|
|
assert src.path == "caveman-skill"
|
|
|
|
|
|
def test_parse_raw_github():
|
|
src = parse_skill_source(
|
|
"https://raw.githubusercontent.com/o/r/main/path/SKILL.md"
|
|
)
|
|
assert src.owner == "o"
|
|
assert src.repo == "r"
|
|
assert src.ref == "main"
|
|
assert src.path == "path/SKILL.md"
|
|
|
|
|
|
def test_rejects_non_github():
|
|
with pytest.raises(SkillImportError):
|
|
parse_skill_source("https://example.com/skill.md")
|
|
|
|
|
|
def test_fetch_bytes_rejects_cross_host_redirect(monkeypatch):
|
|
class _Resp:
|
|
url = "https://evil.example/secret"
|
|
status_code = 200
|
|
content = b"x"
|
|
|
|
def raise_for_status(self):
|
|
return None
|
|
|
|
class _Client:
|
|
def __init__(self, *args, **kwargs):
|
|
pass
|
|
|
|
def __enter__(self):
|
|
return self
|
|
|
|
def __exit__(self, *args):
|
|
return False
|
|
|
|
def get(self, url, headers=None):
|
|
return _Resp()
|
|
|
|
monkeypatch.setattr("services.memory.skill_importer.httpx.Client", _Client)
|
|
_allow_fetch(monkeypatch)
|
|
with pytest.raises(SkillImportError, match="redirect target"):
|
|
_fetch_bytes("https://raw.githubusercontent.com/o/r/main/SKILL.md")
|
|
|
|
|
|
def test_assert_github_url_allows_api_host():
|
|
_assert_github_url(
|
|
"https://api.github.com/repos/o/r/contents?ref=main",
|
|
context="redirect target",
|
|
)
|
|
|
|
|
|
def test_list_github_dir_accepts_api_github_response(monkeypatch):
|
|
monkeypatch.setattr(
|
|
"services.memory.skill_importer._fetch_text",
|
|
lambda url: "# skill\n",
|
|
)
|
|
_allow_fetch(monkeypatch)
|
|
|
|
class _Resp:
|
|
url = "https://api.github.com/repos/o/r/contents?ref=main"
|
|
status_code = 200
|
|
|
|
def raise_for_status(self):
|
|
return None
|
|
|
|
def json(self):
|
|
return [{
|
|
"name": "SKILL.md",
|
|
"type": "file",
|
|
"download_url": "https://raw.githubusercontent.com/o/r/main/SKILL.md",
|
|
}]
|
|
|
|
class _Client:
|
|
def __init__(self, *args, **kwargs):
|
|
pass
|
|
|
|
def __enter__(self):
|
|
return self
|
|
|
|
def __exit__(self, *args):
|
|
return False
|
|
|
|
def get(self, url, headers=None):
|
|
return _Resp()
|
|
|
|
monkeypatch.setattr("services.memory.skill_importer.httpx.Client", _Client)
|
|
|
|
out = {}
|
|
src = ResolvedSource(owner="o", repo="r", ref="main", path="")
|
|
_list_github_dir(src, "", out)
|
|
assert "SKILL.md" in out
|
|
|
|
|
|
def _mock_httpx_client(monkeypatch, response):
|
|
class _Client:
|
|
def __init__(self, *args, **kwargs):
|
|
pass
|
|
|
|
def __enter__(self):
|
|
return self
|
|
|
|
def __exit__(self, *args):
|
|
return False
|
|
|
|
def get(self, url, headers=None):
|
|
return response
|
|
|
|
monkeypatch.setattr("services.memory.skill_importer.httpx.Client", _Client)
|
|
_allow_fetch(monkeypatch)
|
|
|
|
|
|
def test_list_github_dir_surfaces_rate_limit(monkeypatch):
|
|
class _Resp:
|
|
url = "https://api.github.com/repos/o/r/contents?ref=main"
|
|
status_code = 403
|
|
|
|
def json(self):
|
|
return {"message": "API rate limit exceeded for 203.0.113.1"}
|
|
|
|
_mock_httpx_client(monkeypatch, _Resp())
|
|
src = ResolvedSource(owner="o", repo="r", ref="main", path="")
|
|
with pytest.raises(SkillImportError, match="rate limit"):
|
|
_list_github_dir(src, "", {})
|
|
|
|
|
|
def test_fetch_bytes_surfaces_github_error_detail(monkeypatch):
|
|
class _Resp:
|
|
url = "https://raw.githubusercontent.com/o/r/main/SKILL.md"
|
|
status_code = 403
|
|
content = b""
|
|
|
|
def json(self):
|
|
return {"message": "Forbidden"}
|
|
|
|
_mock_httpx_client(monkeypatch, _Resp())
|
|
with pytest.raises(SkillImportError, match="GitHub request failed \\(403\\): Forbidden"):
|
|
_fetch_bytes("https://raw.githubusercontent.com/o/r/main/SKILL.md")
|