From 815dbbebe13f6085150c6dea509bbb2ae9f123e1 Mon Sep 17 00:00:00 2001 From: pureliture Date: Sun, 21 Jun 2026 09:04:42 +0900 Subject: [PATCH] =?UTF-8?q?feat(deploy):=20personal-prod=20user=20?= =?UTF-8?q?=EC=9C=A0=EB=8B=9B=EA=B3=BC=20cache=20=EA=B2=A9=EB=A6=AC=20?= =?UTF-8?q?=EC=B6=94=EA=B0=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit personal-prod 배포에서 기존 scan cache와 충돌하지 않도록 SECURITY_SCANNER_CACHE_ROOT override를 지원한다. 신규 user-level personal systemd unit과 securityscanner.slice를 추가해 :4567 personal table, user manager, resource cap, IO idle scheduling 경로를 분리한다. 검증: uv run pytest -q Co-Authored-By: Codex GPT-5 --- CURRENT.md | 2 +- ...security-scanner-personal-baseline.service | 23 +++ .../security-scanner-personal-baseline.timer | 12 ++ ...ty-scanner-personal-freshness-eval.service | 25 ++++ ...rity-scanner-personal-freshness-eval.timer | 12 ++ ...ecurity-scanner-personal-incr-poll.service | 26 ++++ .../security-scanner-personal-incr-poll.timer | 12 ++ ...rity-scanner-personal-lease-reaper.service | 20 +++ ...curity-scanner-personal-lease-reaper.timer | 12 ++ ...rity-scanner-personal-scan-worker@.service | 27 ++++ .../security-scanner-personal-workers.target | 6 + deploy/systemd/user/securityscanner.slice | 13 ++ governance/autopilot_goal.yml | 3 +- governance/current.yml | 2 +- src/security_scanner/targets/fetcher.py | 3 + tests/test_fetcher.py | 19 +++ tests/test_personal_prod_systemd_units.py | 137 ++++++++++++++++++ 17 files changed, 351 insertions(+), 3 deletions(-) create mode 100644 deploy/systemd/user/security-scanner-personal-baseline.service create mode 100644 deploy/systemd/user/security-scanner-personal-baseline.timer create mode 100644 deploy/systemd/user/security-scanner-personal-freshness-eval.service create mode 100644 deploy/systemd/user/security-scanner-personal-freshness-eval.timer create mode 100644 deploy/systemd/user/security-scanner-personal-incr-poll.service create mode 100644 deploy/systemd/user/security-scanner-personal-incr-poll.timer create mode 100644 deploy/systemd/user/security-scanner-personal-lease-reaper.service create mode 100644 deploy/systemd/user/security-scanner-personal-lease-reaper.timer create mode 100644 deploy/systemd/user/security-scanner-personal-scan-worker@.service create mode 100644 deploy/systemd/user/security-scanner-personal-workers.target create mode 100644 deploy/systemd/user/securityscanner.slice create mode 100644 tests/test_personal_prod_systemd_units.py diff --git a/CURRENT.md b/CURRENT.md index 66ded2f..1854562 100644 --- a/CURRENT.md +++ b/CURRENT.md @@ -4,7 +4,7 @@ - Project: `security-scanner` - Merge mode: `guarded-auto-merge` -- Active goal: `phase-2a-sarif-product-complete` +- Active goal: `personal-prod-deploy` - Last auto merge: `ledger:20260617T003405Z-autopilot-3236f4` - Ledger entries: `4` - Ledger index hash: `sha256:e1893a649a1101b74a087b5eaaa275813a85708c5bb46c4ae70c24e10a111050` diff --git a/deploy/systemd/user/security-scanner-personal-baseline.service b/deploy/systemd/user/security-scanner-personal-baseline.service new file mode 100644 index 0000000..efeafb1 --- /dev/null +++ b/deploy/systemd/user/security-scanner-personal-baseline.service @@ -0,0 +1,23 @@ +[Unit] +Description=security-scanner personal baseline enqueue +Documentation=https://github.com/source-security-dev/security-scanner + +[Service] +Type=oneshot +Slice=securityscanner.slice +Nice=15 +IOSchedulingClass=idle +TasksMax=128 +WorkingDirectory=%h/security-scanner +EnvironmentFile=-%h/.config/security-scanner/personal-prod.env +Environment=PATH=%h/.local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/bin +Environment=SECURITY_SCANNER_STORAGE_BACKEND=dynamodb +Environment=SECURITY_SCANNER_DYNAMO_ENDPOINT=http://localhost:4567 +Environment=SECURITY_SCANNER_DYNAMO_TABLE=security_scanner_personal +Environment=SECURITY_SCANNER_CACHE_ROOT=%h/.cache/security-scanner-personal/repos +ExecStart=%h/.local/bin/uv run security-scanner baseline \ + --rolling-divisor 1 \ + --backpressure-threshold 1000 + +[Install] +WantedBy=default.target diff --git a/deploy/systemd/user/security-scanner-personal-baseline.timer b/deploy/systemd/user/security-scanner-personal-baseline.timer new file mode 100644 index 0000000..e0a7ea8 --- /dev/null +++ b/deploy/systemd/user/security-scanner-personal-baseline.timer @@ -0,0 +1,12 @@ +[Unit] +Description=Scheduler for security-scanner personal baseline enqueue +Documentation=https://github.com/source-security-dev/security-scanner + +[Timer] +OnCalendar=*-*-* 04:00:00 +Persistent=true +RandomizedDelaySec=1800 +Unit=security-scanner-personal-baseline.service + +[Install] +WantedBy=timers.target diff --git a/deploy/systemd/user/security-scanner-personal-freshness-eval.service b/deploy/systemd/user/security-scanner-personal-freshness-eval.service new file mode 100644 index 0000000..23faf71 --- /dev/null +++ b/deploy/systemd/user/security-scanner-personal-freshness-eval.service @@ -0,0 +1,25 @@ +[Unit] +Description=security-scanner personal freshness eval +Documentation=https://github.com/source-security-dev/security-scanner + +[Service] +Type=oneshot +Slice=securityscanner.slice +Nice=15 +IOSchedulingClass=idle +TasksMax=128 +WorkingDirectory=%h/security-scanner +EnvironmentFile=-%h/.config/security-scanner/personal-prod.env +Environment=PATH=%h/.local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/bin +Environment=SECURITY_SCANNER_STORAGE_BACKEND=dynamodb +Environment=SECURITY_SCANNER_DYNAMO_ENDPOINT=http://localhost:4567 +Environment=SECURITY_SCANNER_DYNAMO_TABLE=security_scanner_personal +ExecStart=%h/.local/bin/uv run security-scanner freshness-eval \ + --poll-interval-hours 0.0833333333 \ + --baseline-cadence-hours 24 \ + --margin-hours 1 \ + --backlog-alert-threshold 10 \ + --notification-log %h/.local/state/security-scanner/personal-alerts.log.jsonl + +[Install] +WantedBy=default.target diff --git a/deploy/systemd/user/security-scanner-personal-freshness-eval.timer b/deploy/systemd/user/security-scanner-personal-freshness-eval.timer new file mode 100644 index 0000000..f68987c --- /dev/null +++ b/deploy/systemd/user/security-scanner-personal-freshness-eval.timer @@ -0,0 +1,12 @@ +[Unit] +Description=Scheduler for security-scanner personal freshness eval +Documentation=https://github.com/source-security-dev/security-scanner + +[Timer] +OnCalendar=*:0/10:00 +Persistent=true +RandomizedDelaySec=120 +Unit=security-scanner-personal-freshness-eval.service + +[Install] +WantedBy=timers.target diff --git a/deploy/systemd/user/security-scanner-personal-incr-poll.service b/deploy/systemd/user/security-scanner-personal-incr-poll.service new file mode 100644 index 0000000..27d0666 --- /dev/null +++ b/deploy/systemd/user/security-scanner-personal-incr-poll.service @@ -0,0 +1,26 @@ +[Unit] +Description=security-scanner personal incremental poll +Documentation=https://github.com/source-security-dev/security-scanner + +[Service] +Type=oneshot +Slice=securityscanner.slice +Nice=15 +IOSchedulingClass=idle +TasksMax=128 +WorkingDirectory=%h/security-scanner +EnvironmentFile=-%h/.config/security-scanner/personal-prod.env +Environment=PATH=%h/.local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/bin +Environment=SECURITY_SCANNER_STORAGE_BACKEND=dynamodb +Environment=SECURITY_SCANNER_DYNAMO_ENDPOINT=http://localhost:4567 +Environment=SECURITY_SCANNER_DYNAMO_TABLE=security_scanner_personal +Environment=SECURITY_SCANNER_CACHE_ROOT=%h/.cache/security-scanner-personal/repos +ExecStart=%h/.local/bin/uv run security-scanner discover-updates \ + --enqueue \ + --from-catalog \ + --ls-remote-skip \ + --cadence-seconds 300 \ + --notification-log %h/.local/state/security-scanner/personal-incr-poll.log.jsonl + +[Install] +WantedBy=default.target diff --git a/deploy/systemd/user/security-scanner-personal-incr-poll.timer b/deploy/systemd/user/security-scanner-personal-incr-poll.timer new file mode 100644 index 0000000..88ea128 --- /dev/null +++ b/deploy/systemd/user/security-scanner-personal-incr-poll.timer @@ -0,0 +1,12 @@ +[Unit] +Description=Scheduler for security-scanner personal incremental poll +Documentation=https://github.com/source-security-dev/security-scanner + +[Timer] +OnCalendar=*:0/5:00 +Persistent=true +RandomizedDelaySec=60 +Unit=security-scanner-personal-incr-poll.service + +[Install] +WantedBy=timers.target diff --git a/deploy/systemd/user/security-scanner-personal-lease-reaper.service b/deploy/systemd/user/security-scanner-personal-lease-reaper.service new file mode 100644 index 0000000..29ddcf5 --- /dev/null +++ b/deploy/systemd/user/security-scanner-personal-lease-reaper.service @@ -0,0 +1,20 @@ +[Unit] +Description=security-scanner personal lease reaper +Documentation=https://github.com/source-security-dev/security-scanner + +[Service] +Type=oneshot +Slice=securityscanner.slice +Nice=15 +IOSchedulingClass=idle +TasksMax=128 +WorkingDirectory=%h/security-scanner +EnvironmentFile=-%h/.config/security-scanner/personal-prod.env +Environment=PATH=%h/.local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/bin +Environment=SECURITY_SCANNER_STORAGE_BACKEND=dynamodb +Environment=SECURITY_SCANNER_DYNAMO_ENDPOINT=http://localhost:4567 +Environment=SECURITY_SCANNER_DYNAMO_TABLE=security_scanner_personal +ExecStart=%h/.local/bin/uv run security-scanner reap-expired-leases + +[Install] +WantedBy=default.target diff --git a/deploy/systemd/user/security-scanner-personal-lease-reaper.timer b/deploy/systemd/user/security-scanner-personal-lease-reaper.timer new file mode 100644 index 0000000..2be0c9e --- /dev/null +++ b/deploy/systemd/user/security-scanner-personal-lease-reaper.timer @@ -0,0 +1,12 @@ +[Unit] +Description=Scheduler for security-scanner personal lease reaper +Documentation=https://github.com/source-security-dev/security-scanner + +[Timer] +OnUnitActiveSec=2min +Persistent=true +RandomizedDelaySec=15 +Unit=security-scanner-personal-lease-reaper.service + +[Install] +WantedBy=timers.target diff --git a/deploy/systemd/user/security-scanner-personal-scan-worker@.service b/deploy/systemd/user/security-scanner-personal-scan-worker@.service new file mode 100644 index 0000000..ed6c70e --- /dev/null +++ b/deploy/systemd/user/security-scanner-personal-scan-worker@.service @@ -0,0 +1,27 @@ +[Unit] +Description=security-scanner personal scan-worker instance %i +Documentation=https://github.com/source-security-dev/security-scanner +PartOf=security-scanner-personal-workers.target + +[Service] +Type=simple +Slice=securityscanner.slice +Nice=15 +IOSchedulingClass=idle +TasksMax=256 +WorkingDirectory=%h/security-scanner +EnvironmentFile=-%h/.config/security-scanner/personal-prod.env +Environment=PATH=%h/.local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/bin +Environment=SECURITY_SCANNER_STORAGE_BACKEND=dynamodb +Environment=SECURITY_SCANNER_DYNAMO_ENDPOINT=http://localhost:4567 +Environment=SECURITY_SCANNER_DYNAMO_TABLE=security_scanner_personal +Environment=SECURITY_SCANNER_CACHE_ROOT=%h/.cache/security-scanner-personal/repos +ExecStart=%h/.local/bin/uv run security-scanner scan-worker \ + --daemon \ + --worker-id security-scanner-personal-scan-worker@%i \ + --notification-log %h/.local/state/security-scanner/personal-scan-worker.log.jsonl +Restart=on-failure +RestartSec=10 + +[Install] +WantedBy=security-scanner-personal-workers.target diff --git a/deploy/systemd/user/security-scanner-personal-workers.target b/deploy/systemd/user/security-scanner-personal-workers.target new file mode 100644 index 0000000..8c8064d --- /dev/null +++ b/deploy/systemd/user/security-scanner-personal-workers.target @@ -0,0 +1,6 @@ +[Unit] +Description=security-scanner personal worker pool +Documentation=https://github.com/source-security-dev/security-scanner + +[Install] +WantedBy=default.target diff --git a/deploy/systemd/user/securityscanner.slice b/deploy/systemd/user/securityscanner.slice new file mode 100644 index 0000000..6ff55b0 --- /dev/null +++ b/deploy/systemd/user/securityscanner.slice @@ -0,0 +1,13 @@ +[Unit] +Description=security-scanner personal worker resource slice +Documentation=https://github.com/source-security-dev/security-scanner + +[Slice] +CPUAccounting=true +MemoryAccounting=true +IOAccounting=true +TasksAccounting=true +CPUQuota=150% +MemoryMax=3G +TasksMax=512 +IOWeight=100 diff --git a/governance/autopilot_goal.yml b/governance/autopilot_goal.yml index a99ff7d..81f5ce2 100644 --- a/governance/autopilot_goal.yml +++ b/governance/autopilot_goal.yml @@ -1,5 +1,5 @@ schema_version: 1 -goal_id: phase-2a-sarif-product-complete +goal_id: personal-prod-deploy execution_mode: style: long-single-goal human_gate: stop-conditions-only @@ -20,6 +20,7 @@ allowed_writes: - docs/views/research-and-technical-decisions.md - src/security_scanner/** - tests/** + - deploy/systemd/user/** - examples/** - eval/** - docs/workbench/** diff --git a/governance/current.yml b/governance/current.yml index b06ca03..1ea16e8 100644 --- a/governance/current.yml +++ b/governance/current.yml @@ -37,7 +37,7 @@ gates: proof_ref: '' proof_hash: '' autopilot: - active_goal: phase-2a-sarif-product-complete + active_goal: personal-prod-deploy merge_mode: guarded-auto-merge last_auto_merge: ledger:20260617T003405Z-autopilot-3236f4 open_decisions: [] diff --git a/src/security_scanner/targets/fetcher.py b/src/security_scanner/targets/fetcher.py index 565f47d..490b689 100644 --- a/src/security_scanner/targets/fetcher.py +++ b/src/security_scanner/targets/fetcher.py @@ -43,6 +43,9 @@ class UnsupportedHostError(FetchError): def _default_cache_root() -> Path: + configured = os.environ.get("SECURITY_SCANNER_CACHE_ROOT") + if configured: + return Path(configured).expanduser() return Path.home() / ".cache" / "security-scanner" / "repos" diff --git a/tests/test_fetcher.py b/tests/test_fetcher.py index d4bbb38..0655981 100644 --- a/tests/test_fetcher.py +++ b/tests/test_fetcher.py @@ -140,6 +140,7 @@ def test_existing_cache_path_triggers_git_fetch(monkeypatch, tmp_path): def test_default_cache_root_uses_home(monkeypatch, tmp_path): """When cache_root is omitted, path is ~/.cache/security-scanner/repos//.""" + monkeypatch.delenv("SECURITY_SCANNER_CACHE_ROOT", raising=False) monkeypatch.setattr(Path, "home", classmethod(lambda cls: tmp_path)) calls = [] @@ -164,6 +165,24 @@ def test_default_cache_root_uses_home(monkeypatch, tmp_path): assert cmd[4] == str(expected_path) +def test_default_cache_root_can_be_overridden_by_env(monkeypatch, tmp_path): + cache_root = tmp_path / "personal-cache" + monkeypatch.setenv("SECURITY_SCANNER_CACHE_ROOT", str(cache_root)) + + calls = [] + monkeypatch.setattr( + "security_scanner.targets.fetcher.subprocess.run", + _record_run(calls), + ) + + result = fetch_or_clone("https://github.com/octocat/hello-world") + + expected_path = cache_root / "github.com" / "octocat" / "hello-world" + assert result == expected_path + cmd, _ = calls[0] + assert cmd[4] == str(expected_path) + + def test_missing_gh_falls_back_to_public_git_clone(monkeypatch, tmp_path): calls = [] diff --git a/tests/test_personal_prod_systemd_units.py b/tests/test_personal_prod_systemd_units.py new file mode 100644 index 0000000..624525b --- /dev/null +++ b/tests/test_personal_prod_systemd_units.py @@ -0,0 +1,137 @@ +"""Structure checks for personal-prod user systemd artifacts.""" + +from __future__ import annotations + +import configparser +from pathlib import Path + +import pytest + +USER_SYSTEMD_DIR = Path(__file__).resolve().parents[1] / "deploy" / "systemd" / "user" + +SLICE = USER_SYSTEMD_DIR / "securityscanner.slice" +WORKER_TEMPLATE = USER_SYSTEMD_DIR / "security-scanner-personal-scan-worker@.service" +WORKER_TARGET = USER_SYSTEMD_DIR / "security-scanner-personal-workers.target" + +PERIODIC_UNITS = { + "incr-poll": ( + "security-scanner-personal-incr-poll.service", + "security-scanner-personal-incr-poll.timer", + "*:0/5:00", + ), + "baseline": ( + "security-scanner-personal-baseline.service", + "security-scanner-personal-baseline.timer", + "*-*-* 04:00:00", + ), + "lease-reaper": ( + "security-scanner-personal-lease-reaper.service", + "security-scanner-personal-lease-reaper.timer", + None, + ), + "freshness-eval": ( + "security-scanner-personal-freshness-eval.service", + "security-scanner-personal-freshness-eval.timer", + "*:0/10:00", + ), +} + +ALL_SERVICE_FILES = [ + WORKER_TEMPLATE.name, + *(service for service, _, _ in PERIODIC_UNITS.values()), +] + + +def _parse_unit(path: Path) -> configparser.ConfigParser: + parser = configparser.ConfigParser(strict=False, interpolation=None) + parser.optionxform = str # type: ignore[assignment] + parser.read_string(path.read_text(encoding="utf-8")) + return parser + + +def test_personal_slice_sets_aggregate_caps() -> None: + parser = _parse_unit(SLICE) + slice_section = dict(parser.items("Slice")) + assert slice_section["CPUQuota"] == "150%" + assert slice_section["MemoryMax"] == "3G" + assert slice_section["TasksMax"] == "512" + assert slice_section["IOWeight"] == "100" + + +def test_personal_worker_template_is_capped_and_instanced() -> None: + parser = _parse_unit(WORKER_TEMPLATE) + service = dict(parser.items("Service")) + assert service["Type"] == "simple" + assert service["Slice"] == "securityscanner.slice" + assert service["Nice"] == "15" + assert service["IOSchedulingClass"] == "idle" + assert service["EnvironmentFile"] == "-%h/.config/security-scanner/personal-prod.env" + assert service["WorkingDirectory"] == "%h/security-scanner" + assert "scan-worker" in service["ExecStart"] + assert "--daemon" in service["ExecStart"] + assert "security-scanner-personal-scan-worker@%i" in service["ExecStart"] + assert service["Restart"] == "on-failure" + assert parser.get("Install", "WantedBy") == "security-scanner-personal-workers.target" + + +def test_personal_worker_target_is_user_level() -> None: + parser = _parse_unit(WORKER_TARGET) + assert parser.get("Install", "WantedBy") == "default.target" + + +@pytest.mark.parametrize( + "service_name,timer_name,calendar", + PERIODIC_UNITS.values(), + ids=list(PERIODIC_UNITS.keys()), +) +def test_personal_periodic_units_are_user_level_and_scheduled( + service_name: str, timer_name: str, calendar: str | None +) -> None: + service = _parse_unit(USER_SYSTEMD_DIR / service_name) + service_section = dict(service.items("Service")) + assert service_section["Type"] == "oneshot" + assert service_section["Slice"] == "securityscanner.slice" + assert service_section["Nice"] == "15" + assert service_section["IOSchedulingClass"] == "idle" + assert service_section["EnvironmentFile"] == "-%h/.config/security-scanner/personal-prod.env" + assert "uv run security-scanner" in service_section["ExecStart"] + assert service.get("Install", "WantedBy") == "default.target" + + timer = _parse_unit(USER_SYSTEMD_DIR / timer_name) + timer_section = dict(timer.items("Timer")) + assert timer_section["Unit"] == service_name + if calendar is None: + assert timer_section["OnUnitActiveSec"] == "2min" + else: + assert timer_section["OnCalendar"] == calendar + assert timer.get("Install", "WantedBy") == "timers.target" + + +@pytest.mark.parametrize("service_name", ALL_SERVICE_FILES) +def test_personal_units_use_personal_table_and_no_docker_mutation(service_name: str) -> None: + text = (USER_SYSTEMD_DIR / service_name).read_text(encoding="utf-8") + assert "SECURITY_SCANNER_DYNAMO_ENDPOINT=http://localhost:4567" in text + assert "SECURITY_SCANNER_DYNAMO_TABLE=security_scanner_personal" in text + assert "docker compose" not in text + assert "ExecStartPre" not in text + + +@pytest.mark.parametrize( + "service_name", + [ + "security-scanner-personal-scan-worker@.service", + "security-scanner-personal-incr-poll.service", + "security-scanner-personal-baseline.service", + ], +) +def test_personal_clone_cache_is_isolated(service_name: str) -> None: + text = (USER_SYSTEMD_DIR / service_name).read_text(encoding="utf-8") + assert "SECURITY_SCANNER_CACHE_ROOT=%h/.cache/security-scanner-personal/repos" in text + + +@pytest.mark.parametrize("path", [SLICE, WORKER_TEMPLATE, WORKER_TARGET]) +def test_personal_artifacts_avoid_machine_local_paths_and_accounts(path: Path) -> None: + text = path.read_text(encoding="utf-8") + forbidden = ("/home/", "/Users/", "/srv/", "/var/", "pureliture") + for marker in forbidden: + assert marker not in text