From e9c45519a3b0f716bda301dd171f365ec5718339 Mon Sep 17 00:00:00 2001 From: Tyler Lu Date: Wed, 8 Apr 2026 14:58:22 -0700 Subject: [PATCH] Add ruff format check to CI and enable strict mypy - Run `ruff format` across codebase (cosmetic: quotes, whitespace, trailing commas) - Add `ruff format --check` step to CI workflow - Enable `strict = true` for mypy (legacy files already excluded via ignore_errors) Co-Authored-By: Claude Opus 4.6 (1M context) --- .github/workflows/lint-type-test.yml | 5 +- deploy.py | 101 +++++++++-------- device.py | 155 +++++++++++++++------------ pyproject.toml | 5 +- src/beeutil/__init__.py | 19 ++-- src/beeutil/image_cache.py | 114 +++++++++++--------- src/beeutil/secrets.py | 45 ++++---- src/plugin/__init__.py | 3 +- src/plugin/example.py | 122 +++++++++++---------- util/__init__.py | 37 +++---- util/state_dump.py | 84 ++++++++------- util/test_secrets.py | 76 +++++++------ util/upload_secrets.py | 71 +++++++----- 13 files changed, 453 insertions(+), 384 deletions(-) diff --git a/.github/workflows/lint-type-test.yml b/.github/workflows/lint-type-test.yml index 5e9e130..c1f4292 100644 --- a/.github/workflows/lint-type-test.yml +++ b/.github/workflows/lint-type-test.yml @@ -35,9 +35,12 @@ jobs: - name: Install dev dependencies run: uv sync --group dev - - name: Run Ruff + - name: Ruff lint run: uv run ruff check . + - name: Ruff format + run: uv run ruff format --check . + - name: Run mypy run: uv run mypy diff --git a/deploy.py b/deploy.py index 199fe1b..95d7cb5 100644 --- a/deploy.py +++ b/deploy.py @@ -5,68 +5,75 @@ def get_upload_url(plugin_name, plugin_secret): - url = f'https://beemaps.com/api/plugins/upload/{plugin_name}?secret={plugin_secret}' - print(url) - res = requests.get(url) - if res.status_code != 200: - raise Exception(res.json()) + url = f"https://beemaps.com/api/plugins/upload/{plugin_name}?secret={plugin_secret}" + print(url) + res = requests.get(url) + if res.status_code != 200: + raise Exception(res.json()) + + res_data = res.json() + signed_url = res_data["url"] + return signed_url - res_data = res.json() - signed_url = res_data['url'] - return signed_url def plugin_hash(filepath): - sha256_hash = hashlib.sha256() - with open(filepath, 'rb') as plugin_bin: - for chunk in iter(lambda: plugin_bin.read(4096), b""): - sha256_hash.update(chunk) - return sha256_hash.hexdigest() + sha256_hash = hashlib.sha256() + with open(filepath, "rb") as plugin_bin: + for chunk in iter(lambda: plugin_bin.read(4096), b""): + sha256_hash.update(chunk) + return sha256_hash.hexdigest() + def upload_plugin(filepath, url): - with open(filepath, 'rb') as plugin_bin: - res = requests.put(url, data=plugin_bin) + with open(filepath, "rb") as plugin_bin: + res = requests.put(url, data=plugin_bin) + + if res.status_code != 200: + raise Exception(res.json()) - if res.status_code != 200: - raise Exception(res.json()) -def update_plugin(plugin_name, plugin_secret, filepath, version = 1): - print(f'[{plugin_name}] uploading {filepath}') - upload_url = get_upload_url(plugin_name, plugin_secret) - upload_plugin(filepath, upload_url) - sha256 = plugin_hash(filepath) +def update_plugin(plugin_name, plugin_secret, filepath, version=1): + print(f"[{plugin_name}] uploading {filepath}") + upload_url = get_upload_url(plugin_name, plugin_secret) + upload_plugin(filepath, upload_url) + sha256 = plugin_hash(filepath) - print(f'[{plugin_name}] registering {sha256}') + print(f"[{plugin_name}] registering {sha256}") - url = f'https://beemaps.com/api/plugins/{plugin_name}?secret={plugin_secret}' - data = { - "version": version, - "hash": sha256, - } - res = requests.put(url, json=data) - if res.status_code != 200: - raise Exception(res.json()) + url = f"https://beemaps.com/api/plugins/{plugin_name}?secret={plugin_secret}" + data = { + "version": version, + "hash": sha256, + } + res = requests.put(url, json=data) + if res.status_code != 200: + raise Exception(res.json()) + + return res.json() - return res.json() def plugin_info(plugin_name): - url = f'https://beemaps.com/api/plugins/{plugin_name}' - res = requests.get(url) - if res.status_code != 200: - raise Exception(res.json()) + url = f"https://beemaps.com/api/plugins/{plugin_name}" + res = requests.get(url) + if res.status_code != 200: + raise Exception(res.json()) + + return res.json() - return res.json() -if __name__ == '__main__': - parser = argparse.ArgumentParser(description="Upload and deploy bee plugins.") +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Upload and deploy bee plugins.") - parser.add_argument('-n', '--name', help="Plugin name.", type=str, required=True) - parser.add_argument('-s', '--secret', help="Plugin auth secret.", type=str, required=True) - parser.add_argument('-i', '--input_file', help="Path to build.sh output .py file.", type=str, required=True) - parser.add_argument('-v', '--version', help="Note-level version", default=1) + parser.add_argument("-n", "--name", help="Plugin name.", type=str, required=True) + parser.add_argument("-s", "--secret", help="Plugin auth secret.", type=str, required=True) + parser.add_argument( + "-i", "--input_file", help="Path to build.sh output .py file.", type=str, required=True + ) + parser.add_argument("-v", "--version", help="Note-level version", default=1) - args = parser.parse_args() + args = parser.parse_args() - update_plugin(args.name, args.secret, args.input_file, args.version) + update_plugin(args.name, args.secret, args.input_file, args.version) - info = plugin_info(args.name) - print(info) + info = plugin_info(args.name) + print(info) diff --git a/device.py b/device.py index 9e98aba..56ba762 100644 --- a/device.py +++ b/device.py @@ -6,96 +6,113 @@ from util import do_json_get, do_json_post -HOST_IP = '192.168.0.10' -HOST = f'http://{HOST_IP}:5000' -API_ROUTE = f'{HOST}/api/1' -WIFI_ROUTE = f'{API_ROUTE}/wifiClient' -CONNECTIVITY_ROUTE = f'{API_ROUTE}/config/uploadMode' +HOST_IP = "192.168.0.10" +HOST = f"http://{HOST_IP}:5000" +API_ROUTE = f"{HOST}/api/1" +WIFI_ROUTE = f"{API_ROUTE}/wifiClient" +CONNECTIVITY_ROUTE = f"{API_ROUTE}/config/uploadMode" + +TEMPLATE_PLUGIN_PATH = "/data/plugins/template-plugin/template-plugin" -TEMPLATE_PLUGIN_PATH = '/data/plugins/template-plugin/template-plugin' def run_command_over_ssh(ssh, cmd): - stdin, stdout, stderr = ssh.exec_command(cmd) + stdin, stdout, stderr = ssh.exec_command(cmd) + + stdout_output = stdout.read().decode().strip() + stderr_output = stderr.read().decode().strip() - stdout_output = stdout.read().decode().strip() - stderr_output = stderr.read().decode().strip() + if stdout_output: + return stdout_output + if stderr_output: + raise Exception(stderr_output) - if stdout_output: - return stdout_output - if stderr_output: - raise Exception(stderr_output) def toggle_client_connectivity_mode(mode): - return do_json_post(CONNECTIVITY_ROUTE, {'mode': mode}) + return do_json_post(CONNECTIVITY_ROUTE, {"mode": mode}) -def switch_to_lte_client_mode(): - toggle_client_connectivity_mode('lte') -def switch_to_wifi_client_mode(): - toggle_client_connectivity_mode('wifi') +def switch_to_lte_client_mode(): + toggle_client_connectivity_mode("lte") -def connect_to_wifi_network(ssid, password, security='WPA2', freq=2417): - config = { - 'ssid': ssid, - 'password': password, - 'enabled': 'true', - 'security': security, - 'freq': freq, - } - url = WIFI_ROUTE + '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/settings' - return do_json_post(url, config) +def switch_to_wifi_client_mode(): + toggle_client_connectivity_mode("wifi") -def scan_wifi_networks(): - url = WIFI_ROUTE + '/scan' - return do_json_get(url) -def wifi_settings(): - url = WIFI_ROUTE + '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/settings' - return do_json_get(url) +def connect_to_wifi_network(ssid, password, security="WPA2", freq=2417): + config = { + "ssid": ssid, + "password": password, + "enabled": "true", + "security": security, + "freq": freq, + } -def wifi_status(): - url = WIFI_ROUTE + '/status' - return do_json_get(url) + url = WIFI_ROUTE + "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/settings" + return do_json_post(url, config) -def info(): - url = f'{API_ROUTE}/info' - res = do_json_get(url) - return res -def calibration(): - with paramiko.SSHClient() as ssh: - ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - ssh.connect(HOST_IP, username='root', password="", look_for_keys=False) - return json.loads(str(run_command_over_ssh(ssh, 'cat /data/cache/calibration.json')).strip()) +def scan_wifi_networks(): + url = WIFI_ROUTE + "/scan" + return do_json_get(url) -if __name__ == '__main__': - parser = argparse.ArgumentParser(description="Local dev tooling for Bee Plugin development.") - parser.add_argument('-C', '--calibration', help="Device calibration profile", action='store_true') - parser.add_argument('-I', '--info', help="Device info", action='store_true') - parser.add_argument('-L', '--lte', help="Use LTE for connectivity", action='store_true') - parser.add_argument('-W', '--wifi_info', help="Show WiFi status", action='store_true') - parser.add_argument('-Ws', '--wifi_scan', help="Show visible WiFi networks", action='store_true') - parser.add_argument('-Wi', '--wifi_ssid', help="Use WiFi SSID for connectivity", type=str) - parser.add_argument('-P', '--password', help="Password", type=str, default="") +def wifi_settings(): + url = WIFI_ROUTE + "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/settings" + return do_json_get(url) - args = parser.parse_args() - if args.calibration: - pp(calibration()) +def wifi_status(): + url = WIFI_ROUTE + "/status" + return do_json_get(url) - if args.info: - pp(info()) - if args.wifi_info: - pp(wifi_status()) +def info(): + url = f"{API_ROUTE}/info" + res = do_json_get(url) + return res - if args.wifi_scan: - pp(scan_wifi_networks()) - if args.lte: - switch_to_lte_client_mode() - elif args.wifi_ssid: - switch_to_lte_client_mode() - connect_to_wifi_network(args.wifi_ssid, args.password) +def calibration(): + with paramiko.SSHClient() as ssh: + ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + ssh.connect(HOST_IP, username="root", password="", look_for_keys=False) + return json.loads( + str(run_command_over_ssh(ssh, "cat /data/cache/calibration.json")).strip() + ) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Local dev tooling for Bee Plugin development.") + + parser.add_argument( + "-C", "--calibration", help="Device calibration profile", action="store_true" + ) + parser.add_argument("-I", "--info", help="Device info", action="store_true") + parser.add_argument("-L", "--lte", help="Use LTE for connectivity", action="store_true") + parser.add_argument("-W", "--wifi_info", help="Show WiFi status", action="store_true") + parser.add_argument( + "-Ws", "--wifi_scan", help="Show visible WiFi networks", action="store_true" + ) + parser.add_argument("-Wi", "--wifi_ssid", help="Use WiFi SSID for connectivity", type=str) + parser.add_argument("-P", "--password", help="Password", type=str, default="") + + args = parser.parse_args() + + if args.calibration: + pp(calibration()) + + if args.info: + pp(info()) + + if args.wifi_info: + pp(wifi_status()) + + if args.wifi_scan: + pp(scan_wifi_networks()) + + if args.lte: + switch_to_lte_client_mode() + elif args.wifi_ssid: + switch_to_lte_client_mode() + connect_to_wifi_network(args.wifi_ssid, args.password) diff --git a/pyproject.toml b/pyproject.toml index 67b5097..31bd735 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,10 +57,7 @@ addopts = ["--import-mode=importlib", "-ra", "--strict-markers"] python_version = "3.9" files = ["src"] mypy_path = ["src"] -check_untyped_defs = true -no_implicit_optional = true -warn_redundant_casts = true -warn_unused_ignores = true +strict = true [[tool.mypy.overrides]] module = ["cryptography.*"] diff --git a/src/beeutil/__init__.py b/src/beeutil/__init__.py index c81f0c3..c2dafa1 100644 --- a/src/beeutil/__init__.py +++ b/src/beeutil/__init__.py @@ -12,10 +12,17 @@ from .secrets import DecryptionError, SecretsError, SecretsNetworkError, SecretsNotFoundError __all__ = [ - 'image_cache_status', 'enable_image_collection', 'disable_image_collection', - 'enable_stereo_collection', 'disable_stereo_collection', 'purge_data', - 'list_contents', 'upload_to_s3', - 'secrets', - 'SecretsError', 'DecryptionError', 'SecretsNetworkError', 'SecretsNotFoundError', + "image_cache_status", + "enable_image_collection", + "disable_image_collection", + "enable_stereo_collection", + "disable_stereo_collection", + "purge_data", + "list_contents", + "upload_to_s3", + "secrets", + "SecretsError", + "DecryptionError", + "SecretsNetworkError", + "SecretsNotFoundError", ] - diff --git a/src/beeutil/image_cache.py b/src/beeutil/image_cache.py index 929f6a5..3ac7770 100644 --- a/src/beeutil/image_cache.py +++ b/src/beeutil/image_cache.py @@ -1,76 +1,84 @@ import requests -HOST_URL = 'http://127.0.0.1:5000' -CACHE_ROUTE = f'{HOST_URL}/cache' +HOST_URL = "http://127.0.0.1:5000" +CACHE_ROUTE = f"{HOST_URL}/cache" + def image_cache_status(): - res = requests.get(f'{CACHE_ROUTE}/status') - if res.status_code != 200: - raise Exception(res.json()) + res = requests.get(f"{CACHE_ROUTE}/status") + if res.status_code != 200: + raise Exception(res.json()) + + return status # noqa: F821 - return status # noqa: F821 def enable_image_collection(): - res = requests.get(f'{CACHE_ROUTE}/enable') - if res.status_code != 200: - raise Exception(res.json()) + res = requests.get(f"{CACHE_ROUTE}/enable") + if res.status_code != 200: + raise Exception(res.json()) + + print(res.json()) - print(res.json()) def disable_image_collection(): - res = requests.get(f'{CACHE_ROUTE}/disable') - if res.status_code != 200: - raise Exception(res.json()) + res = requests.get(f"{CACHE_ROUTE}/disable") + if res.status_code != 200: + raise Exception(res.json()) + + print(res.json()) - print(res.json()) def purge_data(): - res = requests.get(f'{CACHE_ROUTE}/purge') - if res.status_code != 200: - raise Exception(res.json()) + res = requests.get(f"{CACHE_ROUTE}/purge") + if res.status_code != 200: + raise Exception(res.json()) + + print(res.json()) - print(res.json()) def enable_stereo_collection(): - res = requests.post(f'{CACHE_ROUTE}/enableDepthFlag') - if res.status_code != 200: - raise Exception(res.json()) + res = requests.post(f"{CACHE_ROUTE}/enableDepthFlag") + if res.status_code != 200: + raise Exception(res.json()) + + print(res.json()) - print(res.json()) def disable_stereo_collection(): - res = requests.post(f'{CACHE_ROUTE}/disableDepthFlag') - if res.status_code != 200: - raise Exception(res.json()) - - print(res.json()) - -def list_contents(since = None, until = None): - url = f'{CACHE_ROUTE}/list' - if since is not None or until is not None: - url += '?' - if since is not None: - url += f'since={since}' - if until is not None: - url += '&' - if until is not None: - url += f'until={until}' - - res = requests.get(url) - if res.status_code != 200: - raise Exception(res.json()) - - contents = res.json() - return contents + res = requests.post(f"{CACHE_ROUTE}/disableDepthFlag") + if res.status_code != 200: + raise Exception(res.json()) + + print(res.json()) + + +def list_contents(since=None, until=None): + url = f"{CACHE_ROUTE}/list" + if since is not None or until is not None: + url += "?" + if since is not None: + url += f"since={since}" + if until is not None: + url += "&" + if until is not None: + url += f"until={until}" + + res = requests.get(url) + if res.status_code != 200: + raise Exception(res.json()) + + contents = res.json() + return contents + def upload_to_s3(prefix, handle, aws_bucket, aws_region, aws_secret, aws_key): - url = f'{CACHE_ROUTE}/uploadS3/{handle}?prefix={prefix}&key={aws_key}&bucket={aws_bucket}®ion={aws_region}' - headers = { - 'authorization': aws_secret, - } - res = requests.post(url, headers) + url = f"{CACHE_ROUTE}/uploadS3/{handle}?prefix={prefix}&key={aws_key}&bucket={aws_bucket}®ion={aws_region}" + headers = { + "authorization": aws_secret, + } + res = requests.post(url, headers) - if res.status_code != 200: - raise Exception(res.json()) + if res.status_code != 200: + raise Exception(res.json()) - print (res.json()) + print(res.json()) diff --git a/src/beeutil/secrets.py b/src/beeutil/secrets.py index 87799bc..9dd232a 100644 --- a/src/beeutil/secrets.py +++ b/src/beeutil/secrets.py @@ -23,14 +23,14 @@ from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC -SALT = b'hivemapper-plugin-secrets' +SALT = b"hivemapper-plugin-secrets" PBKDF2_ITERATIONS = 100000 KEY_LENGTH = 32 IV_LENGTH = 16 BLOCK_SIZE = 128 -PLUGIN_DIR = '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/data/plugins' -ODC_API_BASE = 'http://127.0.0.1:5000/api/1' +PLUGIN_DIR = "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/data/plugins" +ODC_API_BASE = "http://127.0.0.1:5000/api/1" logger = logging.getLogger(__name__) @@ -38,31 +38,35 @@ class SecretsError(Exception): pass + class DecryptionError(SecretsError): pass + class SecretsNetworkError(SecretsError): pass + class SecretsNotFoundError(SecretsError): pass + def _derive_key(plugin_id: str) -> bytes: kdf = PBKDF2HMAC( algorithm=hashes.SHA256(), length=KEY_LENGTH, salt=SALT, iterations=PBKDF2_ITERATIONS, - backend=default_backend() + backend=default_backend(), ) - return kdf.derive(plugin_id.encode('utf-8')) + return kdf.derive(plugin_id.encode("utf-8")) def encrypt(plugin_id: str, env: dict) -> str: """Encrypt a dict of KV pairs. Used by deploy tooling.""" key = _derive_key(plugin_id) iv = os.urandom(IV_LENGTH) - plaintext = json.dumps(env).encode('utf-8') + plaintext = json.dumps(env).encode("utf-8") padder = padding.PKCS7(BLOCK_SIZE).padder() padded = padder.update(plaintext) + padder.finalize() @@ -72,17 +76,17 @@ def encrypt(plugin_id: str, env: dict) -> str: ciphertext = encryptor.update(padded) + encryptor.finalize() blob = { - 'iv': base64.b64encode(iv).decode('ascii'), - 'ciphertext': base64.b64encode(ciphertext).decode('ascii'), + "iv": base64.b64encode(iv).decode("ascii"), + "ciphertext": base64.b64encode(ciphertext).decode("ascii"), } - return base64.b64encode(json.dumps(blob).encode('utf-8')).decode('ascii') + return base64.b64encode(json.dumps(blob).encode("utf-8")).decode("ascii") def decrypt(plugin_id: str, encrypted_blob: str) -> dict: try: - blob = json.loads(base64.b64decode(encrypted_blob).decode('utf-8')) - iv = base64.b64decode(blob['iv']) - ciphertext = base64.b64decode(blob['ciphertext']) + blob = json.loads(base64.b64decode(encrypted_blob).decode("utf-8")) + iv = base64.b64decode(blob["iv"]) + ciphertext = base64.b64decode(blob["ciphertext"]) key = _derive_key(plugin_id) cipher = Cipher(algorithms.AES(key), modes.CBC(iv), backend=default_backend()) @@ -92,7 +96,7 @@ def decrypt(plugin_id: str, encrypted_blob: str) -> dict: unpadder = padding.PKCS7(BLOCK_SIZE).unpadder() plaintext = unpadder.update(padded) + unpadder.finalize() - return json.loads(plaintext.decode('utf-8')) + return json.loads(plaintext.decode("utf-8")) except (KeyError, ValueError, json.JSONDecodeError) as e: raise DecryptionError(f"Malformed encrypted blob: {e}") from e except Exception as e: @@ -104,11 +108,11 @@ def _parse_dotenv(path: str) -> dict: with open(path) as f: for line in f: line = line.strip() - if not line or line.startswith('#'): + if not line or line.startswith("#"): continue - if '=' not in line: + if "=" not in line: continue - key, _, value = line.partition('=') + key, _, value = line.partition("=") key = key.strip() value = value.strip() if len(value) >= 2 and value[0] == value[-1] and value[0] in ('"', "'"): @@ -118,11 +122,11 @@ def _parse_dotenv(path: str) -> dict: def _dotenv_path(plugin_name: str) -> str: - return os.path.join(PLUGIN_DIR, plugin_name, '.env') + return os.path.join(PLUGIN_DIR, plugin_name, ".env") def _fetch_from_odc(plugin_name: str) -> tuple: - url = f'{ODC_API_BASE}/plugin/secrets/{plugin_name}' + url = f"{ODC_API_BASE}/plugin/secrets/{plugin_name}" try: response = requests.get(url, timeout=10) except requests.RequestException as e: @@ -134,8 +138,8 @@ def _fetch_from_odc(plugin_name: str) -> tuple: raise SecretsNetworkError(f"ODC API error: {response.status_code}") data = response.json() - plugin_id = data.get('_id') - encrypted_blob = data.get('encrypted_secrets') + plugin_id = data.get("_id") + encrypted_blob = data.get("encrypted_secrets") if not plugin_id or not encrypted_blob: raise SecretsNotFoundError("ODC response missing _id or encrypted_secrets") @@ -191,4 +195,3 @@ def get(plugin_name: str, key: str) -> str: def load(plugin_name: str) -> dict: return dict(_load(plugin_name)) - diff --git a/src/plugin/__init__.py b/src/plugin/__init__.py index 3b5acce..d9f7898 100644 --- a/src/plugin/__init__.py +++ b/src/plugin/__init__.py @@ -1,4 +1,3 @@ from .example import main -__all__ = ['main'] - +__all__ = ["main"] diff --git a/src/plugin/example.py b/src/plugin/example.py index 997c440..7d11689 100644 --- a/src/plugin/example.py +++ b/src/plugin/example.py @@ -5,80 +5,86 @@ import beeutil -PLUGIN_NAME = 'your-plugin-name' +PLUGIN_NAME = "your-plugin-name" CAPTURE_STEREO = False LOOP_DELAY = 5 UPLOAD_THREADS = 1 VERBOSE = True + def vlog(msg): - if VERBOSE: - print(f'[{time.asctime()}] {msg}') + if VERBOSE: + print(f"[{time.asctime()}] {msg}") + def _setup(state): - vlog('enabling image caching') - beeutil.enable_image_collection() + vlog("enabling image caching") + beeutil.enable_image_collection() + + if CAPTURE_STEREO: + vlog("enabling stereo caching") + beeutil.enable_stereo_collection() + + state["session"] = str(uuid.uuid1()) + + vlog("loading env") + try: + beeutil.secrets.load(PLUGIN_NAME) + vlog("env loaded") + except beeutil.SecretsError as e: + vlog(f"ERROR: Failed to load env: {e}") + raise + + vlog(f"initializing {UPLOAD_THREADS} upload workers") + state["uploadQueue"] = queue.Queue() + + def upload_worker(): + while True: + handle = state["uploadQueue"].get() + beeutil.upload_to_s3( + state["session"], + handle, + beeutil.secrets.get(PLUGIN_NAME, "AWS_BUCKET"), + beeutil.secrets.get(PLUGIN_NAME, "AWS_REGION"), + beeutil.secrets.get(PLUGIN_NAME, "AWS_SECRET"), + beeutil.secrets.get(PLUGIN_NAME, "AWS_KEY"), + ) + + state["threads"] = [ + threading.Thread(target=upload_worker, daemon=True).start() for i in range(UPLOAD_THREADS) + ] + state["uploadQueue"] = queue.Queue() - if CAPTURE_STEREO: - vlog('enabling stereo caching') - beeutil.enable_stereo_collection() - state['session'] = str(uuid.uuid1()) +def _loop(state): + contents = beeutil.list_contents(state["last_checked"]) - vlog('loading env') - try: - beeutil.secrets.load(PLUGIN_NAME) - vlog('env loaded') - except beeutil.SecretsError as e: - vlog(f'ERROR: Failed to load env: {e}') - raise + if len(contents) == 0: + vlog(f"no new content since {state['last_checked']}") + return - vlog(f'initializing {UPLOAD_THREADS} upload workers') - state['uploadQueue'] = queue.Queue() + vlog(f"since {state['last_checked']}:") + vlog(contents) - def upload_worker(): - while True: - handle = state['uploadQueue'].get() - beeutil.upload_to_s3( - state['session'], - handle, - beeutil.secrets.get(PLUGIN_NAME, 'AWS_BUCKET'), - beeutil.secrets.get(PLUGIN_NAME, 'AWS_REGION'), - beeutil.secrets.get(PLUGIN_NAME, 'AWS_SECRET'), - beeutil.secrets.get(PLUGIN_NAME, 'AWS_KEY'), - ) - - state['threads'] = [threading.Thread(target=upload_worker, daemon=True).start() for i in range(UPLOAD_THREADS)] - state['uploadQueue'] = queue.Queue() + for handle in contents: + state["uploadQueue"].put(handle) -def _loop(state): - contents = beeutil.list_contents(state['last_checked']) - - if len(contents) == 0: - vlog(f'no new content since {state["last_checked"]}') - return + state["last_checked"] = contents[-1].split("_")[0] - vlog(f'since {state["last_checked"]}:') - vlog(contents) - for handle in contents: - state['uploadQueue'].put(handle) +def main(): + state = { + "last_checked": None, + "session": "", + "threads": None, + "uploadQueue": None, + } - state['last_checked'] = contents[-1].split('_')[0] + vlog("setting up plugin") + _setup(state) -def main(): - state = { - 'last_checked': None, - 'session': '', - 'threads': None, - 'uploadQueue': None, - } - - vlog('setting up plugin') - _setup(state) - - vlog('initializing run loop') - while True: - _loop(state) - time.sleep(LOOP_DELAY) + vlog("initializing run loop") + while True: + _loop(state) + time.sleep(LOOP_DELAY) diff --git a/util/__init__.py b/util/__init__.py index 6d6cb50..6c0fcc0 100644 --- a/util/__init__.py +++ b/util/__init__.py @@ -3,31 +3,32 @@ def do_json_get(url): - res = requests.get(url) + res = requests.get(url) - try: - res.raise_for_status() - except HTTPError: try: - print(res.json()) - except Exception: - pass + res.raise_for_status() + except HTTPError: + try: + print(res.json()) + except Exception: + pass - raise + raise + + return res.json() - return res.json() def do_json_post(url, data=None): - res = requests.post(url, json=data) + res = requests.post(url, json=data) - try: - res.raise_for_status() - except HTTPError: try: - print(res.json()) - except Exception: - pass + res.raise_for_status() + except HTTPError: + try: + print(res.json()) + except Exception: + pass - raise + raise - return res.json() + return res.json() diff --git a/util/state_dump.py b/util/state_dump.py index 8bd2e2e..da252e3 100644 --- a/util/state_dump.py +++ b/util/state_dump.py @@ -11,58 +11,58 @@ def collect_state_dump(host_ip): - timestamp = datetime.now().strftime('%Y-%m-%d-%H-%M-%S') - dump_dir = Path(f'state_dump_{timestamp}') - api_dir = dump_dir / 'api' + timestamp = datetime.now().strftime("%Y-%m-%d-%H-%M-%S") + dump_dir = Path(f"state_dump_{timestamp}") + api_dir = dump_dir / "api" api_dir.mkdir(parents=True, exist_ok=True) - - results = { - "api_endpoints": {}, - "files": {}, - "archive": "pending" - } - + + results = {"api_endpoints": {}, "files": {}, "archive": "pending"} + # 1. API Endpoints endpoints = [ - '/api/1/info', - '/api/1/lte-debug-info', - '/api/1/lte-debug-check-auth', - '/api/1/plugins', - '/api/1/config' + "/api/1/info", + "/api/1/lte-debug-info", + "/api/1/lte-debug-check-auth", + "/api/1/plugins", + "/api/1/config", ] - + # Files via SSH/SCP (to get counts first) remote_paths = [ - ('/data/', '*.log*'), - ('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/data/recording/', '*.log*'), - ('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/data/recording/', '*.db'), - ('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/data/recording/', '*.db-shm'), - ('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/data/recording/', '*.db-wal'), + ("/data/", "*.log*"), + ("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/data/recording/", "*.log*"), + ("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/data/recording/", "*.db"), + ("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/data/recording/", "*.db-shm"), + ("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/data/recording/", "*.db-wal"), ] - + all_remote_files = [] - + try: with paramiko.SSHClient() as ssh: ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - ssh.connect(host_ip, username='root', password="", look_for_keys=False, allow_agent=False) + ssh.connect( + host_ip, username="root", password="", look_for_keys=False, allow_agent=False + ) results["ssh_connection"] = "success" - + # Find all files first to set progress bar total for remote_dir, pattern in remote_paths: - cmd = f'ls -1 {remote_dir}{pattern} 2>/dev/null' + cmd = f"ls -1 {remote_dir}{pattern} 2>/dev/null" stdin, stdout, stderr = ssh.exec_command(cmd) files = stdout.read().decode().splitlines() all_remote_files.extend(files) - total_steps = len(endpoints) + len(all_remote_files) + 2 # +1 for results.json, +1 for archive - + total_steps = ( + len(endpoints) + len(all_remote_files) + 2 + ) # +1 for results.json, +1 for archive + with tqdm(total=total_steps, unit="item", desc="State Dump") as pbar: # PHASE: api fetch pbar.set_postfix_str("api fetch") for endpoint in endpoints: filename = f"{endpoint.split('/')[-1]}.json" - url = f'http://{host_ip}:5000{endpoint}' + url = f"http://{host_ip}:5000{endpoint}" try: response = requests.get(url, timeout=10) if response.status_code == 200: @@ -70,7 +70,9 @@ def collect_state_dump(host_ip): json_path.write_text(json.dumps(response.json(), indent=2)) results["api_endpoints"][endpoint] = "success" else: - results["api_endpoints"][endpoint] = f"failed (status code: {response.status_code})" + results["api_endpoints"][endpoint] = ( + f"failed (status code: {response.status_code})" + ) except Exception as e: results["api_endpoints"][endpoint] = f"failed (error: {str(e)})" pbar.update(1) @@ -79,17 +81,17 @@ def collect_state_dump(host_ip): with SCPClient(ssh.get_transport()) as scp: for remote_file in all_remote_files: pbar.set_postfix_str("file transfer") - + # Mirror the remote directory structure # remote_file is like '/data/recording/odc-api.log' # We want it to be 'dump_dir/data/recording/odc-api.log' # Path(remote_file).relative_to('/') gives us 'data/recording/odc-api.log' - rel_remote_path = Path(remote_file).relative_to('/') + rel_remote_path = Path(remote_file).relative_to("/") dest_path = dump_dir / rel_remote_path - + # Ensure the local subdirectory exists dest_path.parent.mkdir(parents=True, exist_ok=True) - + try: scp.get(remote_file, local_path=str(dest_path)) results["files"][remote_file] = "success" @@ -99,27 +101,27 @@ def collect_state_dump(host_ip): # Save results.json pbar.set_postfix_str("saving summary") - results_path = dump_dir / 'results.json' + results_path = dump_dir / "results.json" results_path.write_text(json.dumps(results, indent=2)) pbar.update(1) # PHASE: archive pbar.set_postfix_str("archive") - zip_filename = f'state-dump-{timestamp}.zip' - archive_name = Path(f'state-dump-{timestamp}') + zip_filename = f"state-dump-{timestamp}.zip" + archive_name = Path(f"state-dump-{timestamp}") try: - with zipfile.ZipFile(zip_filename, 'w', zipfile.ZIP_DEFLATED) as zipf: + with zipfile.ZipFile(zip_filename, "w", zipfile.ZIP_DEFLATED) as zipf: # rglob('*') will iterate through all files and directories - for file_path in dump_dir.rglob('*'): + for file_path in dump_dir.rglob("*"): if file_path.is_file(): rel_path = file_path.relative_to(dump_dir) # Write file into the zip with a prefix directory zipf.write(file_path, arcname=str(archive_name / rel_path)) - + results["archive"] = "success" pbar.update(1) pbar.set_postfix_str("complete") - + # Cleanup the temporary directory shutil.rmtree(dump_dir) except Exception as e: diff --git a/util/test_secrets.py b/util/test_secrets.py index 2f939fe..b6a25d6 100644 --- a/util/test_secrets.py +++ b/util/test_secrets.py @@ -5,7 +5,7 @@ import pytest -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'src')) +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "src")) import beeutil.secrets as secrets from beeutil.secrets import DecryptionError @@ -49,25 +49,25 @@ def test_empty_env(): def test_parse_dotenv(): - with tempfile.NamedTemporaryFile(mode='w', suffix='.env', delete=False) as f: - f.write('# comment line\n') - f.write('AWS_KEY=AKIAIOSFODNN7EXAMPLE\n') + with tempfile.NamedTemporaryFile(mode="w", suffix=".env", delete=False) as f: + f.write("# comment line\n") + f.write("AWS_KEY=AKIAIOSFODNN7EXAMPLE\n") f.write('AWS_SECRET="quoted value"\n') f.write("SINGLE_QUOTED='single'\n") - f.write('EMPTY=\n') - f.write('\n') - f.write('NO_EQUALS_LINE\n') - f.write(' SPACED_KEY = spaced_value \n') + f.write("EMPTY=\n") + f.write("\n") + f.write("NO_EQUALS_LINE\n") + f.write(" SPACED_KEY = spaced_value \n") path = f.name try: env = secrets._parse_dotenv(path) - assert env['AWS_KEY'] == 'AKIAIOSFODNN7EXAMPLE' - assert env['AWS_SECRET'] == 'quoted value' - assert env['SINGLE_QUOTED'] == 'single' - assert env['EMPTY'] == '' - assert 'SPACED_KEY' in env - assert 'NO_EQUALS_LINE' not in env + assert env["AWS_KEY"] == "AKIAIOSFODNN7EXAMPLE" + assert env["AWS_SECRET"] == "quoted value" + assert env["SINGLE_QUOTED"] == "single" + assert env["EMPTY"] == "" + assert "SPACED_KEY" in env + assert "NO_EQUALS_LINE" not in env print(f" parsed {len(env)} keys") return True finally: @@ -82,15 +82,15 @@ def test_atomic_rejects_non_string(): secrets.PLUGIN_DIR = tmpdir try: - plugin_dir = os.path.join(tmpdir, 'bad-plugin') + plugin_dir = os.path.join(tmpdir, "bad-plugin") os.makedirs(plugin_dir) - with open(os.path.join(plugin_dir, '.env'), 'w') as f: - f.write('GOOD=value\n') + with open(os.path.join(plugin_dir, ".env"), "w") as f: + f.write("GOOD=value\n") - secrets.load('bad-plugin') - assert os.environ.get('GOOD') == 'value' + secrets.load("bad-plugin") + assert os.environ.get("GOOD") == "value" - del os.environ['GOOD'] + del os.environ["GOOD"] return True finally: secrets.PLUGIN_DIR = orig_dir @@ -105,19 +105,19 @@ def test_get_from_dotenv(): secrets.PLUGIN_DIR = tmpdir try: - plugin_dir = os.path.join(tmpdir, 'test-plugin') + plugin_dir = os.path.join(tmpdir, "test-plugin") os.makedirs(plugin_dir) - with open(os.path.join(plugin_dir, '.env'), 'w') as f: - f.write('MY_KEY=my_value\nMY_SECRET=my_secret\n') + with open(os.path.join(plugin_dir, ".env"), "w") as f: + f.write("MY_KEY=my_value\nMY_SECRET=my_secret\n") - assert secrets.get('test-plugin', 'MY_KEY') == 'my_value' - assert secrets.get('test-plugin', 'MY_SECRET') == 'my_secret' + assert secrets.get("test-plugin", "MY_KEY") == "my_value" + assert secrets.get("test-plugin", "MY_SECRET") == "my_secret" with pytest.raises(KeyError): - secrets.get('test-plugin', 'MISSING') + secrets.get("test-plugin", "MISSING") - del os.environ['MY_KEY'] - del os.environ['MY_SECRET'] + del os.environ["MY_KEY"] + del os.environ["MY_SECRET"] return True finally: secrets.PLUGIN_DIR = orig_dir @@ -132,22 +132,20 @@ def test_load_returns_copy(): secrets.PLUGIN_DIR = tmpdir try: - plugin_dir = os.path.join(tmpdir, 'copy-plugin') + plugin_dir = os.path.join(tmpdir, "copy-plugin") os.makedirs(plugin_dir) - with open(os.path.join(plugin_dir, '.env'), 'w') as f: - f.write('A=1\nB=2\n') + with open(os.path.join(plugin_dir, ".env"), "w") as f: + f.write("A=1\nB=2\n") - result = secrets.load('copy-plugin') - assert result == {'A': '1', 'B': '2'} + result = secrets.load("copy-plugin") + assert result == {"A": "1", "B": "2"} - result['C'] = '3' - assert 'C' not in secrets.load('copy-plugin') + result["C"] = "3" + assert "C" not in secrets.load("copy-plugin") - del os.environ['A'] - del os.environ['B'] + del os.environ["A"] + del os.environ["B"] return True finally: secrets.PLUGIN_DIR = orig_dir secrets.clear_cache() - - diff --git a/util/upload_secrets.py b/util/upload_secrets.py index 3b6585b..3809575 100644 --- a/util/upload_secrets.py +++ b/util/upload_secrets.py @@ -15,6 +15,7 @@ --env-file .env \ --base-url https://hivemapper.com """ + import argparse import json import os @@ -22,29 +23,33 @@ import requests -sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'src')) +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "src")) from beeutil.secrets import _parse_dotenv, encrypt -DEFAULT_BASE_URL = 'https://hivemapper.com' +DEFAULT_BASE_URL = "https://hivemapper.com" def get_plugin_id(base_url: str, plugin_name: str, plugin_secret: str) -> str: - url = f'{base_url}/api/plugins/{plugin_name}?secret={plugin_secret}' + url = f"{base_url}/api/plugins/{plugin_name}?secret={plugin_secret}" resp = requests.get(url, timeout=15) resp.raise_for_status() data = resp.json() - plugin_id = data.get('_id') or str(data.get('id', '')) + plugin_id = data.get("_id") or str(data.get("id", "")) if not plugin_id: - raise ValueError(f"Could not resolve _id for plugin '{plugin_name}' — response: {json.dumps(data)}") + raise ValueError( + f"Could not resolve _id for plugin '{plugin_name}' — response: {json.dumps(data)}" + ) return plugin_id -def upload_encrypted_secrets(base_url: str, plugin_name: str, plugin_secret: str, encrypted_blob: str): - url = f'{base_url}/api/plugins/{plugin_name}/secrets?secret={plugin_secret}' +def upload_encrypted_secrets( + base_url: str, plugin_name: str, plugin_secret: str, encrypted_blob: str +): + url = f"{base_url}/api/plugins/{plugin_name}/secrets?secret={plugin_secret}" resp = requests.put( url, - json={'encrypted_secrets': encrypted_blob}, - headers={'Content-Type': 'application/json'}, + json={"encrypted_secrets": encrypted_blob}, + headers={"Content-Type": "application/json"}, timeout=15, ) resp.raise_for_status() @@ -52,41 +57,57 @@ def upload_encrypted_secrets(base_url: str, plugin_name: str, plugin_secret: str def main(): - parser = argparse.ArgumentParser(description='Encrypt and upload plugin secrets to Hivemapper backend.') - parser.add_argument('--plugin-name', required=True, help='Plugin name as registered in Hivemapper') - parser.add_argument('--plugin-secret', required=True, help='Plugin API key (the "secret" field in MongoDB)') - parser.add_argument('--env-file', required=True, help='Path to .env file containing key=value pairs') - parser.add_argument('--base-url', default=DEFAULT_BASE_URL, help=f'Hivemapper backend URL (default: {DEFAULT_BASE_URL})') - parser.add_argument('--dry-run', action='store_true', help='Encrypt and print blob without uploading') + parser = argparse.ArgumentParser( + description="Encrypt and upload plugin secrets to Hivemapper backend." + ) + parser.add_argument( + "--plugin-name", required=True, help="Plugin name as registered in Hivemapper" + ) + parser.add_argument( + "--plugin-secret", required=True, help='Plugin API key (the "secret" field in MongoDB)' + ) + parser.add_argument( + "--env-file", required=True, help="Path to .env file containing key=value pairs" + ) + parser.add_argument( + "--base-url", + default=DEFAULT_BASE_URL, + help=f"Hivemapper backend URL (default: {DEFAULT_BASE_URL})", + ) + parser.add_argument( + "--dry-run", action="store_true", help="Encrypt and print blob without uploading" + ) args = parser.parse_args() if not os.path.exists(args.env_file): - print(f'Error: {args.env_file} not found', file=sys.stderr) + print(f"Error: {args.env_file} not found", file=sys.stderr) sys.exit(1) env = _parse_dotenv(args.env_file) if not env: - print('Error: .env file is empty or has no valid key=value pairs', file=sys.stderr) + print("Error: .env file is empty or has no valid key=value pairs", file=sys.stderr) sys.exit(1) - print(f'Parsed {len(env)} key(s) from {args.env_file}: {", ".join(env.keys())}') + print(f"Parsed {len(env)} key(s) from {args.env_file}: {', '.join(env.keys())}") print(f'Fetching plugin _id for "{args.plugin_name}"...') plugin_id = get_plugin_id(args.base_url, args.plugin_name, args.plugin_secret) - print(f'Plugin _id: {plugin_id}') + print(f"Plugin _id: {plugin_id}") encrypted_blob = encrypt(plugin_id, env) - print(f'Encrypted blob: {len(encrypted_blob)} chars') + print(f"Encrypted blob: {len(encrypted_blob)} chars") if args.dry_run: - print('\n--- DRY RUN (not uploading) ---') + print("\n--- DRY RUN (not uploading) ---") print(encrypted_blob) return - print(f'Uploading to {args.base_url}/api/plugins/{args.plugin_name}/secrets ...') - result = upload_encrypted_secrets(args.base_url, args.plugin_name, args.plugin_secret, encrypted_blob) - print(f'Done: {json.dumps(result)}') + print(f"Uploading to {args.base_url}/api/plugins/{args.plugin_name}/secrets ...") + result = upload_encrypted_secrets( + args.base_url, args.plugin_name, args.plugin_secret, encrypted_blob + ) + print(f"Done: {json.dumps(result)}") -if __name__ == '__main__': +if __name__ == "__main__": main()