Skip to content

Commit d0a1905

Browse files
authored
fix(registry): publish both Starlink plugins in the GitHub catalog (#4)
* chore(deps): remove unused lock entries Signed-off-by: Michael Freeman <mfreeman@carverauto.dev> * fix(registry): discover Starlink plugins from GitHub Signed-off-by: Michael Freeman <mfreeman@carverauto.dev> * deploy(prod): pin the Starlink catalog fix Signed-off-by: Michael Freeman <mfreeman@carverauto.dev> * fix(k8s): normalize the sealed GitHub token Signed-off-by: Michael Freeman <mfreeman@carverauto.dev> --------- Signed-off-by: Michael Freeman <mfreeman@carverauto.dev>
1 parent e565d72 commit d0a1905

8 files changed

Lines changed: 98 additions & 11 deletions

File tree

‎k8s/README.md‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,8 @@ The application expects two credential sources at deploy time:
2222
Required keys for `developer-portal-secrets`:
2323

2424
- `secret_key_base`
25-
- `github_token` (optional; raises GitHub API rate limits for the plugin registry)
25+
- `github_token` (authenticated GitHub access for regular catalog refreshes;
26+
anonymous API limits are too low for the full plugin and add-on scan)
2627

2728
A non-applied example is provided in [secret-template.yaml](/home/mfreeman/src/community/k8s/base/secret-template.yaml).
2829
A non-applied DB bootstrap example is provided in [db-credentials-template.yaml](/home/mfreeman/src/community/k8s/base/db-credentials-template.yaml).
@@ -50,5 +51,5 @@ Required non-secret runtime config comes from `developer-portal-config`:
5051
- `PGHOST`
5152
- `PGPORT`
5253
- `PGDATABASE`
53-
- `FORGEJO_API_BASE_URL` / `FORGEJO_CONTENT_BASE_URL` (in-cluster fetch)
54-
- `GITHUB_API_BASE_URL` (optional override; user-facing plugin links are rewritten to GitHub either way)
54+
- `GITHUB_API_BASE_URL` (`https://api.github.com`; manifests and release indexes
55+
are fetched from GitHub)

‎k8s/base/configmap.yaml‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,5 +10,4 @@ data:
1010
PGHOST: developer-portal-pg-rw.serviceradar-developer.svc.cluster.local
1111
PGPORT: "5432"
1212
PGDATABASE: developer_portal
13-
FORGEJO_API_BASE_URL: http://forgejo-http.forgejo.svc.cluster.local:3000/api/v1
14-
FORGEJO_CONTENT_BASE_URL: http://forgejo-http.forgejo.svc.cluster.local:3000
13+
GITHUB_API_BASE_URL: https://api.github.com

‎k8s/prod/kustomization.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,4 +28,4 @@ replicas:
2828

2929
images:
3030
- name: registry.carverauto.dev/serviceradar/developer-portal
31-
newTag: sha-ee09e056075f
31+
newTag: sha-b4dbcf86c5a8

‎k8s/prod/secrets.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ metadata:
55
namespace: serviceradar-developer
66
spec:
77
encryptedData:
8+
github_token: AgDI7naMtX/sTNuZHkrxPdwa9IaM38EDO0FDztmgFyGnGC+350HpvhebVs8/64Fb69OrNscS0fMnoiyTEJAoQlR0/Bc3C25IceCUWmaeeX4jsLjWwR75z6ylry8pafP5tPugETmuX8Qcah8pgTgH4UyuRflxgReyYa0yLpO6J7d8ysboSmthpPbSpYfXelfNYoAavGqI/2eD+wGiopxTVwRzetT5fa6E0O/NAFRfqV0AGrLNizaKarz3zBMal05UcK6YcOkc7/0ztbRodChjsUFBcioT+cdmT36LSdRI0dIy6aqhMEYn6vBOvPtul7Bnttd92NJLVa4pxsizLfcr8Z9EOy+/I6nhAABaV+wfRPPlf5sj9u3e//z4hm5P0Jh1eF5zqdoQd2rq5aPf4I39a2hOseQhqgM/3/5lnAeWBAG5bsvLfPmdnHUp2q7h3QWpL16VFGfGev7lRFdhVVfeObKX5aj015VaSCz/ZBiz6pX4NH/vgHH0Yveq3ewoeE7VltPiaI7nfYY1mzEggn/2uQ4aDbYxGmAxFGWfWCf9Fu0cqtFsBZ8ZMQf/mM1k8MgmL85z6igCVKaZYZENcybiIbeP0O134537kkaeHM+IvOoYCEhaTEw2mLutmqGJA3a3GjWDYEV+iJZbpv4nAUsQVt7nAMIbtQGvek9IGSb6Ox2I/79N4LLdVvi03uYkDRL2RoGQwHfnrlbI9V4agee6KEXR7RWlaH3zUMYjFBApyzCg4wVoM8uSerbO
89
secret_key_base: AgDGLBofWsa8VYBdeUuwzLi0zuXBjwbclxrCKhuptNqj812ahc/wP87b/toKS3TME8dfLIAaYu/p8OK5PU+697wmnsq0XZgtctMbApW1wnF5Z7sGVFfvQd5Iy4hedUnKPi/1lQh4sQ+iknD8eppGWeAOODgJYRAw74zM8x7r458u25KanSxLIl3deA6FmcZk9scZQyn9CAm3uetKQ7tRw4TY0m9CoDSepFgGHrX1rwur2W5nnmn2VznegBgyLIpIBgqEs48AQltNGbveuyDTD9GWtLJrOpKMGhpCg4/pkqn2gutiwXGl8HEEk9waP5vakb71u/0lDuHm9qJnUiOuKlM33lwmySUvbuU5aJ/UdRa3k1kiEU3uJExX8e2QaA8TzSnkytpkI7kYrvN+xovGG+XZQPpU1KXzGnmTsiDh1yf2qOfPe80MoWBE2MvNs693xwnlWJ/VmAKDmgkfqEYFGEafkbCFSR1+Leyfu2OrHXzYjmVfMQzxHeT76NoDSdwelGxP5MDQihNQGD2wmGjk9pXzXJg6V+gobhlNyxj6LhsN2fR7ADGMiH5aHZRVvygsDVIYTB+LeDa+BpNytX2MGYIiZkAYP42hR1W9KBnS6aOSV3wVui27RIE4XFWgknFbKsk9rzWVh08mzEGgGiaR+VvyIU7hVL8bjuS/MlHrhJJwVpDJIHRty+uV3mPqh+Txy8bcUa0l3J8Dm+wrWcBCy0sX1RI69lGFrGXIcRbhs582/ruFF3K3YXvEZI5MO0iJnkMQCi+YYWnQH7ECQ/kOBJCg9RfSNoa3T2zt1chJ1jBAf2tYhObqUEV+iOgVJflqYwHoQdDex9tuve75lEC+cKlSwiJ547jawX/FTQaSKPFgAQ==
910
template:
1011
metadata:

‎k8s/staging/secrets.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ metadata:
55
namespace: serviceradar-developer-staging
66
spec:
77
encryptedData:
8+
github_token: AgCCNjSDW1pAjjocQsLbVBvRpm2jc1Jlj9q9BVlIriNhs3QGtwUIlIUTvNAA3Gjq5hUnIcM0F52Y2QWXu94r0191U7Qr7WgC/+BAzIZoEEXYPg07Jh9YlMoFVIxvN82K/P276nGMY+DQOsqF7Y6IhpUAjH6iztuREg/oVp/eBM1BImgP2CDF0HWhvvvmwow3mBqw5haOXKkDob0AOILCKOR1m3u6O6UHB7CdMAtVRHabcg6/6N7nKZljgJCDjgBhUvi5FjBXvAQ4C6JNo3rrzqt0FnDi9smZ6Q2CcQpJRYgmgbQuxZln8vhDwXPJhGNZT1KygCSXXrjvrbQqnaychR2fhU55qEIgAmCnDmyr4de6nVf0A7IHlIdcN6B3O2DOHCA/83dq78U1IvJQ6r87bgv1JBo61k8txEo6srOZ8LtPIRlYkFd523wbVLLt++ZsbhXzp4N+DuDTLwQYkJj3N8+bIWSPi0uDZvY4xmcSzB1EZIU2Qb3ID+DA4rfDJzVrCR8HwX4WEaipayuzdIGkH0TE1ZW3ZjX1IUg/pJYL5mdIQHLptHv14wGFtCtULQnNDqxUH2jzDoyOeCk5OEE9G8l0GxQpp/otmKjTpGX7E4nIvQOHvVIFA0QWibQ5SCMzdgh25CchI9mFukw56T6tPZb3HuxM+VZ5+lsGMI9jTbCWQYAqiAFdJM39XWT0Y/8bmL0SLVIdZVWSvdAiDARtxNSgAy6tnJA2ielmLLmCv7geTrFcVhIPFIGQ
89
secret_key_base: AgBKdKBisc1+dFyZeqSzPzkHPTUs36AtOBSj6bp0W2h0vDl6TxChoZfa3aLH04Ma6V2WIQcpm0DQOdcCgQbq7llcjmZHvmnCeOCcyiTjhVI6xQ95EVRYQD81Yc5bCvXJQ/suAiv7yw+gff1h4VVgJUVpCSASqTKfiKch8IM4nuKNrrEagczqCq+BPGWtN3W/kF0T4q8kdDVliTDc7Ute8liUtRgkVIfch45mxpAKxBSoCdULWi5T2l5dG09msr8M8bMzF3FEMLuTVlQ0/pwge+fc2h8hesn04sWWzuZZVhsfSUyCw7YCoYOga1cJT3SSf6d00YfeN9pFFqJ2jf4mpkdXgVs4/Zr/m9VMs1BEbAHB2GjqSd4ui4VpU+TjhfAC501FQwGmKoe82krpRTM7ZGk1u7BhA5LLgdmbIuG+urRp8lClQo2tM7WW5U16miKYMBiOZmBYBuAFz8Kbodr9o9Vr1xh//XjQi9M1SjfV61UTI2C8JzUFv39pbG2soYZLJDkJ3UafMboo6a+OguUz6jCfQV8RXy+tKQtSxENOD2/xMIHXi/2lq7Q5kq20U3L5k67hDLHhmub17zY1Miwy90ItwZmX8IcJzQWKPxLxs9T4dRaimn3p1Kwu7BIT/Cne7oPtDHcZy90fvozDtBAeloxYwmwRvNt7ynZmdKcTiZAM7fLr35gPD1tTlrSLLwSPHRw6eunkA2rBiPaoJYLNNRA/pFT9bkd2G1pfjrtJHniaKfo+fEJlQjGZ4jTaCWU0ONK2dud9DtJuniStBz54UilgIGAwJOFXe/3W1UuyWZWeFEOI4MLmSOdbEbZh1upl1X3A/Y0Q0z7NV/EElZR/RFJLm8sa8Hzpp5e6piEJlMEJnw==
910
template:
1011
metadata:

‎lib/developer_portal/registry/forgejo_source.ex‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -13,10 +13,12 @@ defmodule DeveloperPortal.Registry.ForgejoSource do
1313

1414
@manifest_files [
1515
%{name: "plugin.yaml", kind: "check"},
16+
%{name: "plugin.local.yaml", kind: "check", schema_variant: "local"},
1617
%{name: "plugin.console.yaml", kind: "console"},
1718
%{name: "plugin.inventory_sync.yaml", kind: "inventory_sync"},
1819
%{name: "plugin.stream.yaml", kind: "stream"},
1920
%{name: "manifest.json", kind: "check"},
21+
%{name: "manifest.local.json", kind: "check", schema_variant: "local"},
2022
%{name: "manifest.console.json", kind: "console"},
2123
%{name: "manifest.inventory_sync.json", kind: "inventory_sync"},
2224
%{name: "manifest.stream.json", kind: "stream"}
@@ -64,7 +66,7 @@ defmodule DeveloperPortal.Registry.ForgejoSource do
6466

6567
manifest_entry ->
6668
[
67-
build_plugin(config, manifest_entry, manifest.kind, language, index, %{
69+
build_plugin(config, manifest_entry, manifest, language, index, %{
6870
source_url: source_url,
6971
readme_url: readme_url,
7072
directory: directory,
@@ -83,7 +85,7 @@ defmodule DeveloperPortal.Registry.ForgejoSource do
8385
[]
8486
end
8587

86-
defp build_plugin(config, manifest_entry, kind, language, index, urls) do
88+
defp build_plugin(config, manifest_entry, %{kind: kind} = variant, language, index, urls) do
8789
manifest = fetch_manifest!(config, manifest_entry)
8890
slug = fetch_id!(manifest)
8991
release = Map.get(index, slug)
@@ -94,6 +96,7 @@ defmodule DeveloperPortal.Registry.ForgejoSource do
9496
# can never crash the whole refresh.
9597
published = if signed, do: release, else: nil
9698
%{source_url: source_url, readme_url: readme_url, directory: directory, dist: dist} = urls
99+
schema_variant = Map.get(variant, :schema_variant, kind)
97100

98101
%Plugin{
99102
slug: slug,
@@ -110,7 +113,7 @@ defmodule DeveloperPortal.Registry.ForgejoSource do
110113
source_url: PublicURL.githubize(source_url),
111114
readme_url: PublicURL.githubize(readme_url),
112115
manifest_url: PublicURL.githubize(download_url(manifest_entry)),
113-
config_schema_url: PublicURL.githubize(config_schema_url(kind, directory, dist)),
116+
config_schema_url: PublicURL.githubize(config_schema_url(schema_variant, directory, dist)),
114117
wasm_url: PublicURL.githubize(find_download_url(dist, @wasm_candidates)),
115118
artifact_url: PublicURL.githubize(find_download_url(dist, @checksum_candidates)),
116119
signature_url: PublicURL.githubize(find_download_url(dist, @signature_candidates)),
@@ -338,6 +341,7 @@ defmodule DeveloperPortal.Registry.ForgejoSource do
338341
end
339342

340343
defp schema_candidates("stream"), do: ["config.stream.schema.json"]
344+
defp schema_candidates("local"), do: ["config.local.schema.json"]
341345
defp schema_candidates("console"), do: ["config.console.schema.json", "config.schema.json"]
342346

343347
defp schema_candidates("inventory_sync"),

‎mix.lock‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,6 @@
3232
"iterex": {:hex, :iterex, "0.1.2", "58f9b9b9a22a55cbfc7b5234a9c9c63eaac26d276b3db80936c0e1c60355a5a6", [:mix], [], "hexpm", "2e103b8bcc81757a9af121f6dc0df312c9a17220f302b1193ef720460d03029d"},
3333
"jason": {:hex, :jason, "1.4.5", "2e3a008590b0b8d7388c20293e9dcc9cf3e5d642fd2a114e4cbbb52e595d940a", [:mix], [{:decimal, "~> 1.0 or ~> 2.0 or ~> 3.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "b0c823996102bcd0239b3c2444eb00409b72f6a140c1950bc8b457d836b30684"},
3434
"lazy_html": {:hex, :lazy_html, "0.1.12", "31a55ee622918fce988c94b06232227b42daa64e4eab14ac32081d0f3fd8db6f", [:make, :mix], [{:cc_precompiler, "~> 0.1", [hex: :cc_precompiler, repo: "hexpm", optional: false]}, {:elixir_make, "~> 0.9", [hex: :elixir_make, repo: "hexpm", optional: false]}, {:fine, "~> 0.1.0", [hex: :fine, repo: "hexpm", optional: false]}], "hexpm", "8a0da594776caee58782c6f93b2abaa5bdb809daf8d43351a561f7de9dc2e2a8"},
35-
"libgraph": {:hex, :libgraph, "0.16.0", "3936f3eca6ef826e08880230f806bfea13193e49bf153f93edcf0239d4fd1d07", [:mix], [], "hexpm", "41ca92240e8a4138c30a7e06466acc709b0cbb795c643e9e17174a178982d6bf"},
3635
"makeup": {:hex, :makeup, "1.2.1", "e90ac1c65589ef354378def3ba19d401e739ee7ee06fb47f94c687016e3713d1", [:mix], [{:nimble_parsec, "~> 1.4", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "d36484867b0bae0fea568d10131197a4c2e47056a6fbe84922bf6ba71c8d17ce"},
3736
"makeup_elixir": {:hex, :makeup_elixir, "1.0.1", "e928a4f984e795e41e3abd27bfc09f51db16ab8ba1aebdba2b3a575437efafc2", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.2.3 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "7284900d412a3e5cfd97fdaed4f5ed389b8f2b4cb49efc0eb3bd10e2febf9507"},
3837
"makeup_json": {:hex, :makeup_json, "1.0.0", "03886f244d9db4179a9ecfffde5e711480e04e16ef91ca484d2b51ca3251a18f", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.1", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "5c8c559e658c7f7e91b96c4b8c40f5912ea0adff44b7afe73e4639d9c3f53b94"},
@@ -72,7 +71,6 @@
7271
"telemetry_metrics": {:hex, :telemetry_metrics, "1.1.0", "5bd5f3b5637e0abea0426b947e3ce5dd304f8b3bc6617039e2b5a008adc02f8f", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "e7b79e8ddfde70adb6db8a6623d1778ec66401f366e9a8f5dd0955c56bc8ce67"},
7372
"telemetry_poller": {:hex, :telemetry_poller, "1.3.0", "d5c46420126b5ac2d72bc6580fb4f537d35e851cc0f8dbd571acf6d6e10f5ec7", [:rebar3], [{:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "51f18bed7128544a50f75897db9974436ea9bfba560420b646af27a9a9b35211"},
7473
"thousand_island": {:hex, :thousand_island, "1.5.0", "f50a213cac97262b6d5ebb85745aa2c00fec1413191e6e66834788d45425cecb", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "708923d40523e43cf99041ab37a0d4b0ec426ac6438fa3716ab23d919eaeb412"},
75-
"unicode_util_compat": {:hex, :unicode_util_compat, "0.7.1", "a48703a25c170eedadca83b11e88985af08d35f37c6f664d6dcfb106a97782fc", [:rebar3], [], "hexpm", "b3a917854ce3ae233619744ad1e0102e05673136776fb2fa76234f3e03b23642"},
7674
"websock": {:hex, :websock, "0.5.3", "2f69a6ebe810328555b6fe5c831a851f485e303a7c8ce6c5f675abeb20ebdadc", [:mix], [], "hexpm", "6105453d7fac22c712ad66fab1d45abdf049868f253cf719b625151460b8b453"},
7775
"websock_adapter": {:hex, :websock_adapter, "0.6.0", "73db5ab8aaefd1a876a97ce3e6afc96562625de69ef17a4e04426e034849d0b8", [:mix], [{:bandit, ">= 0.6.0", [hex: :bandit, repo: "hexpm", optional: true]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.6", [hex: :plug_cowboy, repo: "hexpm", optional: true]}, {:websock, "~> 0.5", [hex: :websock, repo: "hexpm", optional: false]}], "hexpm", "50021a85bce8f203b086705d9e0c5415e2c7eb05d319111b0428fe71f9934617"},
7876
"yamerl": {:hex, :yamerl, "0.10.0", "4ff81fee2f1f6a46f1700c0d880b24d193ddb74bd14ef42cb0bcf46e81ef2f8e", [:rebar3], [], "hexpm", "346adb2963f1051dc837a2364e4acf6eb7d80097c0f53cbdc3046ec8ec4b4e6e"},
Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
defmodule DeveloperPortal.Registry.ForgejoSourceTest do
2+
use ExUnit.Case, async: true
3+
4+
alias DeveloperPortal.Registry.ForgejoSource
5+
6+
for {cloud_file, local_file} <- [
7+
{"plugin.yaml", "plugin.local.yaml"},
8+
{"manifest.json", "manifest.local.json"}
9+
] do
10+
test "discovers #{local_file} alongside #{cloud_file} with its own configuration schema" do
11+
cloud_file = unquote(cloud_file)
12+
local_file = unquote(local_file)
13+
14+
Req.Test.stub(__MODULE__, fn conn ->
15+
case conn.request_path do
16+
"/repos/example/catalog/contents/plugins" ->
17+
Req.Test.json(conn, [entry("weather", "dir")])
18+
19+
"/repos/example/catalog/contents/plugins/weather" ->
20+
files = [cloud_file, local_file, "config.schema.json", "config.local.schema.json"]
21+
Req.Test.json(conn, Enum.map(files, &entry("weather/" <> &1, "file")))
22+
23+
"/repos/example/catalog/releases" ->
24+
Req.Test.json(conn, [])
25+
26+
"/raw/plugins/weather/" <> file ->
27+
id = if file == cloud_file, do: "weather-cloud", else: "weather-local"
28+
29+
conn
30+
|> Plug.Conn.put_resp_content_type("text/plain")
31+
|> Plug.Conn.send_resp(200, manifest(file, id))
32+
end
33+
end)
34+
35+
assert {:ok, plugins} =
36+
ForgejoSource.fetch_plugins(
37+
api_base_url: "https://catalog.example.test",
38+
content_base_url: nil,
39+
owner: "example",
40+
repo: "catalog",
41+
ref: "main",
42+
plugin_root: "plugins",
43+
req_options: [plug: {Req.Test, __MODULE__}, retry: false]
44+
)
45+
46+
assert [%{slug: "weather-cloud"} = cloud, %{slug: "weather-local"} = local] = plugins
47+
assert cloud.sample_kind == "check"
48+
assert local.sample_kind == "check"
49+
assert String.ends_with?(cloud.manifest_url, "/" <> cloud_file)
50+
assert String.ends_with?(local.manifest_url, "/" <> local_file)
51+
assert cloud.config_schema_url == raw_url("weather/config.schema.json")
52+
assert local.config_schema_url == raw_url("weather/config.local.schema.json")
53+
end
54+
end
55+
56+
defp entry(path, type) do
57+
%{
58+
"name" => Path.basename(path),
59+
"path" => "plugins/" <> path,
60+
"type" => type,
61+
"html_url" => "https://catalog.example.test/source/plugins/" <> path,
62+
"download_url" => raw_url(path)
63+
}
64+
end
65+
66+
defp raw_url(path), do: "https://catalog.example.test/raw/plugins/" <> path
67+
68+
defp manifest(file, id) do
69+
attrs = %{
70+
"id" => id,
71+
"name" => id,
72+
"version" => "0.0.1",
73+
"description" => "Synthetic weather monitoring plugin",
74+
"outputs" => "plugin_result"
75+
}
76+
77+
if Path.extname(file) == ".json" do
78+
Jason.encode!(attrs)
79+
else
80+
Enum.map_join(attrs, "\n", fn {key, value} -> "#{key}: #{value}" end)
81+
end
82+
end
83+
end

0 commit comments

Comments
 (0)