added state-driven jsons

.gitea/workflows/dry-run.yml

@@ -3,36 +3,79 @@ name: Dry Run
 on:
   pull_request:
     paths:
-      - "netbird.json"
+      - "state/*.json"
 
 jobs:
-  dry-run:
+  detect:
     runs-on: ubuntu-latest
+    outputs:
+      envs: ${{ steps.changed.outputs.envs }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Detect changed environments
+        id: changed
+        run: |
+          FILES=$(git diff --name-only ${{ github.event.pull_request.base.sha }} ${{ github.sha }} -- 'state/*.json')
+          ENVS="[]"
+          for f in $FILES; do
+            ENV=$(basename "$f" .json)
+            ENVS=$(echo "$ENVS" | jq -c ". + [\"$ENV\"]")
+          done
+          echo "envs=$ENVS" >> "$GITHUB_OUTPUT"
+          echo "Changed environments: $ENVS"
+
+  dry-run:
+    needs: detect
+    runs-on: ubuntu-latest
+    if: needs.detect.outputs.envs != '[]'
+    strategy:
+      matrix:
+        env: ${{ fromJson(needs.detect.outputs.envs) }}
     steps:
       - uses: actions/checkout@v4
 
+      - name: Resolve environment secrets
+        id: env
+        run: |
+          ENV_UPPER=$(echo "${{ matrix.env }}" | tr '[:lower:]-' '[:upper:]_')
+          echo "token_key=${ENV_UPPER}_RECONCILER_TOKEN" >> "$GITHUB_OUTPUT"
+          echo "url_key=${ENV_UPPER}_RECONCILER_URL" >> "$GITHUB_OUTPUT"
+
       - name: Run dry-run reconcile
         id: plan
+        env:
+          RECONCILER_TOKEN: ${{ secrets[steps.env.outputs.token_key] }}
+          RECONCILER_URL: ${{ secrets[steps.env.outputs.url_key] }}
         run: |
+          if [ -z "$RECONCILER_URL" ] || [ -z "$RECONCILER_TOKEN" ]; then
+            echo "No secrets configured for environment '${{ matrix.env }}' — skipping"
+            echo "response={}" >> "$GITHUB_OUTPUT"
+            exit 0
+          fi
           RESPONSE=$(curl -sf \
             -X POST \
-            -H "Authorization: Bearer ${{ secrets.RECONCILER_TOKEN }}" \
+            -H "Authorization: Bearer ${RECONCILER_TOKEN}" \
             -H "Content-Type: application/json" \
-            -d @netbird.json \
+            -d @state/${{ matrix.env }}.json \
-            "${{ secrets.RECONCILER_URL }}/reconcile?dry_run=true")
+            "${RECONCILER_URL}/reconcile?dry_run=true")
           echo "response<<EOF" >> "$GITHUB_OUTPUT"
           echo "$RESPONSE" >> "$GITHUB_OUTPUT"
           echo "EOF" >> "$GITHUB_OUTPUT"
 
       - name: Format plan as markdown
         id: format
+        if: steps.plan.outputs.response != '{}'
         run: |
           cat <<'SCRIPT' > format.py
           import json, sys
           data = json.loads(sys.stdin.read())
           ops = data.get("operations", [])
           summary = data.get("summary", {})
-          lines = ["## NetBird Reconciliation Plan\n"]
+          env = sys.argv[1]
+          lines = [f"## Reconciliation Plan: `{env}`\n"]
           if not ops:
               lines.append("No changes detected.\n")
           else:
@@ -45,12 +88,13 @@ jobs:
               lines.append(f"**Summary:** {s.get('created',0)} create, {s.get('updated',0)} update, {s.get('deleted',0)} delete")
           print("\n".join(lines))
           SCRIPT
-          COMMENT=$(echo '${{ steps.plan.outputs.response }}' | python3 format.py)
+          COMMENT=$(echo '${{ steps.plan.outputs.response }}' | python3 format.py "${{ matrix.env }}")
           echo "comment<<EOF" >> "$GITHUB_OUTPUT"
           echo "$COMMENT" >> "$GITHUB_OUTPUT"
           echo "EOF" >> "$GITHUB_OUTPUT"
 
       - name: Post PR comment
+        if: steps.plan.outputs.response != '{}'
         env:
           GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
         run: |

.gitea/workflows/reconcile.yml

@@ -5,53 +5,99 @@ on:
     branches:
       - main
     paths:
-      - "netbird.json"
+      - "state/*.json"
 
 jobs:
-  reconcile:
+  detect:
     runs-on: ubuntu-latest
+    outputs:
+      envs: ${{ steps.changed.outputs.envs }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 2
+
+      - name: Detect changed environments
+        id: changed
+        run: |
+          FILES=$(git diff --name-only HEAD~1 HEAD -- 'state/*.json')
+          ENVS="[]"
+          for f in $FILES; do
+            ENV=$(basename "$f" .json)
+            ENVS=$(echo "$ENVS" | jq -c ". + [\"$ENV\"]")
+          done
+          echo "envs=$ENVS" >> "$GITHUB_OUTPUT"
+          echo "Changed environments: $ENVS"
+
+  reconcile:
+    needs: detect
+    runs-on: ubuntu-latest
+    if: needs.detect.outputs.envs != '[]'
+    strategy:
+      matrix:
+        env: ${{ fromJson(needs.detect.outputs.envs) }}
     steps:
       - uses: actions/checkout@v4
 
-      - name: Sync events
+      - name: Resolve environment secrets
+        id: env
         run: |
+          ENV_UPPER=$(echo "${{ matrix.env }}" | tr '[:lower:]-' '[:upper:]_')
+          echo "token_key=${ENV_UPPER}_RECONCILER_TOKEN" >> "$GITHUB_OUTPUT"
+          echo "url_key=${ENV_UPPER}_RECONCILER_URL" >> "$GITHUB_OUTPUT"
+          echo "age_key=${ENV_UPPER}_AGE_PUBLIC_KEY" >> "$GITHUB_OUTPUT"
+
+      - name: Sync events
+        env:
+          RECONCILER_TOKEN: ${{ secrets[steps.env.outputs.token_key] }}
+          RECONCILER_URL: ${{ secrets[steps.env.outputs.url_key] }}
+        run: |
+          if [ -z "$RECONCILER_URL" ] || [ -z "$RECONCILER_TOKEN" ]; then
+            echo "No secrets configured for environment '${{ matrix.env }}' — skipping"
+            exit 0
+          fi
           curl -sf \
             -X POST \
-            -H "Authorization: Bearer ${{ secrets.RECONCILER_TOKEN }}" \
+            -H "Authorization: Bearer ${RECONCILER_TOKEN}" \
-            "${{ secrets.RECONCILER_URL }}/sync-events"
+            "${RECONCILER_URL}/sync-events"
 
       - name: Pull latest (poller may have committed)
         run: git pull --rebase
 
       - name: Apply reconcile
         id: reconcile
+        env:
+          RECONCILER_TOKEN: ${{ secrets[steps.env.outputs.token_key] }}
+          RECONCILER_URL: ${{ secrets[steps.env.outputs.url_key] }}
         run: |
           RESPONSE=$(curl -sf \
             -X POST \
-            -H "Authorization: Bearer ${{ secrets.RECONCILER_TOKEN }}" \
+            -H "Authorization: Bearer ${RECONCILER_TOKEN}" \
             -H "Content-Type: application/json" \
-            -d @netbird.json \
+            -d @state/${{ matrix.env }}.json \
-            "${{ secrets.RECONCILER_URL }}/reconcile")
+            "${RECONCILER_URL}/reconcile")
           echo "response<<EOF" >> "$GITHUB_OUTPUT"
           echo "$RESPONSE" >> "$GITHUB_OUTPUT"
           echo "EOF" >> "$GITHUB_OUTPUT"
 
           STATUS=$(echo "$RESPONSE" | jq -r '.status')
           if [ "$STATUS" = "error" ]; then
-            echo "Reconcile failed"
+            echo "Reconcile failed for ${{ matrix.env }}"
             echo "$RESPONSE" | jq .
             exit 1
           fi
 
       - name: Encrypt and upload setup keys
         if: success()
+        env:
+          AGE_PUBLIC_KEY: ${{ secrets[steps.env.outputs.age_key] }}
         run: |
           KEYS=$(echo '${{ steps.reconcile.outputs.response }}' | jq -r '.created_keys // empty')
-          if [ -n "$KEYS" ] && [ "$KEYS" != "{}" ] && [ "$KEYS" != "null" ]; then
+          if [ -n "$KEYS" ] && [ "$KEYS" != "{}" ] && [ "$KEYS" != "null" ] && [ -n "$AGE_PUBLIC_KEY" ]; then
-            echo "$KEYS" | age -r "${{ secrets.AGE_PUBLIC_KEY }}" -o setup-keys.age
+            echo "$KEYS" | age -r "$AGE_PUBLIC_KEY" -o setup-keys-${{ matrix.env }}.age
-            echo "Setup keys encrypted to setup-keys.age"
+            echo "Setup keys for ${{ matrix.env }} encrypted"
           else
-            echo "No new keys created"
+            echo "No new keys created for ${{ matrix.env }}"
             exit 0
           fi
 
@@ -59,6 +105,6 @@ jobs:
         if: success()
         uses: actions/upload-artifact@v4
         with:
-          name: setup-keys
+          name: setup-keys-${{ matrix.env }}
-          path: setup-keys.age
+          path: setup-keys-${{ matrix.env }}.age
           if-no-files-found: ignore

poc/README.md

@@ -5,13 +5,13 @@ testing of the NetBird GitOps reconciler.
 
 ## Stack overview
 
-| Component       | Purpose                                     |
+| Component    | Purpose                                     |
-| --------------- | ------------------------------------------- |
+| ------------ | ------------------------------------------- |
-| Caddy           | TLS termination, reverse proxy              |
+| Caddy        | TLS termination, reverse proxy              |
-| NetBird         | Management, Signal, Relay, Dashboard, TURN  |
+| NetBird      | Management, Signal, Relay, Dashboard, TURN  |
-| Reconciler      | Declarative config → NetBird API reconciler |
+| Reconciler   | Declarative config → NetBird API reconciler |
-| Gitea           | Git server for GitOps source-of-truth       |
+| Gitea        | Git server for GitOps source-of-truth       |
-| Gitea Runner    | Executes CI workflows (Actions)             |
+| Gitea Runner | Executes CI workflows (Actions)             |
 
 All services run as Docker containers on a single VPS, connected via a `netbird`
 Docker bridge network. Caddy handles ACME certificates automatically.
@@ -69,7 +69,8 @@ idempotent.
 1. Open `https://gitea.vps-a.networkmonitor.cc` and complete the install wizard
 2. Create an admin account (user: `blastpilot`)
 3. Create org `BlastPilot` and repo `netbird-gitops`
-4. Generate a Gitea API token (**Settings → Applications**) → `vault_gitea_token`
+4. Generate a Gitea API token (**Settings → Applications**) →
+   `vault_gitea_token`
 5. Go to **Site Administration → Actions → Runners** → copy runner registration
    token → `vault_gitea_admin_password` and `vault_gitea_runner_token`
 
@@ -80,6 +81,7 @@ ansible-playbook -i inventory.yml playbook.yml
 ```
 
 This run will:
+
 - Start the reconciler with a valid NetBird API token
 - Register and start the Gitea Actions runner
 - Wire the reconciler to poll Gitea for `netbird.json` changes
@@ -93,6 +95,7 @@ git push poc main
 ```
 
 Then configure Gitea repo secrets (Settings → Actions → Secrets):
+
 - `RECONCILER_TOKEN` — the reconciler bearer token
 - `RECONCILER_URL` — `https://vps-a.networkmonitor.cc/reconciler`
 - `GITEA_TOKEN` — same Gitea API token

poc/ansible/templates/reconciler.env.j2

@@ -5,6 +5,7 @@ GITEA_ENABLED={{ gitea_enabled }}
 GITEA_URL=http://gitea:{{ gitea_http_port }}
 GITEA_TOKEN={{ vault_gitea_token }}
 GITEA_REPO={{ gitea_org_name }}/{{ gitea_repo_name }}
+STATE_FILE=state/test.json
 POLL_INTERVAL_SECONDS=30
 PORT={{ reconciler_port }}
 DATA_DIR=/data

src/config.ts

@@ -11,6 +11,7 @@ const BaseSchema = z.object({
   pollIntervalSeconds: z.coerce.number().int().positive().default(30),
   port: z.coerce.number().int().positive().default(8080),
   dataDir: z.string().default("/data"),
+  stateFile: z.string().default("netbird.json"),
 });
 
 const GiteaFieldsSchema = z.object({
@@ -67,5 +68,6 @@ export function loadConfig(): Config {
     pollIntervalSeconds: Deno.env.get("POLL_INTERVAL_SECONDS"),
     port: Deno.env.get("PORT"),
     dataDir: Deno.env.get("DATA_DIR"),
+    stateFile: Deno.env.get("STATE_FILE"),
   });
 }

src/integration.test.ts

@@ -100,6 +100,7 @@ const MOCK_CONFIG: Config = {
   pollIntervalSeconds: 30,
   port: 8080,
   dataDir: "/data",
+  stateFile: "netbird.json",
 };
 
 /** Desired state with one group and one setup key referencing it. */
@@ -156,6 +157,7 @@ const STANDALONE_CONFIG: Config = {
   pollIntervalSeconds: 30,
   port: 8080,
   dataDir: "/data",
+  stateFile: "netbird.json",
 };
 
 function buildStandaloneHandler(calls: ApiCall[]) {

src/poller/loop.ts

@@ -63,7 +63,7 @@ async function pollOnceGitea(
   const pollerState = await loadPollerState(config.dataDir);
 
   // Fetch current desired state from Gitea (main branch)
-  const file = await gitea.getFileContent("netbird.json", "main");
+  const file = await gitea.getFileContent(config.stateFile, "main");
   const desired: DesiredState = DesiredStateSchema.parse(
     JSON.parse(file.content),
   );
@@ -255,7 +255,7 @@ async function processEnrollment(
   fileSha: string,
   onCommit: (newSha: string, newDesired: DesiredState) => void,
 ): Promise<void> {
-  const { netbird } = ctx;
+  const { config, netbird } = ctx;
   const { setupKeyName, peerId, peerHostname } = enrollment;
 
   // Rename the peer to match the setup key name
@@ -283,7 +283,7 @@ async function processEnrollment(
 
   try {
     await gitea.updateFile(
-      "netbird.json",
+      config.stateFile,
       content,
       fileSha,
       `chore: mark ${setupKeyName} as enrolled`,
@@ -293,7 +293,7 @@ async function processEnrollment(
     // Fetch the new SHA for subsequent commits in this poll cycle.
     // The updateFile response from Gitea doesn't return the new blob SHA
     // in a convenient form, so we re-read it.
-    const freshFile = await gitea.getFileContent("netbird.json", "main");
+    const freshFile = await gitea.getFileContent(config.stateFile, "main");
     onCommit(freshFile.sha, updated);
 
     console.log(JSON.stringify({