Add isolated staging and synthetic weather workflow

.dockerignore

@@ -29,3 +29,5 @@
 .idea/shelf
 .idea/workspace.xml
 fly.toml
+web/node_modules
+web/dist

.env-sample

@@ -1,6 +1,11 @@
 POSTGRES_DB=aaa
 POSTGRES_USER=aaa
 POSTGRES_PASSWORD=aaa
+POSTGRES_HOST=postgres
+
+APP_BIND_ADDRESS=127.0.0.1
+APP_PORT=9090
+ENABLE_SCHEDULED_JOBS=true
 
 METEO_USER=aaa
 METEO_PASSWORD=aaa
@@ -13,4 +18,4 @@ LVGMC_PASSWORD=aaa
 HARMONIE_EDR_API_KEY=aaa
 HARMONIE_EDR_URL=aaa
 HARMONIE_STAC_API_KEY=aaa
-HARMONIE_STAC_URL=aaa
+HARMONIE_STAC_URL=aaa

README.md

@@ -10,6 +10,16 @@ Rename `.env-sample` to `.env` and fill in credentials
 docker-compose up --build --no-cache --force-recreate
 ```
 
+## Local synthetic weather data
+
+The local database can be populated with deterministic hourly sample data for all 33 stations. The seed covers the previous 18 months through the current hour and can be run repeatedly without creating duplicates.
+
+```sh
+docker compose --profile tools run --rm seed
+```
+
+This workflow is intended only for development and UI testing. It uses the normal `weather` table and application APIs; it does not provide LVGMC forecast CSV or HARMONIE GRIB fixtures.
+
 ## Web
 [http://0.0.0.0:9090/](http://0.0.0.0:9090/)
 
@@ -68,4 +78,4 @@ df -h
 
 ```
 fly scale memory 512
-```
+```

dev/seed_weather.sql

@@ -0,0 +1,154 @@
+\set ON_ERROR_STOP on
+
+BEGIN;
+
+CREATE TABLE IF NOT EXISTS weather (
+    dateTime TIMESTAMP NOT NULL,
+    city VARCHAR(255) NOT NULL,
+    tempMax DOUBLE PRECISION,
+    tempMin DOUBLE PRECISION,
+    tempAvg DOUBLE PRECISION,
+    precipitation DOUBLE PRECISION,
+    windAvg DOUBLE PRECISION,
+    windMax DOUBLE PRECISION,
+    visibilityMin DOUBLE PRECISION,
+    visibilityAvg DOUBLE PRECISION,
+    snowAvg DOUBLE PRECISION,
+    atmPressure DOUBLE PRECISION,
+    dewPoint DOUBLE PRECISION,
+    humidity DOUBLE PRECISION,
+    sunDuration DOUBLE PRECISION,
+    phenomena TEXT[],
+    UNIQUE(city, dateTime)
+);
+
+WITH
+settings AS (
+    SELECT
+        date_trunc('hour', CURRENT_TIMESTAMP)::timestamp AS end_at,
+        (date_trunc('hour', CURRENT_TIMESTAMP) - INTERVAL '18 months')::timestamp AS start_at
+),
+cities(city, climate_offset) AS (
+    VALUES
+        ('Ainaži', -0.8), ('Alūksne', -2.3), ('Bauska', 0.7), ('Dagda', -1.7),
+        ('Daugavgrīva', 0.6), ('Daugavpils', -0.9), ('Dobele', 0.8), ('Gulbene', -2.0),
+        ('Jelgava', 0.9), ('Kalnciems', 0.7), ('Kolka', 0.3), ('Kuldīga', 0.8),
+        ('Lielpēči', 0.2), ('Liepāja', 1.1), ('Madona', -1.8), ('Mērsrags', 0.2),
+        ('Pāvilosta', 0.9), ('Piedruja', -1.4), ('Priekuļi', -1.6), ('Rēzekne', -1.2),
+        ('Rīga', 1.3), ('Rucava', 1.0), ('Rūjiena', -1.0), ('Saldus', 0.3),
+        ('Sigulda', -0.8), ('Sīļi', -1.3), ('Skrīveri', -0.5), ('Skulte', 0.0),
+        ('Stende', -0.3), ('Ventspils', 0.9), ('Vičaki', 0.2), ('Zīlāni', -1.1),
+        ('Zosēni', -2.1)
+),
+hours AS (
+    SELECT generate_series(settings.start_at, settings.end_at, INTERVAL '1 hour') AS observed_at
+    FROM settings
+),
+base AS (
+    SELECT
+        hours.observed_at,
+        cities.city,
+        cities.climate_offset,
+        abs(hashtext(cities.city || hours.observed_at::text)) AS sample_hash,
+        7.5
+            + 12.5 * sin(2 * pi() * (extract(doy FROM hours.observed_at) - 172) / 365.25)
+            + 2.8 * sin(2 * pi() * (extract(hour FROM hours.observed_at) - 9) / 24)
+            + cities.climate_offset
+            + 1.8 * sin(extract(epoch FROM hours.observed_at) / 173000 + cities.climate_offset) AS temperature
+    FROM hours
+    CROSS JOIN cities
+),
+weather_values AS (
+    SELECT
+        *,
+        CASE
+            WHEN sample_hash % 100 < 13
+                THEN round(((sample_hash % 190) / 10.0 + 0.2)::numeric, 1)::double precision
+            ELSE 0.0
+        END AS rain,
+        round((1.2 + (sample_hash % 65) / 10.0)::numeric, 1)::double precision AS wind,
+        round((58 + (sample_hash % 34) + 8 * cos(2 * pi() * extract(doy FROM observed_at) / 365.25))::numeric, 1)::double precision AS relative_humidity
+    FROM base
+),
+final_values AS (
+    SELECT
+        *,
+        GREATEST(0.0, LEAST(100.0, relative_humidity)) AS bounded_humidity,
+        CASE
+            WHEN temperature < 1.0 AND rain > 0
+                THEN round((rain * 0.8 + (sample_hash % 25) / 10.0)::numeric, 1)::double precision
+            WHEN temperature < -2.0
+                THEN round(((sample_hash % 80) / 10.0)::numeric, 1)::double precision
+            ELSE 0.0
+        END AS snow,
+        CASE
+            WHEN extract(hour FROM observed_at) BETWEEN 7 AND 18 AND rain = 0
+                THEN round((35 + sample_hash % 26)::numeric, 1)::double precision
+            ELSE 0.0
+        END AS sunshine_minutes
+    FROM weather_values
+)
+INSERT INTO weather (
+    dateTime, city, tempMax, tempMin, tempAvg, precipitation, windAvg, windMax,
+    visibilityMin, visibilityAvg, snowAvg, atmPressure, dewPoint, humidity,
+    sunDuration, phenomena
+)
+SELECT
+    observed_at,
+    city,
+    CASE WHEN sample_hash % 997 = 0 THEN NULL ELSE round((temperature + 1.8 + (sample_hash % 12) / 10.0)::numeric, 1)::double precision END,
+    CASE WHEN sample_hash % 991 = 0 THEN NULL ELSE round((temperature - 1.6 - (sample_hash % 10) / 10.0)::numeric, 1)::double precision END,
+    round(temperature::numeric, 1)::double precision,
+    rain,
+    wind,
+    round((wind + 1.5 + (sample_hash % 70) / 10.0)::numeric, 1)::double precision,
+    CASE
+        WHEN rain > 12 THEN round((0.8 + sample_hash % 20 / 10.0)::numeric, 1)::double precision
+        WHEN bounded_humidity > 88 THEN round((1.5 + sample_hash % 40 / 10.0)::numeric, 1)::double precision
+        ELSE round((8 + sample_hash % 80 / 10.0)::numeric, 1)::double precision
+    END,
+    CASE
+        WHEN rain > 12 THEN round((3 + sample_hash % 40 / 10.0)::numeric, 1)::double precision
+        ELSE round((12 + sample_hash % 90 / 10.0)::numeric, 1)::double precision
+    END,
+    snow,
+    round((1000 + sample_hash % 310 / 10.0 + 5 * sin(extract(epoch FROM observed_at) / 250000))::numeric, 1)::double precision,
+    round((temperature - (100 - bounded_humidity) / 5.0)::numeric, 1)::double precision,
+    CASE WHEN sample_hash % 983 = 0 THEN NULL ELSE round(bounded_humidity::numeric, 1)::double precision END,
+    sunshine_minutes,
+    CASE
+        WHEN snow > 0.5 THEN ARRAY['snow']::text[]
+        WHEN rain > 12 THEN ARRAY['heavy rain', 'overcast']::text[]
+        WHEN rain > 0 THEN ARRAY['rain']::text[]
+        WHEN bounded_humidity > 88 THEN ARRAY['fog']::text[]
+        WHEN wind > 6.5 THEN ARRAY['windy']::text[]
+        WHEN sunshine_minutes > 0 THEN ARRAY['clear']::text[]
+        ELSE ARRAY['cloudy']::text[]
+    END
+FROM final_values
+ON CONFLICT (city, dateTime) DO UPDATE SET
+    tempMax = EXCLUDED.tempMax,
+    tempMin = EXCLUDED.tempMin,
+    tempAvg = EXCLUDED.tempAvg,
+    precipitation = EXCLUDED.precipitation,
+    windAvg = EXCLUDED.windAvg,
+    windMax = EXCLUDED.windMax,
+    visibilityMin = EXCLUDED.visibilityMin,
+    visibilityAvg = EXCLUDED.visibilityAvg,
+    snowAvg = EXCLUDED.snowAvg,
+    atmPressure = EXCLUDED.atmPressure,
+    dewPoint = EXCLUDED.dewPoint,
+    humidity = EXCLUDED.humidity,
+    sunDuration = EXCLUDED.sunDuration,
+    phenomena = EXCLUDED.phenomena;
+
+ANALYZE weather;
+
+COMMIT;
+
+SELECT
+    count(*) AS synthetic_rows,
+    count(DISTINCT city) AS stations,
+    min(dateTime) AS first_observation,
+    max(dateTime) AS last_observation
+FROM weather;

docker-compose.yml

@@ -1,4 +1,27 @@
 services:
+  seed:
+    image: postgres:16.1
+    profiles: ["tools"]
+    environment:
+      PGPASSWORD: ${POSTGRES_PASSWORD}
+    volumes:
+      - ./dev/seed_weather.sql:/seed/seed_weather.sql:ro
+    command:
+      - psql
+      - -h
+      - postgres
+      - -U
+      - ${POSTGRES_USER}
+      - -d
+      - ${POSTGRES_DB}
+      - -v
+      - ON_ERROR_STOP=1
+      - -f
+      - /seed/seed_weather.sql
+    depends_on:
+      postgres:
+        condition: service_healthy
+
   node:
     image: node:22.14.0
     working_dir: /web
@@ -8,8 +31,6 @@ services:
 
   postgres:
     image: postgres:16.1
-    ports:
-      - "5432:5432"
     healthcheck:
       test: [ "CMD-SHELL", "pg_isready -d ${POSTGRES_DB} -U ${POSTGRES_USER}" ]
       interval: 5s
@@ -38,6 +59,7 @@ services:
       HARMONIE_EDR_URL: ${HARMONIE_EDR_URL}
       HARMONIE_STAC_API_KEY: ${HARMONIE_STAC_API_KEY}
       HARMONIE_STAC_URL: ${HARMONIE_STAC_URL}
+      ENABLE_SCHEDULED_JOBS: ${ENABLE_SCHEDULED_JOBS:-true}
     build:
       context: .
       dockerfile: Dockerfile.local
@@ -46,10 +68,12 @@ services:
       - scala-data:/data
       - ./web/dist:/web/dist
     ports:
-      - "9090:8080"
+      - "${APP_BIND_ADDRESS:-127.0.0.1}:${APP_PORT:-9090}:8080"
     depends_on:
-      - node
+      node:
-      - postgres
+        condition: service_completed_successfully
+      postgres:
+        condition: service_healthy
 
 volumes:
-  scala-data:
+  scala-data:

docs/UPDATE_ROADMAP.md

@@ -0,0 +1,151 @@
+# WeatherTool update roadmap
+
+This document tracks proposed WeatherTool improvements. Work should be delivered in small, reviewable phases rather than as one large rewrite. Each phase should leave the application runnable and independently testable.
+
+## Environments and workflow
+
+- **Windows development:** primary source-editing and UI-review workspace.
+- **Rocky staging:** production-like Docker deployment at `http://192.168.1.101:9190`.
+- **Git over SSH:** Windows pushes reviewed commits to a private bare repository on Rocky; the Rocky staging checkout pulls those commits and rebuilds.
+- **Production:** remains separate until changes are reviewed, tested, and explicitly approved for workplace use.
+
+Do not synchronize `.env`, database directories, generated dependencies, build output, or provider credentials between machines.
+
+## Working principles
+
+1. Make one coherent change at a time.
+2. Record the existing behavior before intentionally changing it.
+3. Keep dependency updates separate from UI redesign and functional changes.
+4. Test on Windows, then deploy the same commit to Rocky staging.
+5. Use synthetic or sanitized data outside the workplace environment.
+6. Never enable external provider schedules with placeholder credentials.
+7. Do not connect staging to workplace services without explicit authorization.
+
+## Phase 0 — Reproducible development baseline
+
+Status: in progress
+
+- [x] Review backend, frontend, deployment, and security structure.
+- [x] Run the project locally through Docker Desktop.
+- [x] Add configurable host port and scheduled-job switch.
+- [x] Add deterministic synthetic station data for all 33 stations.
+- [x] Create an isolated Rocky Linux staging deployment.
+- [x] Keep staging PostgreSQL private to its Compose network.
+- [ ] Commit the baseline changes and establish the Git-over-SSH workflow.
+- [ ] Document normal build, seed, deploy, backup, and rollback commands.
+- [ ] Capture representative screenshots and expected API responses.
+
+## Phase 1 — Behavior discovery and bug inventory
+
+Status: pending
+
+- [ ] Walk through every page with synthetic data.
+- [ ] Document the actual purpose and intended users of each workflow.
+- [ ] Separate analytical dashboard features from broadcast-graphic authoring tools.
+- [ ] Record unclear controls, missing units, broken states, and layout problems.
+- [ ] Fix the `atmPressire` frontend field typo.
+- [ ] Fix shifted Database export columns caused by duplicated `tempMax`.
+- [ ] Correct malformed integer route handling.
+- [ ] Add consistent loading, empty, and error states.
+
+## Phase 2 — Test safety net
+
+Status: pending
+
+- [ ] Add backend route tests for representative station and country queries.
+- [ ] Add database integration tests for aggregation and export behavior.
+- [ ] Restore and expand CSV parser tests.
+- [ ] Add GRIB parser boundary and malformed-file tests.
+- [ ] Add security tests for invalid fields, filenames, offsets, lengths, and date ranges.
+- [ ] Add frontend type checking and critical workflow smoke tests.
+- [ ] Run tests automatically before staging deployment.
+
+## Phase 3 — Dependency modernization
+
+Status: pending
+
+The first observed frontend install reported 15 vulnerabilities: 1 critical, 10 high, 3 moderate, and 1 low. Exact advisories must be reviewed before choosing upgrades.
+
+- [ ] Capture and review the full npm audit report.
+- [ ] Update direct frontend dependencies in controlled groups.
+- [ ] Replace or remove obsolete frontend packages where appropriate.
+- [ ] Rebuild and visually compare every page after frontend upgrades.
+- [ ] Update Scala within the supported 2.13 line before considering larger migration.
+- [ ] Update http4s, Doobie, Circe, Cats Effect, Logback, and test libraries in compatible groups.
+- [ ] Replace release-candidate dependencies with stable releases where possible.
+- [ ] Update Docker base images deliberately and pin reproducible versions.
+- [ ] Verify database compatibility and generated artifacts after every group.
+
+Dependency changes must not be combined with a visual redesign unless a package migration strictly requires it.
+
+## Phase 4 — Security hardening
+
+Status: pending
+
+- [ ] Rotate and remove the API key exposed in a source comment.
+- [ ] Remove credentials from connection-error messages.
+- [ ] Protect or remove debug and administrative endpoints.
+- [ ] Convert state-changing `GET` routes to appropriate methods.
+- [ ] Introduce closed, validated weather-field and aggregation types.
+- [ ] Eliminate raw user-controlled SQL identifiers.
+- [ ] Validate and constrain filenames, resolved paths, offsets, and byte lengths.
+- [ ] Add query-range, response-size, request-rate, and timeout limits.
+- [ ] Restrict CORS to intended origins.
+- [ ] Define authentication and authorization requirements for workplace deployment.
+
+## Phase 5 — Runtime reliability and operations
+
+Status: pending
+
+- [ ] Manage custom executors as resources and close them cleanly.
+- [ ] Remove explicit `System.gc()` calls.
+- [ ] Supervise scheduled jobs independently instead of recursively restarting the application.
+- [ ] Add application health and readiness endpoints.
+- [ ] Add structured logging without leaking secrets.
+- [ ] Define PostgreSQL and GRIB-data backup/restore procedures.
+- [ ] Add container resource limits and deployment health checks.
+- [ ] Document monitoring, update, rollback, and incident procedures.
+
+## Phase 6 — Information architecture and UI redesign
+
+Status: pending
+
+- [ ] Identify primary user roles and their most frequent tasks.
+- [ ] Separate historical analysis, live station monitoring, database inspection, HARMONIE visualization, and broadcast graphics.
+- [ ] Replace technical/internal labels with task-oriented language.
+- [ ] Replace the character-based weather-icon entry workflow with a visual picker or automatic mapping.
+- [ ] Explain or automate manual wind and weather-icon inputs.
+- [ ] Add units, legends, contextual help, and clear date semantics.
+- [ ] Establish a responsive layout, typography, spacing, and component system.
+- [ ] Design explicit export/download workflows for broadcast assets.
+- [ ] Test target resolutions and real workplace display conditions.
+- [ ] Check keyboard navigation, contrast, focus states, and screen-reader labeling.
+
+## Phase 7 — Real data and production readiness
+
+Status: pending
+
+- [ ] Confirm the actual workplace deployment topology and current deployed commit.
+- [ ] Obtain authorized development credentials or representative fixtures.
+- [ ] Validate LVGMC station and forecast CSV ingestion.
+- [ ] Validate DMI STAC discovery and EDR GRIB downloads.
+- [ ] Test scheduled ingestion failure and recovery behavior.
+- [ ] Rehearse deployment and rollback using sanitized data.
+- [ ] Obtain technical and operational review before workplace rollout.
+
+## Known current limitations
+
+- Staging uses synthetic PostgreSQL station data.
+- LVGMC forecast CSV fixtures are not yet available.
+- HARMONIE GRIB fixtures are not yet available.
+- Scheduled provider downloads are disabled in development and staging.
+- Existing automated test coverage is minimal.
+- The current UI combines analysis and broadcast-authoring concepts without explanation.
+
+## Change log
+
+Record completed work here by date and commit after the Git workflow is established.
+
+| Date | Commit | Summary | Verified on Rocky |
+|---|---|---|---|
+| 2026-08-18 | pending | Docker development baseline, isolated staging, scheduler switch, and synthetic station data | Yes |

src/main/scala/Main.scala

@@ -1,6 +1,6 @@
 import cats.effect._
 import cats.effect.unsafe.implicits.global
-import cats.implicits.catsSyntaxTuple4Parallel
+import cats.implicits._
 import data.DataService
 import db.{DBConnection, PostgresService}
 import fetch.csv.{FileNameService}
@@ -43,7 +43,13 @@ object Main extends IOApp {
       server <- Server.of(postgresService, dataService, fetchLvgmcService)
       serverTask = server.run
 
-      exitCode <- (serverTask, fetchStationsTask, cleanupTask, fetchGribTask).parMapN((_, _, _, _) => ExitCode.Success)
+      scheduledTasks =
+        if (sys.env.get("ENABLE_SCHEDULED_JOBS").exists(_.equalsIgnoreCase("false")))
+          IO.never[Unit]
+        else
+          (fetchStationsTask, cleanupTask, fetchGribTask).parMapN((_, _, _) => ())
+
+      exitCode <- (serverTask, scheduledTasks).parMapN((_, _) => ExitCode.Success)
     } yield exitCode
 
     program.handleErrorWith { error =>
@@ -77,4 +83,4 @@ object Main extends IOApp {
     } { writer =>
       IO(writer.close()).handleErrorWith(e => IO(e.printStackTrace()))
     }
-}
+}