Stop generating synthetic weather data
This commit is contained in:
b0txec
@@ -16,16 +16,6 @@ Rename `.env-sample` to `.env` and fill in credentials
|
||||
docker-compose up --build --no-cache --force-recreate
|
||||
```
|
||||
|
||||
## Local synthetic weather data
|
||||
|
||||
The local database can be populated with deterministic hourly sample data for the repository station set plus Valmiera, which is required by the fixed Faktiskā template. The seed covers the previous 18 months through the current hour and can be run repeatedly without creating duplicates.
|
||||
|
||||
```sh
|
||||
docker compose --profile tools run --rm seed
|
||||
```
|
||||
|
||||
This workflow is intended only for development and UI testing. It uses the normal `weather` table and application APIs; it does not provide LVGMC forecast CSV or HARMONIE GRIB fixtures.
|
||||
|
||||
## Web
|
||||
[http://0.0.0.0:9090/](http://0.0.0.0:9090/)
|
||||
|
||||
|
||||
@@ -1,154 +0,0 @@
|
||||
\set ON_ERROR_STOP on
|
||||
|
||||
BEGIN;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS weather (
|
||||
dateTime TIMESTAMP NOT NULL,
|
||||
city VARCHAR(255) NOT NULL,
|
||||
tempMax DOUBLE PRECISION,
|
||||
tempMin DOUBLE PRECISION,
|
||||
tempAvg DOUBLE PRECISION,
|
||||
precipitation DOUBLE PRECISION,
|
||||
windAvg DOUBLE PRECISION,
|
||||
windMax DOUBLE PRECISION,
|
||||
visibilityMin DOUBLE PRECISION,
|
||||
visibilityAvg DOUBLE PRECISION,
|
||||
snowAvg DOUBLE PRECISION,
|
||||
atmPressure DOUBLE PRECISION,
|
||||
dewPoint DOUBLE PRECISION,
|
||||
humidity DOUBLE PRECISION,
|
||||
sunDuration DOUBLE PRECISION,
|
||||
phenomena TEXT[],
|
||||
UNIQUE(city, dateTime)
|
||||
);
|
||||
|
||||
WITH
|
||||
settings AS (
|
||||
SELECT
|
||||
date_trunc('hour', CURRENT_TIMESTAMP)::timestamp AS end_at,
|
||||
(date_trunc('hour', CURRENT_TIMESTAMP) - INTERVAL '18 months')::timestamp AS start_at
|
||||
),
|
||||
cities(city, climate_offset) AS (
|
||||
VALUES
|
||||
('Ainaži', -0.8), ('Alūksne', -2.3), ('Bauska', 0.7), ('Dagda', -1.7),
|
||||
('Daugavgrīva', 0.6), ('Daugavpils', -0.9), ('Dobele', 0.8), ('Gulbene', -2.0),
|
||||
('Jelgava', 0.9), ('Kalnciems', 0.7), ('Kolka', 0.3), ('Kuldīga', 0.8),
|
||||
('Lielpēči', 0.2), ('Liepāja', 1.1), ('Madona', -1.8), ('Mērsrags', 0.2),
|
||||
('Pāvilosta', 0.9), ('Piedruja', -1.4), ('Priekuļi', -1.6), ('Rēzekne', -1.2),
|
||||
('Rīga', 1.3), ('Rucava', 1.0), ('Rūjiena', -1.0), ('Saldus', 0.3),
|
||||
('Sigulda', -0.8), ('Sīļi', -1.3), ('Skrīveri', -0.5), ('Skulte', 0.0),
|
||||
('Stende', -0.3), ('Valmiera', -1.2), ('Ventspils', 0.9), ('Vičaki', 0.2), ('Zīlāni', -1.1),
|
||||
('Zosēni', -2.1)
|
||||
),
|
||||
hours AS (
|
||||
SELECT generate_series(settings.start_at, settings.end_at, INTERVAL '1 hour') AS observed_at
|
||||
FROM settings
|
||||
),
|
||||
base AS (
|
||||
SELECT
|
||||
hours.observed_at,
|
||||
cities.city,
|
||||
cities.climate_offset,
|
||||
abs(hashtext(cities.city || hours.observed_at::text)) AS sample_hash,
|
||||
7.5
|
||||
+ 12.5 * sin(2 * pi() * (extract(doy FROM hours.observed_at) - 172) / 365.25)
|
||||
+ 2.8 * sin(2 * pi() * (extract(hour FROM hours.observed_at) - 9) / 24)
|
||||
+ cities.climate_offset
|
||||
+ 1.8 * sin(extract(epoch FROM hours.observed_at) / 173000 + cities.climate_offset) AS temperature
|
||||
FROM hours
|
||||
CROSS JOIN cities
|
||||
),
|
||||
weather_values AS (
|
||||
SELECT
|
||||
*,
|
||||
CASE
|
||||
WHEN sample_hash % 100 < 13
|
||||
THEN round(((sample_hash % 190) / 10.0 + 0.2)::numeric, 1)::double precision
|
||||
ELSE 0.0
|
||||
END AS rain,
|
||||
round((1.2 + (sample_hash % 65) / 10.0)::numeric, 1)::double precision AS wind,
|
||||
round((58 + (sample_hash % 34) + 8 * cos(2 * pi() * extract(doy FROM observed_at) / 365.25))::numeric, 1)::double precision AS relative_humidity
|
||||
FROM base
|
||||
),
|
||||
final_values AS (
|
||||
SELECT
|
||||
*,
|
||||
GREATEST(0.0, LEAST(100.0, relative_humidity)) AS bounded_humidity,
|
||||
CASE
|
||||
WHEN temperature < 1.0 AND rain > 0
|
||||
THEN round((rain * 0.8 + (sample_hash % 25) / 10.0)::numeric, 1)::double precision
|
||||
WHEN temperature < -2.0
|
||||
THEN round(((sample_hash % 80) / 10.0)::numeric, 1)::double precision
|
||||
ELSE 0.0
|
||||
END AS snow,
|
||||
CASE
|
||||
WHEN extract(hour FROM observed_at) BETWEEN 7 AND 18 AND rain = 0
|
||||
THEN round((35 + sample_hash % 26)::numeric, 1)::double precision
|
||||
ELSE 0.0
|
||||
END AS sunshine_minutes
|
||||
FROM weather_values
|
||||
)
|
||||
INSERT INTO weather (
|
||||
dateTime, city, tempMax, tempMin, tempAvg, precipitation, windAvg, windMax,
|
||||
visibilityMin, visibilityAvg, snowAvg, atmPressure, dewPoint, humidity,
|
||||
sunDuration, phenomena
|
||||
)
|
||||
SELECT
|
||||
observed_at,
|
||||
city,
|
||||
CASE WHEN sample_hash % 997 = 0 THEN NULL ELSE round((temperature + 1.8 + (sample_hash % 12) / 10.0)::numeric, 1)::double precision END,
|
||||
CASE WHEN sample_hash % 991 = 0 THEN NULL ELSE round((temperature - 1.6 - (sample_hash % 10) / 10.0)::numeric, 1)::double precision END,
|
||||
round(temperature::numeric, 1)::double precision,
|
||||
rain,
|
||||
wind,
|
||||
round((wind + 1.5 + (sample_hash % 70) / 10.0)::numeric, 1)::double precision,
|
||||
CASE
|
||||
WHEN rain > 12 THEN round((0.8 + sample_hash % 20 / 10.0)::numeric, 1)::double precision
|
||||
WHEN bounded_humidity > 88 THEN round((1.5 + sample_hash % 40 / 10.0)::numeric, 1)::double precision
|
||||
ELSE round((8 + sample_hash % 80 / 10.0)::numeric, 1)::double precision
|
||||
END,
|
||||
CASE
|
||||
WHEN rain > 12 THEN round((3 + sample_hash % 40 / 10.0)::numeric, 1)::double precision
|
||||
ELSE round((12 + sample_hash % 90 / 10.0)::numeric, 1)::double precision
|
||||
END,
|
||||
snow,
|
||||
round((1000 + sample_hash % 310 / 10.0 + 5 * sin(extract(epoch FROM observed_at) / 250000))::numeric, 1)::double precision,
|
||||
round((temperature - (100 - bounded_humidity) / 5.0)::numeric, 1)::double precision,
|
||||
CASE WHEN sample_hash % 983 = 0 THEN NULL ELSE round(bounded_humidity::numeric, 1)::double precision END,
|
||||
sunshine_minutes,
|
||||
CASE
|
||||
WHEN snow > 0.5 THEN ARRAY['snow']::text[]
|
||||
WHEN rain > 12 THEN ARRAY['heavy rain', 'overcast']::text[]
|
||||
WHEN rain > 0 THEN ARRAY['rain']::text[]
|
||||
WHEN bounded_humidity > 88 THEN ARRAY['fog']::text[]
|
||||
WHEN wind > 6.5 THEN ARRAY['windy']::text[]
|
||||
WHEN sunshine_minutes > 0 THEN ARRAY['clear']::text[]
|
||||
ELSE ARRAY['cloudy']::text[]
|
||||
END
|
||||
FROM final_values
|
||||
ON CONFLICT (city, dateTime) DO UPDATE SET
|
||||
tempMax = EXCLUDED.tempMax,
|
||||
tempMin = EXCLUDED.tempMin,
|
||||
tempAvg = EXCLUDED.tempAvg,
|
||||
precipitation = EXCLUDED.precipitation,
|
||||
windAvg = EXCLUDED.windAvg,
|
||||
windMax = EXCLUDED.windMax,
|
||||
visibilityMin = EXCLUDED.visibilityMin,
|
||||
visibilityAvg = EXCLUDED.visibilityAvg,
|
||||
snowAvg = EXCLUDED.snowAvg,
|
||||
atmPressure = EXCLUDED.atmPressure,
|
||||
dewPoint = EXCLUDED.dewPoint,
|
||||
humidity = EXCLUDED.humidity,
|
||||
sunDuration = EXCLUDED.sunDuration,
|
||||
phenomena = EXCLUDED.phenomena;
|
||||
|
||||
ANALYZE weather;
|
||||
|
||||
COMMIT;
|
||||
|
||||
SELECT
|
||||
count(*) AS synthetic_rows,
|
||||
count(DISTINCT city) AS stations,
|
||||
min(dateTime) AS first_observation,
|
||||
max(dateTime) AS last_observation
|
||||
FROM weather;
|
||||
@@ -1,27 +1,4 @@
|
||||
services:
|
||||
seed:
|
||||
image: postgres:16.1
|
||||
profiles: ["tools"]
|
||||
environment:
|
||||
PGPASSWORD: ${POSTGRES_PASSWORD}
|
||||
volumes:
|
||||
- ./dev/seed_weather.sql:/seed/seed_weather.sql:ro
|
||||
command:
|
||||
- psql
|
||||
- -h
|
||||
- postgres
|
||||
- -U
|
||||
- ${POSTGRES_USER}
|
||||
- -d
|
||||
- ${POSTGRES_DB}
|
||||
- -v
|
||||
- ON_ERROR_STOP=1
|
||||
- -f
|
||||
- /seed/seed_weather.sql
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
|
||||
node:
|
||||
image: node:22.14.0
|
||||
working_dir: /web
|
||||
|
||||
+15
-4
@@ -173,10 +173,21 @@ running in parallel until each real source is proven, not cut over in one step.
|
||||
falls back to older synthetic data via the existing stale-observation
|
||||
UI rather than breaking), and Kartes' aggregate query API returning
|
||||
sane blended real+synthetic values.
|
||||
- [ ] Let the open-data fetch run for a trial period, then remove the private
|
||||
FTP path (`fetch/lvgmc/FetchService.scala`'s `fetchWeatherStations`,
|
||||
its `LVGMC_*` env vars) and the synthetic seed machinery
|
||||
(`dev/seed_weather.sql`, the `seed` Compose service) once confident.
|
||||
- [x] Stop synthetic data generation now that real data is verified and this
|
||||
tool may plausibly be used against live broadcasts, where mixed
|
||||
real/fake data — both shown identically as "older observation" — is a
|
||||
real hazard, not just messiness: removed `dev/seed_weather.sql`, the
|
||||
`seed` Compose service, and the README section describing it.
|
||||
- [ ] Wipe the synthetic rows already sitting in the `weather` table (a
|
||||
separate step from stopping generation — the seed already ran once
|
||||
before this decision) so history going forward is real data only, not
|
||||
a mix. Cities with no matching open-data station (Valmiera, Cēsis)
|
||||
should show no data rather than falling back to old synthetic rows —
|
||||
deferred by design, revisit once a real source is found or accepted as
|
||||
permanently unavailable.
|
||||
- [ ] Once the open-data fetch has run for a trial period, remove the
|
||||
private FTP path (`fetch/lvgmc/FetchService.scala`'s
|
||||
`fetchWeatherStations`, its `LVGMC_*` env vars) if it remains unused.
|
||||
- [ ] Register free DMI Open Data credentials and validate DMI STAC discovery
|
||||
and EDR GRIB downloads (code already exists and is already scheduled;
|
||||
this is credential registration and verification, not new code —
|
||||
|
||||
Reference in New Issue
Block a user