Make the weather table upsert non-destructive across sources
ON CONFLICT DO UPDATE was a blind full-row overwrite. Open-data station rows always carry null visibilityMin/dewPoint/sunDuration and an empty phenomena array (fields it doesn't publish), so once a second source (FTP) writes real values for those fields, a later open-data write for the same (city, dateTime) would silently null them back out. Switched to COALESCE(excluded.field, weather.field) so a missing value from one source never erases a real value the other already wrote; phenomena needs NULLIF against an empty array specifically, since Scala's List[String] never maps to SQL NULL.
This commit is contained in:
@@ -2,17 +2,17 @@ INSERT INTO weather (dateTime, city, tempMax, tempMin, tempAvg, precipitation, w
|
|||||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||||
ON CONFLICT (city, dateTime)
|
ON CONFLICT (city, dateTime)
|
||||||
DO UPDATE SET
|
DO UPDATE SET
|
||||||
tempMax = excluded.tempMax,
|
tempMax = COALESCE(excluded.tempMax, weather.tempMax),
|
||||||
tempMin = excluded.tempMin,
|
tempMin = COALESCE(excluded.tempMin, weather.tempMin),
|
||||||
tempAvg = excluded.tempAvg,
|
tempAvg = COALESCE(excluded.tempAvg, weather.tempAvg),
|
||||||
precipitation = excluded.precipitation,
|
precipitation = COALESCE(excluded.precipitation, weather.precipitation),
|
||||||
windAvg = excluded.windAvg,
|
windAvg = COALESCE(excluded.windAvg, weather.windAvg),
|
||||||
windMax = excluded.windMax,
|
windMax = COALESCE(excluded.windMax, weather.windMax),
|
||||||
visibilityMin = excluded.visibilityMin,
|
visibilityMin = COALESCE(excluded.visibilityMin, weather.visibilityMin),
|
||||||
visibilityAvg = excluded.visibilityAvg,
|
visibilityAvg = COALESCE(excluded.visibilityAvg, weather.visibilityAvg),
|
||||||
snowAvg = excluded.snowAvg,
|
snowAvg = COALESCE(excluded.snowAvg, weather.snowAvg),
|
||||||
atmPressure = excluded.atmPressure,
|
atmPressure = COALESCE(excluded.atmPressure, weather.atmPressure),
|
||||||
dewPoint = excluded.dewPoint,
|
dewPoint = COALESCE(excluded.dewPoint, weather.dewPoint),
|
||||||
humidity = excluded.humidity,
|
humidity = COALESCE(excluded.humidity, weather.humidity),
|
||||||
sunDuration = excluded.sunDuration,
|
sunDuration = COALESCE(excluded.sunDuration, weather.sunDuration),
|
||||||
phenomena = excluded.phenomena;
|
phenomena = COALESCE(NULLIF(excluded.phenomena, ARRAY[]::text[]), weather.phenomena);
|
||||||
Reference in New Issue
Block a user