Make the weather table upsert non-destructive across sources
ON CONFLICT DO UPDATE was a blind full-row overwrite. Open-data station rows always carry null visibilityMin/dewPoint/sunDuration and an empty phenomena array (fields it doesn't publish), so once a second source (FTP) writes real values for those fields, a later open-data write for the same (city, dateTime) would silently null them back out. Switched to COALESCE(excluded.field, weather.field) so a missing value from one source never erases a real value the other already wrote; phenomena needs NULLIF against an empty array specifically, since Scala's List[String] never maps to SQL NULL.
This commit is contained in:
@@ -2,17 +2,17 @@ INSERT INTO weather (dateTime, city, tempMax, tempMin, tempAvg, precipitation, w
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON CONFLICT (city, dateTime)
|
||||
DO UPDATE SET
|
||||
tempMax = excluded.tempMax,
|
||||
tempMin = excluded.tempMin,
|
||||
tempAvg = excluded.tempAvg,
|
||||
precipitation = excluded.precipitation,
|
||||
windAvg = excluded.windAvg,
|
||||
windMax = excluded.windMax,
|
||||
visibilityMin = excluded.visibilityMin,
|
||||
visibilityAvg = excluded.visibilityAvg,
|
||||
snowAvg = excluded.snowAvg,
|
||||
atmPressure = excluded.atmPressure,
|
||||
dewPoint = excluded.dewPoint,
|
||||
humidity = excluded.humidity,
|
||||
sunDuration = excluded.sunDuration,
|
||||
phenomena = excluded.phenomena;
|
||||
tempMax = COALESCE(excluded.tempMax, weather.tempMax),
|
||||
tempMin = COALESCE(excluded.tempMin, weather.tempMin),
|
||||
tempAvg = COALESCE(excluded.tempAvg, weather.tempAvg),
|
||||
precipitation = COALESCE(excluded.precipitation, weather.precipitation),
|
||||
windAvg = COALESCE(excluded.windAvg, weather.windAvg),
|
||||
windMax = COALESCE(excluded.windMax, weather.windMax),
|
||||
visibilityMin = COALESCE(excluded.visibilityMin, weather.visibilityMin),
|
||||
visibilityAvg = COALESCE(excluded.visibilityAvg, weather.visibilityAvg),
|
||||
snowAvg = COALESCE(excluded.snowAvg, weather.snowAvg),
|
||||
atmPressure = COALESCE(excluded.atmPressure, weather.atmPressure),
|
||||
dewPoint = COALESCE(excluded.dewPoint, weather.dewPoint),
|
||||
humidity = COALESCE(excluded.humidity, weather.humidity),
|
||||
sunDuration = COALESCE(excluded.sunDuration, weather.sunDuration),
|
||||
phenomena = COALESCE(NULLIF(excluded.phenomena, ARRAY[]::text[]), weather.phenomena);
|
||||
Reference in New Issue
Block a user