From cb662e5f53bca229b49c9ecc7a7264cb9760d52c Mon Sep 17 00:00:00 2001 From: Guntis Smaukstelis Date: Fri, 31 Jan 2025 14:56:59 +0200 Subject: [PATCH] Add mapping of meteo discipline, category, product --- src/main/scala/grib/Codes.scala | 193 +++++++++++++++++++++ src/main/scala/grib/Grib.scala | 1 + src/main/scala/grib/GribParser.scala | 17 +- src/main/scala/grib/GribParserTest.scala | 4 + src/main/scala/server/Server.scala | 12 +- src/main/scala/server/ValidateRoutes.scala | 9 +- 6 files changed, 226 insertions(+), 10 deletions(-) create mode 100644 src/main/scala/grib/Codes.scala diff --git a/src/main/scala/grib/Codes.scala b/src/main/scala/grib/Codes.scala new file mode 100644 index 0000000..a39d958 --- /dev/null +++ b/src/main/scala/grib/Codes.scala @@ -0,0 +1,193 @@ +package grib + + +object Codes { + def codesToString(discipline: Int, category: Int, product: Int): String = { + val disciplineName = gribCodes.get(discipline).map(_._1).getOrElse("unknown") + val categoryName = gribCodes.get(discipline).flatMap(_._2.get(category)).map(_._1).getOrElse("unknown") + val productName = gribCodes.get(discipline).flatMap(_._2.get(category)).flatMap(_._2.get(product)).getOrElse("unknown") + + s"$disciplineName, $categoryName, $productName" + } + + private val gribCodes: Map[Int, (String, Map[Int, (String, Map[Int, String])])] = Map( + 0 -> ( + "meteorology", + Map( + 0 -> ( + "temperature", + Map( + 0 -> "temperature", + 1 -> "virtual-temperature", + 2 -> "potential-temperature", + 3 -> "pseudo - adiabatic - potential - temperature", + 4 -> "maximum - temperature", + 5 -> "minimum - temperature", + 6 -> "dev - point - temperature", + 7 -> "dev - point - depression", + 8 -> "lapse - rate", + 9 -> "temperature - anomaly", + 10 -> "latent - heat - net - flux", + 11 -> "sensible - heat - net - flux", + 30 -> "latent - heat - net - flux - fue - to - evaporation", + 31 -> "latent - heat - net - flux - due - to - sublimation", + ), + ), + 1 -> ( + "moisture", + Map( + 0 -> "specific-humidity", + 1 -> "relative-humidity", + 2 -> "humidity - mixing - ratio", + 3 -> "precipitable - water", + 4 -> "vapour - pressure", + 5 -> "saturation - deficit", + 6 -> "evaporation", + 7 -> "precipitation - rate", + 8 -> "total - precipitation", + 52 -> "total - precipitation - rate", + 53 -> "total - snowfall - rate - water - equivalent", + 60 -> "snow - depth - water - equivalent", + 64 -> "total - column - integrated - water - vapour", + 65 -> "rain - precipitation - rate", + 69 -> "total - column - integrate - cloud - water", + 70 -> "total - column - integrate - cloud - ice", + 75 -> "graupel - snow - pellets - prepitation - rate", + 128 -> "total - solid - precipitation - rate", + 192 -> "categorical - rain", + 199 -> "potential - evaporation", + ), + ), + 2 -> ( + "momentum", + Map( + 0 -> "wind - direction", // from which blowing + 1 -> "wind - speed", + 2 -> "u - component - of - wind", + 3 -> "v - component - of - wind", + 4 -> "stream - function", + 5 -> "velocity - potential", + 17 -> "momentum - flux - u - component", + 18 -> "momentum - flux - v - component", + 22 -> "wind - speed - gust", + 23 -> "u - component - of - wind - gust", + 24 -> "v - component - of - wind - gust", + ), + ), + 3 -> ( + "mass", + Map( + 0 -> "pressure", + 1 -> "pressure - reduced - to - msl", + 2 -> "pressure - tendency", + 3 -> "icao - standard - atmosphere - reference - height", + 4 -> "geopotential", + 5 -> "geopotential - height", + 6 -> "geometric - height", + ), + ), + 4 -> ( + "short-wave-radiation", + Map( + 0 -> "net - short - wave - radiation - flux - surface", + 1 -> "net - short - wave - radiation - flux - top - of - atmosphere", + 2 -> "short - wave - radiation - flux", + 3 -> "global - radiation - flux", + 4 -> "brightness - temperature", + 5 -> "radiance - with -respect - to - wave - number", + 6 -> "radiance - with -respect - to - wavelength", + 7 -> "downward - short - wave - radiation - flux", + 8 -> "upward - short - wave - radiation - flux", + 9 -> "net - short - wave - radiation - flux", + ), + ), + 5 -> ( + "long-wave-radiation", + Map( + 4 -> "upward-long-wave-radiation-flux", + 5 -> "net-long-wave-radiation-flux", + ), + ), + 6 -> ( + "cloud", + Map( + 0 -> "cloud - ice", + 1 -> "total - cloud - cover", + 2 -> "convective - cloud - cover", + 3 -> "low - cloud - cover", + 4 -> "medium - cloud - cover", + 5 -> "high - cloud - cover", + 6 -> "cloud - water", + 7 -> "cloud - amount", + 8 -> "cloud - type", + 9 -> "thunderstorm - maximum - tops", + 10 -> "thunderstorm - coverage", + 11 -> "cloud - base", + 12 -> "cloud - top", + 32 -> "fraction - of - cloud - cover", + 199 -> "ice - fraction - of - total - condensate", + ), + ), + 7 -> ( + "thermodynamic-stability", + Map( + 6 -> "convective - available - potential - energy", + 7 -> "convective - inhibition", + ), + ), + 17 -> ( + "electrodynamics", + Map( + 193 -> "unknown-local-use", + ), + ), + 19 -> ( + "physical-atmospheric", + Map( + 0 -> "visibility", + 1 -> "albedo", + 2 -> "thunderstorm - probability", + 3 -> "mixed - layer - depth", + 4 -> "volcanic - ash", + ), + ), + ), + ), + + 1 -> ( + "hydrologic", + Map(), + ), + + 2 -> ( + "land-surface", + Map( + 0 -> ( + "vegetation-biomass", + Map( + 0 -> "land-cover", + ) + ) + ), + ), + + 3 -> ( + "space", + Map( + 2 -> ( + "charged-particle-mass", + Map( + 1 -> "electron-density" + ), + ), + 6 -> ( + "solar-electromagnetic-emissions", + Map( + 3 -> "solar-euv-irradiance", + ), + ), + ), + ), + ) +} + diff --git a/src/main/scala/grib/Grib.scala b/src/main/scala/grib/Grib.scala index 5e0c996..35df7a4 100644 --- a/src/main/scala/grib/Grib.scala +++ b/src/main/scala/grib/Grib.scala @@ -2,6 +2,7 @@ package grib case class Grib( version: Int, length: Long, + title: String, grid: GribGrid, meteo: MeteoParam, time: GribTime, diff --git a/src/main/scala/grib/GribParser.scala b/src/main/scala/grib/GribParser.scala index 6721af8..627fd92 100644 --- a/src/main/scala/grib/GribParser.scala +++ b/src/main/scala/grib/GribParser.scala @@ -53,11 +53,12 @@ object GribParser { GribSection(5, ptr5, len5), GribSection(6, ptr6, len6), ) - grib = Grib(version, gribLength, grid, meteo, time, conversion, bitsPerDataPoint, sections) + title = Codes.codesToString(meteo.discipline, meteo.category, meteo.product) + grib = Grib(version, gribLength, title, grid, meteo, time, conversion, bitsPerDataPoint, sections) } yield grib } - def parse0(path: Path, ptr: Long): IO[(Int, Int, Long, Int)] = { + private def parse0(path: Path, ptr: Long): IO[(Int, Int, Long, Int)] = { val length = 16 for { bytes <- readBytes(path, ptr, length) @@ -68,7 +69,7 @@ object GribParser { } yield (version, discipline, gribLength, length) } - def parse1(path: Path, ptr: Long): IO[(ZonedDateTime, Int)] = { + private def parse1(path: Path, ptr: Long): IO[(ZonedDateTime, Int)] = { for { bytes <- readBytes(path, ptr, 64) length = ByteBuffer.wrap(bytes.slice(0, 4)).getInt @@ -87,7 +88,7 @@ object GribParser { } yield (referenceTime, length) } - def parse3(path: Path, ptr: Long): IO[(GribGrid, Int)] = { + private def parse3(path: Path, ptr: Long): IO[(GribGrid, Int)] = { for { bytes <- readBytes(path, ptr, 64) length = ByteBuffer.wrap(bytes.slice(0, 4)).getInt @@ -97,7 +98,7 @@ object GribParser { } yield (GribGrid(template, cols, rows), length) } - def parse4(path: Path, ptr: Long, discipline: Int, referenceTime: ZonedDateTime): IO[(MeteoParam, GribTime, Int)] = { + private def parse4(path: Path, ptr: Long, discipline: Int, referenceTime: ZonedDateTime): IO[(MeteoParam, GribTime, Int)] = { for { bytes <- readBytes(path, ptr, 64) length = ByteBuffer.wrap(bytes.slice(0, 4)).getInt @@ -127,7 +128,7 @@ object GribParser { } yield (meteoParam, time, length) } - def parse5(path: Path, ptr: Long): IO[(MeteoConversion, Int, Int)] = { + private def parse5(path: Path, ptr: Long): IO[(MeteoConversion, Int, Int)] = { for { bytes <- readBytes(path, ptr, 64) length = ByteBuffer.wrap(bytes.slice(0, 4)).getInt @@ -139,14 +140,14 @@ object GribParser { } yield (conversion, bitsPerDataPoint, length) } - def parse6(path: Path, ptr: Long): IO[Int] = { + private def parse6(path: Path, ptr: Long): IO[Int] = { for { bytes <- readBytes(path, ptr, 64) length = ByteBuffer.wrap(bytes.slice(0, 4)).getInt } yield length } - def readBytes(path: Path, ptr: Long, len: Int): IO[Array[Byte]] = { + private def readBytes(path: Path, ptr: Long, len: Int): IO[Array[Byte]] = { Files[IO].readRange(path, 1024, ptr, ptr + len) .compile .to(Array) diff --git a/src/main/scala/grib/GribParserTest.scala b/src/main/scala/grib/GribParserTest.scala index dfcce88..9685835 100644 --- a/src/main/scala/grib/GribParserTest.scala +++ b/src/main/scala/grib/GribParserTest.scala @@ -8,6 +8,10 @@ import cats.effect.unsafe.implicits.global object GribParserTest { def main(args: Array[String]): Unit = { + val gribTitle = Codes.codesToString(0, 0, 2) + println(gribTitle) + + val fileName = "data/HARMONIE_DINI_SF_2025-01-24T030000Z_2025-01-26T010000Z.grib" val path = Path(fileName) diff --git a/src/main/scala/server/Server.scala b/src/main/scala/server/Server.scala index b3f9d9f..2f36019 100644 --- a/src/main/scala/server/Server.scala +++ b/src/main/scala/server/Server.scala @@ -5,8 +5,9 @@ import cats.implicits.toTraverseOps import com.comcast.ip4s.IpLiteralSyntax import db.PostgresService import fetch.FetchService +import grib.GribParser import parse.Aggregate -import server.ValidateRoutes.{AggFieldList, AggKey, CityList, DateTimeRange, Granularity, ValidateDate, ValidateDateTime, ValidateMonths} +import server.ValidateRoutes.{AggFieldList, AggKey, CityList, DateTimeRange, Granularity, ValidateDate, ValidateDateTime, ValidateMonths, ValidateZonedDateTime} import io.circe.{Json, Printer} import org.http4s._ import org.http4s.dsl.io._ @@ -24,6 +25,7 @@ import parse.Aggregate.{AggregateKey, UserQuery} import org.http4s.circe.jsonEncoder import org.typelevel.log4cats.Logger import org.typelevel.log4cats.slf4j.Slf4jLogger +import fs2.io.file.Path import scala.concurrent.duration.DurationInt @@ -49,6 +51,14 @@ class Server(postgresService: PostgresService, fetch: FetchService, log: Logger[ private case class ResponseWrapper(result: Map[String, Option[Aggregate.AggregateValue]], query: UserQuery) private val apiRoutes = HttpRoutes.of[IO] { + // http://0.0.0.0:8080/api/show/grib/2025-01-24T03:00:00Z +// case GET -> Root / "show" / "grib" / ValidateZonedDateTime(referenceTime) => + case GET -> Root / "show" / "grib" => + // TODO change from hardcoded value to reference time and forecast time + val fileName = "data/HARMONIE_DINI_SF_2025-01-24T030000Z_2025-01-26T010000Z.grib" + GribParser.parseFile(Path(fileName)).flatMap(response => Ok(response.asJson.pretty)) + + case GET -> Root / "show" / "gribName" => Ok("{\"fileName\":\"TODO replace this fake name\"}") // http://0.0.0.0:8080/api/query/city/Liepāja,Rēzekne/20230414_2200-20230501_1230/hour/tempMax/max case GET -> Root / "query" / "city" / CityList(cities) / DateTimeRange(from, to) / Granularity(granularity) / field / AggKey(key) => diff --git a/src/main/scala/server/ValidateRoutes.scala b/src/main/scala/server/ValidateRoutes.scala index a8de66c..e700703 100644 --- a/src/main/scala/server/ValidateRoutes.scala +++ b/src/main/scala/server/ValidateRoutes.scala @@ -4,7 +4,7 @@ import cats.data.NonEmptyList import parse.Aggregate.AggregateKey import parse.WeatherData -import java.time.{LocalDate, LocalDateTime} +import java.time.{LocalDate, LocalDateTime, ZonedDateTime} import java.time.format.DateTimeFormatter import java.time.temporal.ChronoUnit import scala.util.Try @@ -36,6 +36,13 @@ object ValidateRoutes { } } + object ValidateZonedDateTime { + def unapply(str: String): Option[ZonedDateTime] = { + val formatter = DateTimeFormatter.ofPattern("yyyy-MM-ddTHH:mm:ssZ") + Try(ZonedDateTime.parse(str, formatter)).toOption + } + } + object ValidateMonths { def unapply(str: String): Option[List[LocalDate]] = { val formatter = DateTimeFormatter.ofPattern("yyyyMMdd")