2025-02-08 23:34:50 +02:00
|
|
|
package data
|
|
|
|
|
|
|
|
|
|
import cats.effect.IO
|
2025-02-11 11:33:27 +02:00
|
|
|
import cats.implicits.toTraverseOps
|
2025-02-08 23:34:50 +02:00
|
|
|
import fs2.io.file.{Files, Path}
|
|
|
|
|
import grib.{Grib, GribParser}
|
2025-02-13 16:11:50 +02:00
|
|
|
import org.typelevel.log4cats.Logger
|
|
|
|
|
import org.typelevel.log4cats.slf4j.Slf4jLogger
|
2025-02-08 23:34:50 +02:00
|
|
|
|
2025-02-13 16:11:50 +02:00
|
|
|
import java.nio.file.Paths
|
2025-02-10 23:43:58 +02:00
|
|
|
import java.time.{ZoneId, ZoneOffset, ZonedDateTime}
|
2025-02-08 23:34:50 +02:00
|
|
|
import java.time.format.DateTimeFormatter
|
|
|
|
|
import scala.io.Source
|
|
|
|
|
import scala.util.Try
|
|
|
|
|
|
|
|
|
|
object DataService {
|
2025-02-13 16:11:50 +02:00
|
|
|
def of: IO[DataService] = {
|
|
|
|
|
for {
|
|
|
|
|
logger <- Slf4jLogger.create[IO]
|
|
|
|
|
service = new DataService(logger)
|
|
|
|
|
_ <- service.init
|
|
|
|
|
} yield service
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
class DataService(log: Logger[IO]) {
|
|
|
|
|
val BASE_FOLDER = "data"
|
|
|
|
|
val GRIB_FOLDER = s"$BASE_FOLDER/grib"
|
|
|
|
|
|
|
|
|
|
def init: IO[Unit] = {
|
|
|
|
|
for {
|
|
|
|
|
_ <- fs2.io.file.Files[IO].createDirectories(fs2.io.file.Path(BASE_FOLDER))
|
|
|
|
|
_ <- fs2.io.file.Files[IO].createDirectories(fs2.io.file.Path(GRIB_FOLDER))
|
|
|
|
|
_ <- log.info(s"Created directories: $BASE_FOLDER and $GRIB_FOLDER")
|
|
|
|
|
} yield ()
|
|
|
|
|
}
|
2025-02-08 23:34:50 +02:00
|
|
|
|
|
|
|
|
def getFileList(): IO[List[String]] =
|
|
|
|
|
Files[IO]
|
2025-02-13 16:11:50 +02:00
|
|
|
.list(Path(GRIB_FOLDER))
|
2025-02-08 23:34:50 +02:00
|
|
|
.map(_.toString)
|
|
|
|
|
.filter(_.endsWith(".grib"))
|
2025-02-13 16:11:50 +02:00
|
|
|
.map(_.replace(s"$GRIB_FOLDER/", ""))
|
2025-02-08 23:34:50 +02:00
|
|
|
.compile
|
|
|
|
|
.toList
|
|
|
|
|
|
|
|
|
|
def getGribStucture(fileName: String): IO[List[Grib]] = {
|
2025-02-13 16:11:50 +02:00
|
|
|
val filePath = Path(s"$GRIB_FOLDER/$fileName")
|
2025-02-08 23:34:50 +02:00
|
|
|
|
|
|
|
|
Files[IO].exists(filePath).flatMap {
|
|
|
|
|
case true => GribParser.parseFile(filePath)
|
|
|
|
|
case false => IO.raiseError(new Exception(s"File not found: $fileName"))
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
def getBinaryChunk(offset: Int, length: Int, fileName: String): IO[Array[Byte]] = {
|
|
|
|
|
IO {
|
2025-02-13 16:11:50 +02:00
|
|
|
val source = Source.fromFile(s"$GRIB_FOLDER/$fileName", "ISO-8859-1")
|
2025-02-08 23:34:50 +02:00
|
|
|
try {
|
|
|
|
|
source.slice(offset, offset + length).map(_.toByte).toArray
|
|
|
|
|
} finally {
|
|
|
|
|
source.close()
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
def getForecasts(): IO[List[(ZonedDateTime, ZonedDateTime)]] = {
|
|
|
|
|
getFileList()
|
|
|
|
|
.map { fileNameList =>
|
|
|
|
|
fileNameList.flatMap(getTimeFromName)
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2025-02-11 11:33:27 +02:00
|
|
|
def deleteOldForecasts(maxHours: Int = 9): IO[List[String]] = {
|
|
|
|
|
val nowUTC = ZonedDateTime.now(ZoneOffset.UTC)
|
|
|
|
|
val oldThreshold = nowUTC.minusHours(maxHours)
|
|
|
|
|
|
|
|
|
|
for {
|
2025-02-13 16:11:50 +02:00
|
|
|
_ <- log.info("start cleanup")
|
2025-02-11 11:33:27 +02:00
|
|
|
fileList <- getFileList()
|
|
|
|
|
fileDateList = fileList.flatMap(fileName =>
|
|
|
|
|
getTimeFromName(fileName).map(extracted => (fileName, extracted._1))
|
|
|
|
|
)
|
|
|
|
|
deleteList = fileDateList.filter(_._2.isBefore(oldThreshold)).map(_._1)
|
2025-02-13 16:11:50 +02:00
|
|
|
_ <- deleteList.traverse(name => Files[IO].delete(Path(s"$GRIB_FOLDER/${name}")))
|
|
|
|
|
_ <- deleteList.traverse(name => log.info(s"delete: $name"))
|
2025-02-11 11:33:27 +02:00
|
|
|
} yield deleteList
|
|
|
|
|
}
|
|
|
|
|
|
2025-02-08 23:34:50 +02:00
|
|
|
private def getTimeFromName(filename: String): Option[(ZonedDateTime, ZonedDateTime)] = {
|
|
|
|
|
Try {
|
|
|
|
|
val formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HHmm'Z'").withZone(ZoneId.of("UTC"))
|
|
|
|
|
val start = filename.slice(9, 25) // gets "2025-02-05T1500Z"
|
|
|
|
|
val end = filename.slice(26, 42) // gets "2025-02-05T1800Z"
|
|
|
|
|
(
|
2025-02-10 23:43:58 +02:00
|
|
|
ZonedDateTime.parse(start, formatter).toInstant.atZone(ZoneOffset.UTC),
|
|
|
|
|
ZonedDateTime.parse(end, formatter).toInstant.atZone(ZoneOffset.UTC)
|
2025-02-08 23:34:50 +02:00
|
|
|
)
|
|
|
|
|
}.toOption
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
}
|