From b44364d9a6d534c0cb0a9f28d4fe915b8db593c5 Mon Sep 17 00:00:00 2001 From: Guntis Smaukstelis Date: Sun, 16 Mar 2025 15:38:11 +0200 Subject: [PATCH] Write fatal errors to logs, view logs, clear tmp folder --- src/main/scala/Main.scala | 31 +++++++++++++++++++++++++- src/main/scala/data/DataService.scala | 20 +++++++++++++++++ src/main/scala/server/DebugUtils.scala | 6 +++-- src/main/scala/server/Server.scala | 20 +++++++++++++++++ 4 files changed, 74 insertions(+), 3 deletions(-) diff --git a/src/main/scala/Main.scala b/src/main/scala/Main.scala index f49317e..711b537 100644 --- a/src/main/scala/Main.scala +++ b/src/main/scala/Main.scala @@ -1,4 +1,5 @@ import cats.effect._ +import cats.effect.unsafe.implicits.global import cats.implicits.catsSyntaxTuple4Parallel import data.DataService import db.{DBConnection, PostgresService} @@ -6,6 +7,10 @@ import fetch.csv.{FetchService, FileNameService} import fetch.dmi import scheduler.Scheduler import server.Server + +import java.io.{BufferedWriter, FileWriter} +import java.time.LocalDateTime +import java.time.format.DateTimeFormatter import scala.concurrent.duration.DurationInt object Main extends IOApp { @@ -30,7 +35,7 @@ object Main extends IOApp { cleanupTask = scheduler.scheduleTask("Cleanup", List(1), dataService.deleteOldForecasts()).compile.drain fetchGrib <- dmi.FetchService.of(dataService) - fetchGribTask = scheduler.scheduleTask("Fetch Grib", List(2), fetchGrib.fetchRecentForecasts()).compile.drain + fetchGribTask = scheduler.scheduleTask("Fetch Grib", List(3), fetchGrib.fetchRecentForecasts()).compile.drain server <- Server.of(postgresService, dataService, fetchService) serverTask = server.run @@ -41,8 +46,32 @@ object Main extends IOApp { program.handleErrorWith { error => IO.delay { println(s"Fatal error occurred: ${error.getMessage}") + writeErrorToFile(error) error.printStackTrace() } *> IO.sleep(5.seconds) *> run(args) } } + + private def writeErrorToFile (error: Throwable) { + val timestamp = LocalDateTime.now().format(DateTimeFormatter.ofPattern("yyyy-MM-dd_HHmmss")) + val logPath = s"data/tmp/error_$timestamp.txt" + + fileWriterResource(logPath).use { writer => + IO(writer.write(formatError(error))) >> + IO(writer.flush()) + }.unsafeRunSync() + } + + private def formatError(error: Throwable): String = { + val timestamp = LocalDateTime.now().format(DateTimeFormatter.ISO_LOCAL_DATE_TIME) + val stackTrace = error.getStackTrace.mkString("\n ", "\n ", "") + s"[$timestamp] ERROR: ${error.getClass.getName}: ${error.getMessage}$stackTrace\n" + } + + private def fileWriterResource(path: String): Resource[IO, BufferedWriter] = + Resource.make { + IO(new BufferedWriter(new FileWriter(path, true))) // append mode + } { writer => + IO(writer.close()).handleErrorWith(e => IO(e.printStackTrace())) + } } \ No newline at end of file diff --git a/src/main/scala/data/DataService.scala b/src/main/scala/data/DataService.scala index d15b9ba..33dd137 100644 --- a/src/main/scala/data/DataService.scala +++ b/src/main/scala/data/DataService.scala @@ -120,6 +120,7 @@ class DataService(log: Logger[IO]) { for { _ <- log.info("start cleanup") +// _ <- IO.raiseError[Unit](new RuntimeException("Test error in deleteOldForecasts")) fileList <- getFileList() fileDateList = fileList.flatMap(fileName => getTimeFromName(fileName).map(extracted => (fileName, extracted._1)) @@ -137,6 +138,25 @@ class DataService(log: Logger[IO]) { } yield results } + def deleteTmp(): IO[List[DeletionResult]] = { + for { + fileList <- Files[IO] + .list(Path(TMP_FOLDER)) + .map(_.toString) + .map(_.replace(s"$TMP_FOLDER/", "")) + .compile.toList + + results <- fileList.traverse { name => + val path = Path(s"$TMP_FOLDER/${name}") + Files[IO].delete(path).attempt.flatMap { + case Right(_) => log.info(s"delete: $name").as(DeletionResult(name, true, None)) + case Left(error) => log.error(s"Failed to delete $name: ${error.getMessage}") + .as(DeletionResult(name, false, Some(error.getMessage))) + } + } + } yield results + } + private def getTimeFromName(filename: String): Option[(ZonedDateTime, ZonedDateTime)] = { Try { val formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HHmm'Z'").withZone(ZoneId.of("UTC")) diff --git a/src/main/scala/server/DebugUtils.scala b/src/main/scala/server/DebugUtils.scala index 83a3640..73d773c 100644 --- a/src/main/scala/server/DebugUtils.scala +++ b/src/main/scala/server/DebugUtils.scala @@ -10,12 +10,12 @@ object DebugUtils { case class FolderStructureResponse( current: DirectoryStructure, data: DirectoryStructure, + tmp: DirectoryStructure, grib: DirectoryStructure ) private def getDirectoryStructure(path: Path): IO[DirectoryStructure] = { for { - _ <- IO.println(path) absolutePath <- IO(path.toString) files <- Files[IO].list(path).compile.toList fileInfos <- files.traverse { filePath => @@ -29,13 +29,15 @@ object DebugUtils { def getFolderStructure: IO[FolderStructureResponse] = { val currentPath = Path(".") val dataPath = Path("data") + val tmpPath = Path("data/tmp") val gribPath = Path("data/grib") for { current <- getDirectoryStructure(currentPath) data <- getDirectoryStructure(dataPath) + tmp <- getDirectoryStructure(tmpPath) grib <- getDirectoryStructure(gribPath) - response = FolderStructureResponse(current, data, grib) + response = FolderStructureResponse(current, data, tmp, grib) } yield response } } \ No newline at end of file diff --git a/src/main/scala/server/Server.scala b/src/main/scala/server/Server.scala index eac9e56..8bd5b42 100644 --- a/src/main/scala/server/Server.scala +++ b/src/main/scala/server/Server.scala @@ -7,6 +7,7 @@ import data.DataService import db.PostgresService import fetch.csv.FetchService import fetch.lvgmc +import fs2.io.file.{Files, Path} import server.ValidateRoutes.{AggFieldList, AggKey, CityList, DateTimeRange, Granularity, ValidateDate, ValidateDateTime, ValidateInt, ValidateMonths, ValidateZonedDateTime} import io.circe.{Encoder, Json, Printer} import org.http4s._ @@ -74,6 +75,10 @@ class Server(postgresService: PostgresService, dataService: DataService, fetch: case GET -> Root / "grib" / "delete-old-forecasts" => dataService.deleteOldForecasts().flatMap(result => Ok(result.asJson)) + // http://0.0.0.0:8080/api/debug/delete-tmp + case GET -> Root / "debug" / "delete-tmp" => + dataService.deleteTmp().flatMap(result => Ok(result.asJson)) + // http://0.0.0.0:8080/api/debug/time case GET -> Root / "debug" / "time" => val nowUTC = ZonedDateTime.now(ZoneOffset.UTC) @@ -84,6 +89,21 @@ class Server(postgresService: PostgresService, dataService: DataService, fetch: case GET -> Root / "debug" / "folder-structure" => DebugUtils.getFolderStructure.flatMap(response => Ok(response.asJson)) + // http://0.0.0.0:8080/api/debug/file/error_2025-03-16_152201.txt + case GET -> Root / "debug" / "file" / fileName => + val filePath = Path(s"data/tmp/$fileName") + + Files[IO].exists(filePath).flatMap { + case true => + Files[IO].readUtf8(filePath) + .compile + .string + .flatMap(content => Ok(content).map(_.withContentType(`Content-Type`(MediaType.text.plain, Charset.`UTF-8`)))) + .handleErrorWith(err => InternalServerError(s"Failed to read file: ${err.getMessage}")) + case false => + NotFound(s"File not found: $fileName") + } + // http://0.0.0.0:8080/api/query/city/Liepāja,Rēzekne/20230414_2200-20230501_1230/hour/tempMax/max case GET -> Root / "query" / "city" / CityList(cities) / DateTimeRange(from, to) / Granularity(granularity) / field / AggKey(key) => val userQuery = UserQuery(cities, field, key, granularity, from, to)