Write fatal errors to logs, view logs, clear tmp folder

src/main/scala/Main.scala

@@ -1,4 +1,5 @@
 import cats.effect._
+import cats.effect.unsafe.implicits.global
 import cats.implicits.catsSyntaxTuple4Parallel
 import data.DataService
 import db.{DBConnection, PostgresService}
@@ -6,6 +7,10 @@ import fetch.csv.{FetchService, FileNameService}
 import fetch.dmi
 import scheduler.Scheduler
 import server.Server
+
+import java.io.{BufferedWriter, FileWriter}
+import java.time.LocalDateTime
+import java.time.format.DateTimeFormatter
 import scala.concurrent.duration.DurationInt
 
 object Main extends IOApp {
@@ -30,7 +35,7 @@ object Main extends IOApp {
       cleanupTask = scheduler.scheduleTask("Cleanup", List(1), dataService.deleteOldForecasts()).compile.drain
 
       fetchGrib <- dmi.FetchService.of(dataService)
-      fetchGribTask = scheduler.scheduleTask("Fetch Grib", List(2), fetchGrib.fetchRecentForecasts()).compile.drain
+      fetchGribTask = scheduler.scheduleTask("Fetch Grib", List(3), fetchGrib.fetchRecentForecasts()).compile.drain
 
       server <- Server.of(postgresService, dataService, fetchService)
       serverTask = server.run
@@ -41,8 +46,32 @@ object Main extends IOApp {
     program.handleErrorWith { error =>
       IO.delay {
         println(s"Fatal error occurred: ${error.getMessage}")
+        writeErrorToFile(error)
         error.printStackTrace()
       } *> IO.sleep(5.seconds) *> run(args)
     }
   }
+
+  private def writeErrorToFile (error: Throwable) {
+    val timestamp = LocalDateTime.now().format(DateTimeFormatter.ofPattern("yyyy-MM-dd_HHmmss"))
+    val logPath = s"data/tmp/error_$timestamp.txt"
+
+    fileWriterResource(logPath).use { writer =>
+      IO(writer.write(formatError(error))) >>
+        IO(writer.flush())
+    }.unsafeRunSync()
+  }
+
+  private def formatError(error: Throwable): String = {
+    val timestamp = LocalDateTime.now().format(DateTimeFormatter.ISO_LOCAL_DATE_TIME)
+    val stackTrace = error.getStackTrace.mkString("\n  ", "\n  ", "")
+    s"[$timestamp] ERROR: ${error.getClass.getName}: ${error.getMessage}$stackTrace\n"
+  }
+
+  private def fileWriterResource(path: String): Resource[IO, BufferedWriter] =
+    Resource.make {
+      IO(new BufferedWriter(new FileWriter(path, true))) // append mode
+    } { writer =>
+      IO(writer.close()).handleErrorWith(e => IO(e.printStackTrace()))
+    }
 }

src/main/scala/data/DataService.scala

@@ -120,6 +120,7 @@ class DataService(log: Logger[IO]) {
 
     for {
       _ <- log.info("start cleanup")
+//      _ <- IO.raiseError[Unit](new RuntimeException("Test error in deleteOldForecasts"))
       fileList <- getFileList()
       fileDateList = fileList.flatMap(fileName =>
         getTimeFromName(fileName).map(extracted => (fileName, extracted._1))
@@ -137,6 +138,25 @@ class DataService(log: Logger[IO]) {
     } yield results
   }
 
+  def deleteTmp(): IO[List[DeletionResult]] = {
+    for {
+      fileList <- Files[IO]
+        .list(Path(TMP_FOLDER))
+        .map(_.toString)
+        .map(_.replace(s"$TMP_FOLDER/", ""))
+        .compile.toList
+
+      results <- fileList.traverse { name =>
+        val path = Path(s"$TMP_FOLDER/${name}")
+        Files[IO].delete(path).attempt.flatMap {
+          case Right(_) => log.info(s"delete: $name").as(DeletionResult(name, true, None))
+          case Left(error) => log.error(s"Failed to delete $name: ${error.getMessage}")
+            .as(DeletionResult(name, false, Some(error.getMessage)))
+        }
+      }
+    } yield results
+  }
+
   private def getTimeFromName(filename: String): Option[(ZonedDateTime, ZonedDateTime)] = {
     Try {
       val formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HHmm'Z'").withZone(ZoneId.of("UTC"))

src/main/scala/server/DebugUtils.scala

@@ -10,12 +10,12 @@ object DebugUtils {
   case class FolderStructureResponse(
     current: DirectoryStructure,
     data: DirectoryStructure,
+    tmp: DirectoryStructure,
     grib: DirectoryStructure
   )
 
   private def getDirectoryStructure(path: Path): IO[DirectoryStructure] = {
     for {
-      _ <- IO.println(path)
       absolutePath <- IO(path.toString)
       files <- Files[IO].list(path).compile.toList
       fileInfos <- files.traverse { filePath =>
@@ -29,13 +29,15 @@ object DebugUtils {
   def getFolderStructure: IO[FolderStructureResponse] = {
     val currentPath = Path(".")
     val dataPath = Path("data")
+    val tmpPath = Path("data/tmp")
     val gribPath = Path("data/grib")
 
     for {
       current <- getDirectoryStructure(currentPath)
       data <- getDirectoryStructure(dataPath)
+      tmp <- getDirectoryStructure(tmpPath)
       grib <- getDirectoryStructure(gribPath)
-      response = FolderStructureResponse(current, data, grib)
+      response = FolderStructureResponse(current, data, tmp, grib)
     } yield response
   }
 }

src/main/scala/server/Server.scala

@@ -7,6 +7,7 @@ import data.DataService
 import db.PostgresService
 import fetch.csv.FetchService
 import fetch.lvgmc
+import fs2.io.file.{Files, Path}
 import server.ValidateRoutes.{AggFieldList, AggKey, CityList, DateTimeRange, Granularity, ValidateDate, ValidateDateTime, ValidateInt, ValidateMonths, ValidateZonedDateTime}
 import io.circe.{Encoder, Json, Printer}
 import org.http4s._
@@ -74,6 +75,10 @@ class Server(postgresService: PostgresService, dataService: DataService, fetch:
     case GET -> Root / "grib" / "delete-old-forecasts" =>
       dataService.deleteOldForecasts().flatMap(result => Ok(result.asJson))
 
+    // http://0.0.0.0:8080/api/debug/delete-tmp
+    case GET -> Root / "debug" / "delete-tmp" =>
+      dataService.deleteTmp().flatMap(result => Ok(result.asJson))
+
     // http://0.0.0.0:8080/api/debug/time
     case GET -> Root / "debug" / "time" =>
       val nowUTC = ZonedDateTime.now(ZoneOffset.UTC)
@@ -84,6 +89,21 @@ class Server(postgresService: PostgresService, dataService: DataService, fetch:
     case GET -> Root / "debug" / "folder-structure" =>
       DebugUtils.getFolderStructure.flatMap(response => Ok(response.asJson))
 
+    // http://0.0.0.0:8080/api/debug/file/error_2025-03-16_152201.txt
+    case GET -> Root / "debug" / "file" / fileName =>
+      val filePath = Path(s"data/tmp/$fileName")
+
+      Files[IO].exists(filePath).flatMap {
+        case true =>
+          Files[IO].readUtf8(filePath)
+            .compile
+            .string
+            .flatMap(content => Ok(content).map(_.withContentType(`Content-Type`(MediaType.text.plain, Charset.`UTF-8`))))
+            .handleErrorWith(err => InternalServerError(s"Failed to read file: ${err.getMessage}"))
+        case false =>
+          NotFound(s"File not found: $fileName")
+      }
+
     // http://0.0.0.0:8080/api/query/city/Liepāja,Rēzekne/20230414_2200-20230501_1230/hour/tempMax/max
     case GET -> Root / "query" / "city" / CityList(cities) / DateTimeRange(from, to) / Granularity(granularity) / field / AggKey(key) =>
       val userQuery = UserQuery(cities, field, key, granularity, from, to)