From 9ab4cfe3fe10d0e970e644ab7e5a0c3ae89c6e03 Mon Sep 17 00:00:00 2001 From: Guntis Smaukstelis Date: Tue, 4 Nov 2025 22:58:53 +0200 Subject: [PATCH] Changed api to fetch manually only most recent files --- src/main/scala/Main.scala | 5 +- src/main/scala/server/Server.scala | 67 ++++++++++++++++----------- web/src/pages/database/FetchFiles.tsx | 63 ++++++------------------- 3 files changed, 56 insertions(+), 79 deletions(-) diff --git a/src/main/scala/Main.scala b/src/main/scala/Main.scala index d1167eb..83352e3 100644 --- a/src/main/scala/Main.scala +++ b/src/main/scala/Main.scala @@ -3,7 +3,7 @@ import cats.effect.unsafe.implicits.global import cats.implicits.catsSyntaxTuple4Parallel import data.DataService import db.{DBConnection, PostgresService} -import fetch.csv.{FetchService, FileNameService} +import fetch.csv.{FileNameService} import fetch.dmi import fetch.lvgmc import scheduler.Scheduler @@ -23,7 +23,6 @@ object Main extends IOApp { scheduler <- Scheduler.of dataService <- DataService.of - fetchLegacyService <- FetchService.of fetchLvgmcService <- lvgmc.FetchService.of @@ -41,7 +40,7 @@ object Main extends IOApp { fetchGrib <- dmi.FetchService.of(dataService) fetchGribTask = scheduler.scheduleTask("Fetch Grib", List(43), fetchGrib.fetchRecentForecasts()).compile.drain - server <- Server.of(postgresService, dataService, fetchLegacyService) + server <- Server.of(postgresService, dataService, fetchLvgmcService) serverTask = server.run exitCode <- (serverTask, fetchStationsTask, cleanupTask, fetchGribTask).parMapN((_, _, _, _) => ExitCode.Success) diff --git a/src/main/scala/server/Server.scala b/src/main/scala/server/Server.scala index 8bd5b42..75fcaac 100644 --- a/src/main/scala/server/Server.scala +++ b/src/main/scala/server/Server.scala @@ -5,8 +5,9 @@ import cats.implicits.toTraverseOps import com.comcast.ip4s.IpLiteralSyntax import data.DataService import db.PostgresService -import fetch.csv.FetchService -import fetch.lvgmc +import fetch.csv.FileNameService +//import fetch.csv.FetchService +import fetch.lvgmc.FetchService import fs2.io.file.{Files, Path} import server.ValidateRoutes.{AggFieldList, AggKey, CityList, DateTimeRange, Granularity, ValidateDate, ValidateDateTime, ValidateInt, ValidateMonths, ValidateZonedDateTime} import io.circe.{Encoder, Json, Printer} @@ -52,10 +53,24 @@ class Server(postgresService: PostgresService, dataService: DataService, fetch: private val apiRoutes = HttpRoutes.of[IO] { // http://0.0.0.0:8080/api/show/lvgmc-forecast/Latvija_LTV_pilsetas_tekosa_dn.csv case GET -> Root / "show" / "lvgmc-forecast" / fileName => - lvgmc.FetchService.of.flatMap(f => f.fetchFile(fileName)).flatMap(bytes => + fetch.fetchFile(fileName).flatMap(bytes => Ok(bytes).map(_.withContentType(`Content-Type`(MediaType.text.csv))) ) + // http://0.0.0.0:8080/api/fetch/lvgmc/stations + case GET -> Root / "fetch" / "lvgmc" / "stations" => + ( + for { + fileName <- new FileNameService().generateCurrentHour + stationDataStr <- fetch.fetchWeatherStations() + _ <- postgresService.save(fileName, stationDataStr) + } yield stationDataStr + ) + .flatMap(content => Ok(content)) + .handleErrorWith(error => + InternalServerError(s"Failed to fetch stations: ${error.getMessage}") + ) + // http://0.0.0.0:8080/api/show/grib-all-structure case GET -> Root / "show" / "grib-all-structure" => dataService.getAllFileStructure().flatMap(gribList => Ok(gribList.asJson)) @@ -121,29 +136,29 @@ class Server(postgresService: PostgresService, dataService: DataService, fetch: postgresService.queryCountry(from, to, fieldList) .flatMap(result => Ok(result.asJson.pretty)) - // http://0.0.0.0:8080/api/fetch/date/20230514 - case GET -> Root / "fetch" / "date" / ValidateDate(date) => - val result = for { - fetchResultEither <- fetch.fetchFromDate(date).attempt - fetchServiceError = fetchResultEither.left.toOption.map(e => s"FetchServiceError: ${e.getMessage}").toList - fetchResult = fetchResultEither.getOrElse(List.empty) - (fetchErrors, successDownloads) = fetchResult.partitionMap(identity) - _ <- log.info(s"FETCHED SUCCESSFULLY files: ${successDownloads.size}") - saveResults <- successDownloads.traverse { case (name, content) => postgresService.save(name, content).attempt } - (saveErrors, successSaves) = saveResults.partitionMap(identity) -// successes = successDownloads.map(s => s"fetched: ${s._1}") ++ successSaves.map(s => s"saved: $s") - successes = successSaves - errors = fetchServiceError ++ fetchErrors.map(e => s"FetchError: ${e.getMessage}") ++ saveErrors.map(e => s"SaveError: ${e.getMessage}") - _ <- log.error(s"errors: $errors") - _ <- log.info(s"successes: $successes") - } yield (successes, errors) - - result.flatMap { case (successes, errors) => - Ok(Json.obj( - "errors" -> errors.asJson, - "successes" -> successes.asJson - ).pretty) - } +// // http://0.0.0.0:8080/api/fetch/date/20230514 +// case GET -> Root / "fetch" / "date" / ValidateDate(date) => +// val result = for { +// fetchResultEither <- fetch.fetchFromDate(date).attempt +// fetchServiceError = fetchResultEither.left.toOption.map(e => s"FetchServiceError: ${e.getMessage}").toList +// fetchResult = fetchResultEither.getOrElse(List.empty) +// (fetchErrors, successDownloads) = fetchResult.partitionMap(identity) +// _ <- log.info(s"FETCHED SUCCESSFULLY files: ${successDownloads.size}") +// saveResults <- successDownloads.traverse { case (name, content) => postgresService.save(name, content).attempt } +// (saveErrors, successSaves) = saveResults.partitionMap(identity) +//// successes = successDownloads.map(s => s"fetched: ${s._1}") ++ successSaves.map(s => s"saved: $s") +// successes = successSaves +// errors = fetchServiceError ++ fetchErrors.map(e => s"FetchError: ${e.getMessage}") ++ saveErrors.map(e => s"SaveError: ${e.getMessage}") +// _ <- log.error(s"errors: $errors") +// _ <- log.info(s"successes: $successes") +// } yield (successes, errors) +// +// result.flatMap { case (successes, errors) => +// Ok(Json.obj( +// "errors" -> errors.asJson, +// "successes" -> successes.asJson +// ).pretty) +// } // http://0.0.0.0:8080/api/show/months/202304,202305,202306 case GET -> Root / "show" / "months" / ValidateMonths(monthList) => diff --git a/web/src/pages/database/FetchFiles.tsx b/web/src/pages/database/FetchFiles.tsx index 89237d1..3c3f1b0 100644 --- a/web/src/pages/database/FetchFiles.tsx +++ b/web/src/pages/database/FetchFiles.tsx @@ -3,70 +3,33 @@ import { createSignal } from "solid-js"; import { apiHost } from "../../consts"; export function FetchFiles() { - const [getStart, setStart] = createSignal(new Date()); - const [getEnd, setEnd] = createSignal(new Date()); const [getFetchResult, setFetchResult] = createSignal(""); const [isSpinner, setIsSpinner] = createSignal(false); - const startStr = () => moment(getStart()).format("YYYY-MM-DD"); - const endStr = () => moment(getEnd()).format("YYYY-MM-DD"); - - function handleSubmit(e: MouseEvent) { + async function handleSubmit(e: MouseEvent) { e.preventDefault(); - const dates = getDatesBetween(getStart(), getEnd()); - timerFetch(dates); - } - - async function timerFetch(dates: Date[]) { setIsSpinner(true); - for (const date of dates) { - const response = await fetch(`${apiHost}/api/fetch/date/${moment(date).format("YYYYMMDD")}`); - const text = await response.text(); - setFetchResult(text); - await new Promise(resolve => setTimeout(resolve, 200)); - } - setIsSpinner(false); + const response = await fetch(`${apiHost}/api/fetch/lvgmc/stations`); + const text = await response.text(); + setFetchResult(text); + await new Promise(resolve => setTimeout(resolve, 200)); + setIsSpinner(false) } return (
-

Date range fetch

+

Manual fetch

- setStart(new Date(e.target.value))} - /> start -

-

- setEnd(new Date(e.target.value))} - /> end -

-

- +

result:

- { isSpinner() &&
- - Fetching csv files -
+ {isSpinner() &&
+ + Fetching csv files +
} -

{ getFetchResult() }

+

{getFetchResult()}

) } -function getDatesBetween(startDate: Date, endDate: Date): Date[] { - const dates: Date[] = []; - let currentDate = new Date(startDate); // start from the start date - - while (currentDate <= endDate) { - dates.push(new Date(currentDate)); // add current date to the list - currentDate.setDate(currentDate.getDate() + 1); // increment the date - } - - return dates; -} \ No newline at end of file