Changed api to fetch manually only most recent files

src/main/scala/Main.scala

@@ -3,7 +3,7 @@ import cats.effect.unsafe.implicits.global
 import cats.implicits.catsSyntaxTuple4Parallel
 import data.DataService
 import db.{DBConnection, PostgresService}
-import fetch.csv.{FetchService, FileNameService}
+import fetch.csv.{FileNameService}
 import fetch.dmi
 import fetch.lvgmc
 import scheduler.Scheduler
@@ -23,7 +23,6 @@ object Main extends IOApp {
 
       scheduler <- Scheduler.of
       dataService <- DataService.of
-      fetchLegacyService <- FetchService.of
       fetchLvgmcService <- lvgmc.FetchService.of
 
 
@@ -41,7 +40,7 @@ object Main extends IOApp {
       fetchGrib <- dmi.FetchService.of(dataService)
       fetchGribTask = scheduler.scheduleTask("Fetch Grib", List(43), fetchGrib.fetchRecentForecasts()).compile.drain
 
-      server <- Server.of(postgresService, dataService, fetchLegacyService)
+      server <- Server.of(postgresService, dataService, fetchLvgmcService)
       serverTask = server.run
 
       exitCode <- (serverTask, fetchStationsTask, cleanupTask, fetchGribTask).parMapN((_, _, _, _) => ExitCode.Success)

src/main/scala/server/Server.scala

@@ -5,8 +5,9 @@ import cats.implicits.toTraverseOps
 import com.comcast.ip4s.IpLiteralSyntax
 import data.DataService
 import db.PostgresService
-import fetch.csv.FetchService
+import fetch.csv.FileNameService
-import fetch.lvgmc
+//import fetch.csv.FetchService
+import fetch.lvgmc.FetchService
 import fs2.io.file.{Files, Path}
 import server.ValidateRoutes.{AggFieldList, AggKey, CityList, DateTimeRange, Granularity, ValidateDate, ValidateDateTime, ValidateInt, ValidateMonths, ValidateZonedDateTime}
 import io.circe.{Encoder, Json, Printer}
@@ -52,10 +53,24 @@ class Server(postgresService: PostgresService, dataService: DataService, fetch:
   private val apiRoutes = HttpRoutes.of[IO] {
     // http://0.0.0.0:8080/api/show/lvgmc-forecast/Latvija_LTV_pilsetas_tekosa_dn.csv
     case GET -> Root / "show" / "lvgmc-forecast" / fileName =>
-      lvgmc.FetchService.of.flatMap(f => f.fetchFile(fileName)).flatMap(bytes =>
+      fetch.fetchFile(fileName).flatMap(bytes =>
         Ok(bytes).map(_.withContentType(`Content-Type`(MediaType.text.csv)))
       )
 
+    // http://0.0.0.0:8080/api/fetch/lvgmc/stations
+    case GET -> Root / "fetch" / "lvgmc" / "stations" =>
+      (
+        for {
+          fileName <- new FileNameService().generateCurrentHour
+          stationDataStr <- fetch.fetchWeatherStations()
+          _ <- postgresService.save(fileName, stationDataStr)
+        } yield stationDataStr
+      )
+        .flatMap(content => Ok(content))
+        .handleErrorWith(error =>
+          InternalServerError(s"Failed to fetch stations: ${error.getMessage}")
+        )
+
     // http://0.0.0.0:8080/api/show/grib-all-structure
     case GET -> Root / "show" / "grib-all-structure" =>
       dataService.getAllFileStructure().flatMap(gribList => Ok(gribList.asJson))
@@ -121,29 +136,29 @@ class Server(postgresService: PostgresService, dataService: DataService, fetch:
       postgresService.queryCountry(from, to, fieldList)
         .flatMap(result => Ok(result.asJson.pretty))
 
-    // http://0.0.0.0:8080/api/fetch/date/20230514
+//    // http://0.0.0.0:8080/api/fetch/date/20230514
-    case GET -> Root / "fetch" / "date" / ValidateDate(date) =>
+//    case GET -> Root / "fetch" / "date" / ValidateDate(date) =>
-      val result = for {
+//      val result = for {
-        fetchResultEither <- fetch.fetchFromDate(date).attempt
+//        fetchResultEither <- fetch.fetchFromDate(date).attempt
-        fetchServiceError = fetchResultEither.left.toOption.map(e => s"FetchServiceError: ${e.getMessage}").toList
+//        fetchServiceError = fetchResultEither.left.toOption.map(e => s"FetchServiceError: ${e.getMessage}").toList
-        fetchResult = fetchResultEither.getOrElse(List.empty)
+//        fetchResult = fetchResultEither.getOrElse(List.empty)
-        (fetchErrors, successDownloads) = fetchResult.partitionMap(identity)
+//        (fetchErrors, successDownloads) = fetchResult.partitionMap(identity)
-        _ <- log.info(s"FETCHED SUCCESSFULLY files: ${successDownloads.size}")
+//        _ <- log.info(s"FETCHED SUCCESSFULLY files: ${successDownloads.size}")
-        saveResults <- successDownloads.traverse { case (name, content) => postgresService.save(name, content).attempt }
+//        saveResults <- successDownloads.traverse { case (name, content) => postgresService.save(name, content).attempt }
-        (saveErrors, successSaves) = saveResults.partitionMap(identity)
+//        (saveErrors, successSaves) = saveResults.partitionMap(identity)
-//        successes = successDownloads.map(s => s"fetched: ${s._1}") ++ successSaves.map(s => s"saved: $s")
+////        successes = successDownloads.map(s => s"fetched: ${s._1}") ++ successSaves.map(s => s"saved: $s")
-        successes = successSaves
+//        successes = successSaves
-        errors = fetchServiceError ++ fetchErrors.map(e => s"FetchError: ${e.getMessage}") ++ saveErrors.map(e => s"SaveError: ${e.getMessage}")
+//        errors = fetchServiceError ++ fetchErrors.map(e => s"FetchError: ${e.getMessage}") ++ saveErrors.map(e => s"SaveError: ${e.getMessage}")
-        _ <- log.error(s"errors: $errors")
+//        _ <- log.error(s"errors: $errors")
-        _ <- log.info(s"successes: $successes")
+//        _ <- log.info(s"successes: $successes")
-      } yield (successes, errors)
+//      } yield (successes, errors)
-
+//
-      result.flatMap { case (successes, errors) =>
+//      result.flatMap { case (successes, errors) =>
-        Ok(Json.obj(
+//        Ok(Json.obj(
-            "errors" -> errors.asJson,
+//            "errors" -> errors.asJson,
-            "successes" -> successes.asJson
+//            "successes" -> successes.asJson
-          ).pretty)
+//          ).pretty)
-      }
+//      }
 
     // http://0.0.0.0:8080/api/show/months/202304,202305,202306
     case GET -> Root / "show" / "months" / ValidateMonths(monthList) =>

web/src/pages/database/FetchFiles.tsx

@@ -3,70 +3,33 @@ import { createSignal } from "solid-js";
 import { apiHost } from "../../consts";
 
 export function FetchFiles() {
-    const [getStart, setStart] = createSignal(new Date());
-    const [getEnd, setEnd] = createSignal(new Date());
     const [getFetchResult, setFetchResult] = createSignal("");
     const [isSpinner, setIsSpinner] = createSignal(false);
 
-    const startStr = () => moment(getStart()).format("YYYY-MM-DD");
+    async function handleSubmit(e: MouseEvent) {
-    const endStr = () => moment(getEnd()).format("YYYY-MM-DD");
-
-    function handleSubmit(e: MouseEvent) {
         e.preventDefault();
-        const dates = getDatesBetween(getStart(), getEnd());
-        timerFetch(dates);
-    }
-
-    async function timerFetch(dates: Date[]) {
         setIsSpinner(true);
-        for (const date of dates) {
+        const response = await fetch(`${apiHost}/api/fetch/lvgmc/stations`);
-            const response = await fetch(`${apiHost}/api/fetch/date/${moment(date).format("YYYYMMDD")}`);
+        const text = await response.text();
-            const text = await response.text();
+        setFetchResult(text);
-            setFetchResult(text);
+        await new Promise(resolve => setTimeout(resolve, 200));
-            await new Promise(resolve => setTimeout(resolve, 200));
+        setIsSpinner(false)
-        }
-        setIsSpinner(false);
     }
 
     return (
         <form>
-            <h3>Date range fetch</h3>
+            <h3>Manual fetch</h3>
             <p>
-                <input
+                <input type="submit" onClick={handleSubmit} value="Fetch most recent .csv file" />
-                    type="date"
-                    value={ startStr() }
-                    onChange={e => setStart(new Date(e.target.value))}
-                /> start
-            </p>
-            <p>
-                <input
-                    type="date"
-                    value={ endStr() }
-                    onChange={e => setEnd(new Date(e.target.value))}
-                /> end
-            </p>
-            <p>
-                <input type="submit" onClick={handleSubmit} value="Fetch .csv files" />
             </p>
             <p>result:</p>
-            { isSpinner() && <div>
+            {isSpinner() && <div>
-                    <span class="spinner"></span>
+                <span class="spinner"></span>
-                    <span style={{ "padding-left": "16px" }}>Fetching csv files</span>
+                <span style={{ "padding-left": "16px" }}>Fetching csv files</span>
-                </div>
+            </div>
             }
-            <p>{ getFetchResult() }</p>
+            <p>{getFetchResult()}</p>
         </form>
     )
 }
 
-function getDatesBetween(startDate: Date, endDate: Date): Date[] {
-    const dates: Date[] = [];
-    let currentDate = new Date(startDate); // start from the start date
-
-    while (currentDate <= endDate) {
-        dates.push(new Date(currentDate)); // add current date to the list
-        currentDate.setDate(currentDate.getDate() + 1); // increment the date
-    }
-
-    return dates;
-}