bot/WeatherTool
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Changed api to fetch manually only most recent files

Browse Source
This commit is contained in:
Guntis Smaukstelis
2025-11-04 22:58:53 +02:00
parent 56b85cbae2
commit 9ab4cfe3fe
3 changed files with 56 additions and 79 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/main/scala/Main.scala
+2 -3
View File
@@ -3,7 +3,7 @@ import cats.effect.unsafe.implicits.global
import cats.implicits.catsSyntaxTuple4Parallel
import data.DataService
import db.{DBConnection, PostgresService}
import fetch.csv.{FetchService, FileNameService}
import fetch.csv.{FileNameService}
import fetch.dmi
import fetch.lvgmc
import scheduler.Scheduler
@@ -23,7 +23,6 @@ object Main extends IOApp {
scheduler <- Scheduler.of
dataService <- DataService.of
fetchLegacyService <- FetchService.of
fetchLvgmcService <- lvgmc.FetchService.of
@@ -41,7 +40,7 @@ object Main extends IOApp {
fetchGrib <- dmi.FetchService.of(dataService)
fetchGribTask = scheduler.scheduleTask("Fetch Grib", List(43), fetchGrib.fetchRecentForecasts()).compile.drain
server <- Server.of(postgresService, dataService, fetchLegacyService)
server <- Server.of(postgresService, dataService, fetchLvgmcService)
serverTask = server.run
exitCode <- (serverTask, fetchStationsTask, cleanupTask, fetchGribTask).parMapN((_, _, _, _) => ExitCode.Success)
src/main/scala/server/Server.scala
+41 -26
View File
@@ -5,8 +5,9 @@ import cats.implicits.toTraverseOps
import com.comcast.ip4s.IpLiteralSyntax
import data.DataService
import db.PostgresService
import fetch.csv.FetchService
import fetch.lvgmc
import fetch.csv.FileNameService
//import fetch.csv.FetchService
import fetch.lvgmc.FetchService
import fs2.io.file.{Files, Path}
import server.ValidateRoutes.{AggFieldList, AggKey, CityList, DateTimeRange, Granularity, ValidateDate, ValidateDateTime, ValidateInt, ValidateMonths, ValidateZonedDateTime}
import io.circe.{Encoder, Json, Printer}
@@ -52,10 +53,24 @@ class Server(postgresService: PostgresService, dataService: DataService, fetch:
private val apiRoutes = HttpRoutes.of[IO] {
// http://0.0.0.0:8080/api/show/lvgmc-forecast/Latvija_LTV_pilsetas_tekosa_dn.csv
case GET -> Root / "show" / "lvgmc-forecast" / fileName =>
lvgmc.FetchService.of.flatMap(f => f.fetchFile(fileName)).flatMap(bytes =>
fetch.fetchFile(fileName).flatMap(bytes =>
Ok(bytes).map(_.withContentType(`Content-Type`(MediaType.text.csv)))
)
// http://0.0.0.0:8080/api/fetch/lvgmc/stations
case GET -> Root / "fetch" / "lvgmc" / "stations" =>
(
for {
fileName <- new FileNameService().generateCurrentHour
stationDataStr <- fetch.fetchWeatherStations()
_ <- postgresService.save(fileName, stationDataStr)
} yield stationDataStr
)
.flatMap(content => Ok(content))
.handleErrorWith(error =>
InternalServerError(s"Failed to fetch stations: ${error.getMessage}")
)
// http://0.0.0.0:8080/api/show/grib-all-structure
case GET -> Root / "show" / "grib-all-structure" =>
dataService.getAllFileStructure().flatMap(gribList => Ok(gribList.asJson))
@@ -121,29 +136,29 @@ class Server(postgresService: PostgresService, dataService: DataService, fetch:
postgresService.queryCountry(from, to, fieldList)
.flatMap(result => Ok(result.asJson.pretty))
// http://0.0.0.0:8080/api/fetch/date/20230514
case GET -> Root / "fetch" / "date" / ValidateDate(date) =>
val result = for {
fetchResultEither <- fetch.fetchFromDate(date).attempt
fetchServiceError = fetchResultEither.left.toOption.map(e => s"FetchServiceError: ${e.getMessage}").toList
fetchResult = fetchResultEither.getOrElse(List.empty)
(fetchErrors, successDownloads) = fetchResult.partitionMap(identity)
_ <- log.info(s"FETCHED SUCCESSFULLY files: ${successDownloads.size}")
saveResults <- successDownloads.traverse { case (name, content) => postgresService.save(name, content).attempt }
(saveErrors, successSaves) = saveResults.partitionMap(identity)
// successes = successDownloads.map(s => s"fetched: ${s._1}") ++ successSaves.map(s => s"saved: $s")
successes = successSaves
errors = fetchServiceError ++ fetchErrors.map(e => s"FetchError: ${e.getMessage}") ++ saveErrors.map(e => s"SaveError: ${e.getMessage}")
_ <- log.error(s"errors: $errors")
_ <- log.info(s"successes: $successes")
} yield (successes, errors)
result.flatMap { case (successes, errors) =>
Ok(Json.obj(
"errors" -> errors.asJson,
"successes" -> successes.asJson
).pretty)
}
// // http://0.0.0.0:8080/api/fetch/date/20230514
// case GET -> Root / "fetch" / "date" / ValidateDate(date) =>
// val result = for {
// fetchResultEither <- fetch.fetchFromDate(date).attempt
// fetchServiceError = fetchResultEither.left.toOption.map(e => s"FetchServiceError: ${e.getMessage}").toList
// fetchResult = fetchResultEither.getOrElse(List.empty)
// (fetchErrors, successDownloads) = fetchResult.partitionMap(identity)
// _ <- log.info(s"FETCHED SUCCESSFULLY files: ${successDownloads.size}")
// saveResults <- successDownloads.traverse { case (name, content) => postgresService.save(name, content).attempt }
// (saveErrors, successSaves) = saveResults.partitionMap(identity)
//// successes = successDownloads.map(s => s"fetched: ${s._1}") ++ successSaves.map(s => s"saved: $s")
// successes = successSaves
// errors = fetchServiceError ++ fetchErrors.map(e => s"FetchError: ${e.getMessage}") ++ saveErrors.map(e => s"SaveError: ${e.getMessage}")
// _ <- log.error(s"errors: $errors")
// _ <- log.info(s"successes: $successes")
// } yield (successes, errors)
//
// result.flatMap { case (successes, errors) =>
// Ok(Json.obj(
// "errors" -> errors.asJson,
// "successes" -> successes.asJson
// ).pretty)
// }
// http://0.0.0.0:8080/api/show/months/202304,202305,202306
case GET -> Root / "show" / "months" / ValidateMonths(monthList) =>
web/src/pages/database/FetchFiles.tsx
+7 -44
View File
@@ -3,70 +3,33 @@ import { createSignal } from "solid-js";
import { apiHost } from "../../consts";
export function FetchFiles() {
const [getStart, setStart] = createSignal(new Date());
const [getEnd, setEnd] = createSignal(new Date());
const [getFetchResult, setFetchResult] = createSignal("");
const [isSpinner, setIsSpinner] = createSignal(false);
const startStr = () => moment(getStart()).format("YYYY-MM-DD");
const endStr = () => moment(getEnd()).format("YYYY-MM-DD");
function handleSubmit(e: MouseEvent) {
async function handleSubmit(e: MouseEvent) {
e.preventDefault();
const dates = getDatesBetween(getStart(), getEnd());
timerFetch(dates);
}
async function timerFetch(dates: Date[]) {
setIsSpinner(true);
for (const date of dates) {
const response = await fetch(`${apiHost}/api/fetch/date/${moment(date).format("YYYYMMDD")}`);
const response = await fetch(`${apiHost}/api/fetch/lvgmc/stations`);
const text = await response.text();
setFetchResult(text);
await new Promise(resolve => setTimeout(resolve, 200));
}
setIsSpinner(false);
setIsSpinner(false)
}
return (
<form>
<h3>Date range fetch</h3>
<h3>Manual fetch</h3>
<p>
<input
type="date"
value={ startStr() }
onChange={e => setStart(new Date(e.target.value))}
/> start
</p>
<p>
<input
type="date"
value={ endStr() }
onChange={e => setEnd(new Date(e.target.value))}
/> end
</p>
<p>
<input type="submit" onClick={handleSubmit} value="Fetch .csv files" />
<input type="submit" onClick={handleSubmit} value="Fetch most recent .csv file" />
</p>
<p>result:</p>
{ isSpinner() && <div>
{isSpinner() && <div>
<span class="spinner"></span>
<span style={{ "padding-left": "16px" }}>Fetching csv files</span>
</div>
}
<p>{ getFetchResult() }</p>
<p>{getFetchResult()}</p>
</form>
)
}
function getDatesBetween(startDate: Date, endDate: Date): Date[] {
const dates: Date[] = [];
let currentDate = new Date(startDate); // start from the start date
while (currentDate <= endDate) {
dates.push(new Date(currentDate)); // add current date to the list
currentDate.setDate(currentDate.getDate() + 1); // increment the date
}
return dates;
}