Changed api to fetch manually only most recent files
This commit is contained in:
Guntis Smaukstelis
@@ -3,7 +3,7 @@ import cats.effect.unsafe.implicits.global
|
||||
import cats.implicits.catsSyntaxTuple4Parallel
|
||||
import data.DataService
|
||||
import db.{DBConnection, PostgresService}
|
||||
import fetch.csv.{FetchService, FileNameService}
|
||||
import fetch.csv.{FileNameService}
|
||||
import fetch.dmi
|
||||
import fetch.lvgmc
|
||||
import scheduler.Scheduler
|
||||
@@ -23,7 +23,6 @@ object Main extends IOApp {
|
||||
|
||||
scheduler <- Scheduler.of
|
||||
dataService <- DataService.of
|
||||
fetchLegacyService <- FetchService.of
|
||||
fetchLvgmcService <- lvgmc.FetchService.of
|
||||
|
||||
|
||||
@@ -41,7 +40,7 @@ object Main extends IOApp {
|
||||
fetchGrib <- dmi.FetchService.of(dataService)
|
||||
fetchGribTask = scheduler.scheduleTask("Fetch Grib", List(43), fetchGrib.fetchRecentForecasts()).compile.drain
|
||||
|
||||
server <- Server.of(postgresService, dataService, fetchLegacyService)
|
||||
server <- Server.of(postgresService, dataService, fetchLvgmcService)
|
||||
serverTask = server.run
|
||||
|
||||
exitCode <- (serverTask, fetchStationsTask, cleanupTask, fetchGribTask).parMapN((_, _, _, _) => ExitCode.Success)
|
||||
|
||||
@@ -5,8 +5,9 @@ import cats.implicits.toTraverseOps
|
||||
import com.comcast.ip4s.IpLiteralSyntax
|
||||
import data.DataService
|
||||
import db.PostgresService
|
||||
import fetch.csv.FetchService
|
||||
import fetch.lvgmc
|
||||
import fetch.csv.FileNameService
|
||||
//import fetch.csv.FetchService
|
||||
import fetch.lvgmc.FetchService
|
||||
import fs2.io.file.{Files, Path}
|
||||
import server.ValidateRoutes.{AggFieldList, AggKey, CityList, DateTimeRange, Granularity, ValidateDate, ValidateDateTime, ValidateInt, ValidateMonths, ValidateZonedDateTime}
|
||||
import io.circe.{Encoder, Json, Printer}
|
||||
@@ -52,10 +53,24 @@ class Server(postgresService: PostgresService, dataService: DataService, fetch:
|
||||
private val apiRoutes = HttpRoutes.of[IO] {
|
||||
// http://0.0.0.0:8080/api/show/lvgmc-forecast/Latvija_LTV_pilsetas_tekosa_dn.csv
|
||||
case GET -> Root / "show" / "lvgmc-forecast" / fileName =>
|
||||
lvgmc.FetchService.of.flatMap(f => f.fetchFile(fileName)).flatMap(bytes =>
|
||||
fetch.fetchFile(fileName).flatMap(bytes =>
|
||||
Ok(bytes).map(_.withContentType(`Content-Type`(MediaType.text.csv)))
|
||||
)
|
||||
|
||||
// http://0.0.0.0:8080/api/fetch/lvgmc/stations
|
||||
case GET -> Root / "fetch" / "lvgmc" / "stations" =>
|
||||
(
|
||||
for {
|
||||
fileName <- new FileNameService().generateCurrentHour
|
||||
stationDataStr <- fetch.fetchWeatherStations()
|
||||
_ <- postgresService.save(fileName, stationDataStr)
|
||||
} yield stationDataStr
|
||||
)
|
||||
.flatMap(content => Ok(content))
|
||||
.handleErrorWith(error =>
|
||||
InternalServerError(s"Failed to fetch stations: ${error.getMessage}")
|
||||
)
|
||||
|
||||
// http://0.0.0.0:8080/api/show/grib-all-structure
|
||||
case GET -> Root / "show" / "grib-all-structure" =>
|
||||
dataService.getAllFileStructure().flatMap(gribList => Ok(gribList.asJson))
|
||||
@@ -121,29 +136,29 @@ class Server(postgresService: PostgresService, dataService: DataService, fetch:
|
||||
postgresService.queryCountry(from, to, fieldList)
|
||||
.flatMap(result => Ok(result.asJson.pretty))
|
||||
|
||||
// http://0.0.0.0:8080/api/fetch/date/20230514
|
||||
case GET -> Root / "fetch" / "date" / ValidateDate(date) =>
|
||||
val result = for {
|
||||
fetchResultEither <- fetch.fetchFromDate(date).attempt
|
||||
fetchServiceError = fetchResultEither.left.toOption.map(e => s"FetchServiceError: ${e.getMessage}").toList
|
||||
fetchResult = fetchResultEither.getOrElse(List.empty)
|
||||
(fetchErrors, successDownloads) = fetchResult.partitionMap(identity)
|
||||
_ <- log.info(s"FETCHED SUCCESSFULLY files: ${successDownloads.size}")
|
||||
saveResults <- successDownloads.traverse { case (name, content) => postgresService.save(name, content).attempt }
|
||||
(saveErrors, successSaves) = saveResults.partitionMap(identity)
|
||||
// successes = successDownloads.map(s => s"fetched: ${s._1}") ++ successSaves.map(s => s"saved: $s")
|
||||
successes = successSaves
|
||||
errors = fetchServiceError ++ fetchErrors.map(e => s"FetchError: ${e.getMessage}") ++ saveErrors.map(e => s"SaveError: ${e.getMessage}")
|
||||
_ <- log.error(s"errors: $errors")
|
||||
_ <- log.info(s"successes: $successes")
|
||||
} yield (successes, errors)
|
||||
|
||||
result.flatMap { case (successes, errors) =>
|
||||
Ok(Json.obj(
|
||||
"errors" -> errors.asJson,
|
||||
"successes" -> successes.asJson
|
||||
).pretty)
|
||||
}
|
||||
// // http://0.0.0.0:8080/api/fetch/date/20230514
|
||||
// case GET -> Root / "fetch" / "date" / ValidateDate(date) =>
|
||||
// val result = for {
|
||||
// fetchResultEither <- fetch.fetchFromDate(date).attempt
|
||||
// fetchServiceError = fetchResultEither.left.toOption.map(e => s"FetchServiceError: ${e.getMessage}").toList
|
||||
// fetchResult = fetchResultEither.getOrElse(List.empty)
|
||||
// (fetchErrors, successDownloads) = fetchResult.partitionMap(identity)
|
||||
// _ <- log.info(s"FETCHED SUCCESSFULLY files: ${successDownloads.size}")
|
||||
// saveResults <- successDownloads.traverse { case (name, content) => postgresService.save(name, content).attempt }
|
||||
// (saveErrors, successSaves) = saveResults.partitionMap(identity)
|
||||
//// successes = successDownloads.map(s => s"fetched: ${s._1}") ++ successSaves.map(s => s"saved: $s")
|
||||
// successes = successSaves
|
||||
// errors = fetchServiceError ++ fetchErrors.map(e => s"FetchError: ${e.getMessage}") ++ saveErrors.map(e => s"SaveError: ${e.getMessage}")
|
||||
// _ <- log.error(s"errors: $errors")
|
||||
// _ <- log.info(s"successes: $successes")
|
||||
// } yield (successes, errors)
|
||||
//
|
||||
// result.flatMap { case (successes, errors) =>
|
||||
// Ok(Json.obj(
|
||||
// "errors" -> errors.asJson,
|
||||
// "successes" -> successes.asJson
|
||||
// ).pretty)
|
||||
// }
|
||||
|
||||
// http://0.0.0.0:8080/api/show/months/202304,202305,202306
|
||||
case GET -> Root / "show" / "months" / ValidateMonths(monthList) =>
|
||||
|
||||
@@ -3,70 +3,33 @@ import { createSignal } from "solid-js";
|
||||
import { apiHost } from "../../consts";
|
||||
|
||||
export function FetchFiles() {
|
||||
const [getStart, setStart] = createSignal(new Date());
|
||||
const [getEnd, setEnd] = createSignal(new Date());
|
||||
const [getFetchResult, setFetchResult] = createSignal("");
|
||||
const [isSpinner, setIsSpinner] = createSignal(false);
|
||||
|
||||
const startStr = () => moment(getStart()).format("YYYY-MM-DD");
|
||||
const endStr = () => moment(getEnd()).format("YYYY-MM-DD");
|
||||
|
||||
function handleSubmit(e: MouseEvent) {
|
||||
async function handleSubmit(e: MouseEvent) {
|
||||
e.preventDefault();
|
||||
const dates = getDatesBetween(getStart(), getEnd());
|
||||
timerFetch(dates);
|
||||
}
|
||||
|
||||
async function timerFetch(dates: Date[]) {
|
||||
setIsSpinner(true);
|
||||
for (const date of dates) {
|
||||
const response = await fetch(`${apiHost}/api/fetch/date/${moment(date).format("YYYYMMDD")}`);
|
||||
const text = await response.text();
|
||||
setFetchResult(text);
|
||||
await new Promise(resolve => setTimeout(resolve, 200));
|
||||
}
|
||||
setIsSpinner(false);
|
||||
const response = await fetch(`${apiHost}/api/fetch/lvgmc/stations`);
|
||||
const text = await response.text();
|
||||
setFetchResult(text);
|
||||
await new Promise(resolve => setTimeout(resolve, 200));
|
||||
setIsSpinner(false)
|
||||
}
|
||||
|
||||
return (
|
||||
<form>
|
||||
<h3>Date range fetch</h3>
|
||||
<h3>Manual fetch</h3>
|
||||
<p>
|
||||
<input
|
||||
type="date"
|
||||
value={ startStr() }
|
||||
onChange={e => setStart(new Date(e.target.value))}
|
||||
/> start
|
||||
</p>
|
||||
<p>
|
||||
<input
|
||||
type="date"
|
||||
value={ endStr() }
|
||||
onChange={e => setEnd(new Date(e.target.value))}
|
||||
/> end
|
||||
</p>
|
||||
<p>
|
||||
<input type="submit" onClick={handleSubmit} value="Fetch .csv files" />
|
||||
<input type="submit" onClick={handleSubmit} value="Fetch most recent .csv file" />
|
||||
</p>
|
||||
<p>result:</p>
|
||||
{ isSpinner() && <div>
|
||||
<span class="spinner"></span>
|
||||
<span style={{ "padding-left": "16px" }}>Fetching csv files</span>
|
||||
</div>
|
||||
{isSpinner() && <div>
|
||||
<span class="spinner"></span>
|
||||
<span style={{ "padding-left": "16px" }}>Fetching csv files</span>
|
||||
</div>
|
||||
}
|
||||
<p>{ getFetchResult() }</p>
|
||||
<p>{getFetchResult()}</p>
|
||||
</form>
|
||||
)
|
||||
}
|
||||
|
||||
function getDatesBetween(startDate: Date, endDate: Date): Date[] {
|
||||
const dates: Date[] = [];
|
||||
let currentDate = new Date(startDate); // start from the start date
|
||||
|
||||
while (currentDate <= endDate) {
|
||||
dates.push(new Date(currentDate)); // add current date to the list
|
||||
currentDate.setDate(currentDate.getDate() + 1); // increment the date
|
||||
}
|
||||
|
||||
return dates;
|
||||
}
|
||||
Reference in New Issue
Block a user