Getting lvgmc csv files in frontend

This commit is contained in:
Guntis Smaukstelis
2025-03-10 16:56:24 +02:00
parent 9d6b86e218
commit 841b4b6ec9
6 changed files with 62 additions and 6 deletions
@@ -63,7 +63,7 @@ class FetchService(log: Logger[IO]) {
} }
} }
private def retrieveFileAsString(ftpClient: FTPClient, remotePath: String): IO[String] = IO.blocking { private def retrieveFileAsBytes(ftpClient: FTPClient, remotePath: String): IO[Array[Byte]] = IO.blocking {
val outputStream = new ByteArrayOutputStream() val outputStream = new ByteArrayOutputStream()
val success = ftpClient.retrieveFile(remotePath, outputStream) val success = ftpClient.retrieveFile(remotePath, outputStream)
@@ -71,16 +71,16 @@ class FetchService(log: Logger[IO]) {
throw new RuntimeException(s"Failed to retrieve file: $remotePath, reply: ${ftpClient.getReplyString}") throw new RuntimeException(s"Failed to retrieve file: $remotePath, reply: ${ftpClient.getReplyString}")
} }
outputStream.toString("UTF-8") outputStream.toByteArray()
}.onError(e => log.error(e)(s"Error retrieving file $remotePath")) }.onError(e => log.error(e)(s"Error retrieving file $remotePath"))
def fetchFile(fileName: String): IO[String] = { def fetchFile(fileName: String): IO[Array[Byte]] = {
val remotePath = s"/ltv/tabulas/$fileName" val remotePath = s"/ltv/tabulas/$fileName"
createFtpClient.use { ftpClient => createFtpClient.use { ftpClient =>
for { for {
_ <- log.info(s"Fetching file: $remotePath") _ <- log.info(s"Fetching file: $remotePath")
content <- retrieveFileAsString(ftpClient, remotePath) content <- retrieveFileAsBytes(ftpClient, remotePath)
_ <- log.info(s"Successfully fetched file: $fileName") _ <- log.info(s"Successfully fetched file: $fileName")
} yield content } yield content
} }
@@ -18,7 +18,7 @@ object FetchServiceTest {
val program = for { val program = for {
fetch <- FetchService.of fetch <- FetchService.of
result <- fetch.fetchFile("Latvija_LTV_pilsetas_tekosa_dn.csv") result <- fetch.fetchFile("Latvija_LTV_pilsetas_tekosa_dn.csv")
_ <- IO.println(result) _ <- IO.println(s"fetched file, size: ${result.length}")
} yield () } yield ()
program program
} }
+10
View File
@@ -6,6 +6,7 @@ import com.comcast.ip4s.IpLiteralSyntax
import data.DataService import data.DataService
import db.PostgresService import db.PostgresService
import fetch.csv.FetchService import fetch.csv.FetchService
import fetch.lvgmc
import fs2.io.file.Files import fs2.io.file.Files
import server.ValidateRoutes.{AggFieldList, AggKey, CityList, DateTimeRange, Granularity, ValidateDate, ValidateDateTime, ValidateInt, ValidateMonths, ValidateZonedDateTime} import server.ValidateRoutes.{AggFieldList, AggKey, CityList, DateTimeRange, Granularity, ValidateDate, ValidateDateTime, ValidateInt, ValidateMonths, ValidateZonedDateTime}
import io.circe.{Encoder, Json, Printer} import io.circe.{Encoder, Json, Printer}
@@ -28,6 +29,7 @@ import org.typelevel.log4cats.Logger
import org.typelevel.log4cats.slf4j.Slf4jLogger import org.typelevel.log4cats.slf4j.Slf4jLogger
import parse.csv.Aggregate import parse.csv.Aggregate
import fs2.io.file.{Files, Path} import fs2.io.file.{Files, Path}
import org.http4s.headers.`Content-Type`
import java.time.{ZoneOffset, ZonedDateTime} import java.time.{ZoneOffset, ZonedDateTime}
import scala.concurrent.duration.DurationInt import scala.concurrent.duration.DurationInt
@@ -54,6 +56,13 @@ class Server(postgresService: PostgresService, dataService: DataService, fetch:
private case class ResponseWrapper(result: Map[String, Option[Aggregate.AggregateValue]], query: UserQuery) private case class ResponseWrapper(result: Map[String, Option[Aggregate.AggregateValue]], query: UserQuery)
private val apiRoutes = HttpRoutes.of[IO] { private val apiRoutes = HttpRoutes.of[IO] {
// http://0.0.0.0:8080/api/show/lvgmc-forecast/Latvija_LTV_pilsetas_tekosa_dn.csv
case GET -> Root / "show" / "lvgmc-forecast" / fileName =>
println("lvgmc-forecast GET")
lvgmc.FetchService.of.flatMap(f => f.fetchFile(fileName)).flatMap(bytes =>
Ok(bytes).map(_.withContentType(`Content-Type`(MediaType.text.csv)))
)
case GET -> Root / "show" / "gribName" => Ok("{\"fileName\":\"TODO replace this fake name\"}") case GET -> Root / "show" / "gribName" => Ok("{\"fileName\":\"TODO replace this fake name\"}")
// http://0.0.0.0:8080/api/show/grib-all-structure // http://0.0.0.0:8080/api/show/grib-all-structure
@@ -198,6 +207,7 @@ class Server(postgresService: PostgresService, dataService: DataService, fetch:
"/latvia" -> staticcontent.fileService[IO](FileService.Config("./web/dist")), "/latvia" -> staticcontent.fileService[IO](FileService.Config("./web/dist")),
"/database" -> staticcontent.fileService[IO](FileService.Config("./web/dist")), "/database" -> staticcontent.fileService[IO](FileService.Config("./web/dist")),
"/harmonie" -> staticcontent.fileService[IO](FileService.Config("./web/dist")), "/harmonie" -> staticcontent.fileService[IO](FileService.Config("./web/dist")),
"/lvgmc-forecast" -> staticcontent.fileService[IO](FileService.Config("./web/dist")),
).orNotFound ).orNotFound
def run: IO[ExitCode] = def run: IO[ExitCode] =
+3 -1
View File
@@ -9,12 +9,13 @@ import { Station } from './station/Station';
import { apiHost } from './consts'; import { apiHost } from './consts';
import { WeatherIconStub } from './components/weatherIcons/WeatherIconStub'; import { WeatherIconStub } from './components/weatherIcons/WeatherIconStub';
import { Harmonie } from './harmonie/Harmonie' import { Harmonie } from './harmonie/Harmonie'
import { LvgmcForecast } from './lvgmc-forecast/LvgmcForecast'
console.log("env:", import.meta.env.MODE) console.log("env:", import.meta.env.MODE)
console.log("api host:", apiHost) console.log("api host:", apiHost)
const App: Component = () => { const App: Component = () => {
const pages = ['station', 'cities', 'latvia', 'database', 'harmonie'] const pages = ['station', 'cities', 'latvia', 'database', 'harmonie', 'lvgmc-forecast']
return ( return (
<div> <div>
<div class="grid-1-1"> <div class="grid-1-1">
@@ -32,6 +33,7 @@ const App: Component = () => {
<Router> <Router>
<Route path="/" component={Harmonie} /> <Route path="/" component={Harmonie} />
<Route path="/harmonie" component={Harmonie} /> <Route path="/harmonie" component={Harmonie} />
<Route path="/lvgmc-forecast" component={LvgmcForecast} />
<Route path="/station" component={Station} /> <Route path="/station" component={Station} />
<Route path="/cities" component={Cities} /> <Route path="/cities" component={Cities} />
<Route path="/latvia" component={Country} /> <Route path="/latvia" component={Country} />
+26
View File
@@ -25,6 +25,32 @@ export const fetchJson = (url: string, options = {}) => {
}) })
} }
export const fetchText = (url: string, options = {}) => {
return fetch(url, options).then(async response => {
let responseCode = ''
if (!response.ok) {
responseCode = `error ${response.status}`
}
let text = ''
let responseMessage = ''
try {
text = await response.text()
} catch (err) {
responseMessage = 'Invalid TEXT response'
}
if (responseCode) {
if (responseMessage) throw new Error(`${responseCode}: ${responseMessage}`)
else throw new Error(responseCode)
} else if (responseMessage) {
throw new Error(responseMessage)
}
return text
})
}
export const fetchBuffer = (url: string, options = {}) => { export const fetchBuffer = (url: string, options = {}) => {
return fetch(url, options).then(async response => { return fetch(url, options).then(async response => {
+18
View File
@@ -0,0 +1,18 @@
import { Component } from 'solid-js'
import { apiHost } from '../consts'
import { fetchText } from '../helpers/fetch'
// Eiropa_LTV_pilsetas_nakama_dn.csv
// Eiropa_LTV_pilsetas_tekosa_dn.csv
// Latvija_LTV_pilsetas_nakama_dnn.csv
// Latvija_LTV_pilsetas_tekosa_dn.csv
// Latvija_faktiskais_laiks.csv
export const LvgmcForecast: Component<{}> = () => {
fetchText(`${apiHost}/api/show/lvgmc-forecast/Latvija_LTV_pilsetas_tekosa_dn.csv`)
.then(console.log)
return <>
LvgmcForecast
</>
}