Add granularity for aggregation, group by hour/day/month/year

src/main/scala/parse/Aggregate.scala

@@ -4,14 +4,18 @@ import cats.implicits.{catsSyntaxOptionId, toFoldableOps}
 import io.circe.generic.semiauto.deriveEncoder
 import io.circe.syntax.EncoderOps
 import io.circe.{Encoder, Json}
+
 import java.time.LocalDateTime
+import java.time.format.DateTimeFormatter
+import java.time.temporal.ChronoUnit
 
 object Aggregate {
 
   final case class UserQuery(
     cities: List[String],
     field: String,
-    key: AggregateKey
+    key: AggregateKey,
+    granularity: ChronoUnit,
   )
 
   sealed trait AggregateKey
@@ -46,7 +50,7 @@ object Aggregate {
 
   sealed trait AggregateValue
   final case class DoubleValue(value: Double) extends AggregateValue
-  final case class TimeDoubleList(list: List[(LocalDateTime, Option[Double])]) extends AggregateValue
+  final case class TimeDoubleList(list: List[(String, Option[Double])]) extends AggregateValue
   final case class StringListList(list: List[List[String]]) extends AggregateValue
   final case class DistinctStringList(list: List[String]) extends AggregateValue
   object AggregateValue {
@@ -80,21 +84,46 @@ object Aggregate {
     implicit val distinctStringListEncoder: Encoder[DistinctStringList] = deriveEncoder[DistinctStringList]
   }
 
-  def extractDoubleFieldValues(field: String, weatherData: List[WeatherData]): List[Option[Double]] =
+  private def convertDateTime(granularity: ChronoUnit)(data: (LocalDateTime, _)): String = {
+    val dateTime = data._1
+    granularity match {
+      case ChronoUnit.DAYS => dateTime.toLocalDate.toString
+      case ChronoUnit.MONTHS => dateTime.format(DateTimeFormatter.ofPattern("yyyy-MM"))
+      case ChronoUnit.YEARS => dateTime.format(DateTimeFormatter.ofPattern("yyyy"))
+      case _ => dateTime.withMinute(0).toString
+    }
+  }
+
+  private def groupByGranularity(
+     list: List[(LocalDateTime, Option[Double])],
+     granularity: ChronoUnit,
+     aggregateKey: AggregateKey,
+  ): List[(String, Option[Double])] = {
+    list
+      .groupBy(convertDateTime(granularity))
+      .toList
+      .map { case(dateTime, groupedList) => (dateTime, flattenField(aggregateKey, groupedList)) }
+  }
+
+  def aggregateByField(
+    field: String,
+    granularity: ChronoUnit,
+    data: List[WeatherStationData],
+  ): List[(String, Option[Double])] =
     field match {
-      case "tempMax" => weatherData.map(_.tempMax)
+      case "tempMax" => groupByGranularity(data.map(d => (d.timestamp, d.weather.tempMax)), granularity, AggregateKey.Max)
-      case "tempMin" => weatherData.map(_.tempMin)
+      case "tempMin" => groupByGranularity(data.map(d => (d.timestamp, d.weather.tempMin)), granularity, AggregateKey.Min)
-      case "tempAvg" => weatherData.map(_.tempAvg)
+      case "tempAvg" => groupByGranularity(data.map(d => (d.timestamp, d.weather.tempAvg)), granularity, AggregateKey.Avg)
-      case "precipitation" => weatherData.map(_.precipitation)
+      case "precipitation" => groupByGranularity(data.map(d => (d.timestamp, d.weather.precipitation)), granularity, AggregateKey.Sum)
-      case "windAvg" => weatherData.map(_.windAvg)
+      case "windAvg" => groupByGranularity(data.map(d => (d.timestamp, d.weather.windAvg)), granularity, AggregateKey.Avg)
-      case "windMax" => weatherData.map(_.windMax)
+      case "windMax" => groupByGranularity(data.map(d => (d.timestamp, d.weather.windMax)), granularity, AggregateKey.Max)
-      case "visibilityMin" => weatherData.map(_.visibilityMin)
+      case "visibilityMin" => groupByGranularity(data.map(d => (d.timestamp, d.weather.visibilityMin)), granularity, AggregateKey.Min)
-      case "visibilityAvg" => weatherData.map(_.visibilityAvg)
+      case "visibilityAvg" => groupByGranularity(data.map(d => (d.timestamp, d.weather.visibilityAvg)), granularity, AggregateKey.Avg)
-      case "snowAvg" => weatherData.map(_.snowAvg)
+      case "snowAvg" => groupByGranularity(data.map(d => (d.timestamp, d.weather.snowAvg)), granularity, AggregateKey.Avg)
-      case "atmPressire" => weatherData.map(_.atmPressure)
+      case "atmPressure" => groupByGranularity(data.map(d => (d.timestamp, d.weather.atmPressure)), granularity, AggregateKey.Avg)
-      case "dewPoint" => weatherData.map(_.dewPoint)
+      case "dewPoint" => groupByGranularity(data.map(d => (d.timestamp, d.weather.dewPoint)), granularity, AggregateKey.Avg)
-      case "humidity" => weatherData.map(_.humidity)
+      case "humidity" => groupByGranularity(data.map(d => (d.timestamp, d.weather.humidity)), granularity, AggregateKey.Avg)
-      case "sunDuration" => weatherData.map(_.sunDuration)
+      case "sunDuration" => groupByGranularity(data.map(d => (d.timestamp, d.weather.sunDuration)), granularity, AggregateKey.Sum)
       case _ => List.empty
     }
 
@@ -102,18 +131,31 @@ object Aggregate {
     def roundDecimal: Double = BigDecimal(d).setScale(1, BigDecimal.RoundingMode.HALF_UP).toDouble
   }
 
+  private def flattenField(
+   AggKey: AggregateKey,
+   values: List[(LocalDateTime, Option[Double])],
+  ): Option[Double] = {
+    val flatValues = values.flatMap(_._2)
+    AggKey match {
+      case AggregateKey.Min => flatValues.minimumOption
+      case AggregateKey.Max => flatValues.maximumOption
+      case AggregateKey.Avg => flatValues.reduceOption(_ + _).map(_ / flatValues.length)
+      case AggregateKey.Sum => flatValues.sum.some
+      case _ => None
+    }
+  }
+
   def aggregateDoubleValues(
     AggKey: AggregateKey,
-    values: List[Option[Double]],
+    values: List[(String, Option[Double])],
-    timestamps: List[LocalDateTime],
   ): Option[AggregateValue] = {
-    val flatValues = values.flatten
+    val flatValues = values.flatMap(_._2)
     AggKey match {
       case AggregateKey.Min => flatValues.minimumOption.map(value => DoubleValue(value.roundDecimal))
       case AggregateKey.Max => flatValues.maximumOption.map(value => DoubleValue(value.roundDecimal))
       case AggregateKey.Avg => flatValues.reduceOption(_ + _).map(sum => DoubleValue((sum / flatValues.length).roundDecimal))
       case AggregateKey.Sum => flatValues.sum.some.map(value => DoubleValue(value.roundDecimal))
-      case AggregateKey.List => TimeDoubleList(timestamps.zip(values).sorted).some
+      case AggregateKey.List => TimeDoubleList(values.sorted).some
       case AggregateKey.Distinct => None
     }
   }

src/main/scala/parse/Main.scala

@@ -0,0 +1,63 @@
+package parse
+
+import cats.effect.unsafe.implicits.global
+import db.DBService
+import io.circe.syntax.EncoderOps
+import parse.Aggregate.{AggregateKey, UserQuery}
+
+import java.time.LocalDateTime
+import java.time.format.DateTimeFormatter
+import java.time.temporal.ChronoUnit
+import parse.Aggregate.AggregateValueImplicits.aggregateValueEncoder
+
+// TODO remake this as a test with different granularities and especially check avg value calculations
+object Main {
+  def main(args: Array[String]): Unit = {
+    val formatter = DateTimeFormatter.ofPattern("yyyyMMdd_HHmm")
+
+    def groupByGranularity(granularity: ChronoUnit) = {
+      granularity match {
+        case ChronoUnit.DAYS => (dateTime: LocalDateTime) => dateTime.toLocalDate
+        case ChronoUnit.MONTHS => (dateTime: LocalDateTime) => dateTime.toLocalDate.withDayOfMonth(1)
+        case ChronoUnit.YEARS => (dateTime: LocalDateTime) => dateTime.toLocalDate.withDayOfYear(1)
+        case _ => (dateTime: LocalDateTime) => dateTime.withMinute(0).withSecond(0).withNano(0)
+      }
+    }
+
+    val list = List(
+      (LocalDateTime.parse("20230515_0900", formatter), Some(20d)),
+      (LocalDateTime.parse("20230515_1000", formatter), None),
+      (LocalDateTime.parse("20230515_1100", formatter), Some(6d)),
+      (LocalDateTime.parse("20230516_1100", formatter), Some(10d)),
+      (LocalDateTime.parse("20230416_1100", formatter), Some(10d)),
+    )
+
+    val grouped = list.groupBy{ case (dateTime, _) => groupByGranularity(ChronoUnit.HOURS)(dateTime) }
+
+//    println(grouped)
+
+//    grouped.foreach { case (key, value) =>
+//      println(s"$key -> $value")
+//    }
+
+    val from = LocalDateTime.parse("20230627_0000", formatter)
+    val to = LocalDateTime.parse("20230627_2359", formatter)
+
+    val dbService = DBService.of.unsafeRunSync()
+    val lines = dbService.getInRange(from, to).unsafeRunSync()
+    val query = UserQuery(List("Ainaži"), "tempAvg", AggregateKey.Avg, ChronoUnit.HOURS)
+    val parsed = Parser.queryData(query, lines)
+    println(parsed.asJson)
+
+
+//    val query2 = UserQuery(List("Ainaži"), "tempAvg", AggregateKey.Avg, ChronoUnit.DAYS)
+//    val parsed2 = Parser.queryData(query2, lines)
+//    println(parsed2.asJson)
+
+
+//    val query3 = UserQuery(List("Ainaži"), "tempAvg", AggregateKey.Avg, ChronoUnit.MONTHS)
+//    val parsed3 = Parser.queryData(query3, lines)
+//    println(parsed3.asJson)
+
+  }
+}

src/main/scala/parse/Parser.scala

@@ -1,9 +1,10 @@
 package parse
 
-import parse.Aggregate.{AggregateValue, UserQuery, aggregateDoubleValues, aggregatePhenomenaValues, extractDoubleFieldValues}
+import parse.Aggregate.{AggregateValue, UserQuery, aggregateDoubleValues, aggregatePhenomenaValues, aggregateByField}
 
 import java.time.LocalDateTime
 import java.time.format.DateTimeFormatter
+import java.time.temporal.ChronoUnit
 import scala.util.Try
 
 object Parser {
@@ -48,8 +49,8 @@ object Parser {
           (city -> aggregatePhenomenaValues(userQuery.key, phenomenaList))
         }
         case field => {
-          val doubleList = extractDoubleFieldValues(field, weatherStationData.map(_.weather))
+          val doubleList = aggregateByField(field, userQuery.granularity, weatherStationData)
-          (city -> aggregateDoubleValues(userQuery.key, doubleList, weatherStationData.map(_.timestamp)))
+          (city -> aggregateDoubleValues(userQuery.key, doubleList))
 
         }
       }

src/main/scala/server/Server.scala

@@ -6,7 +6,7 @@ import com.comcast.ip4s.IpLiteralSyntax
 import db.DataService
 import fetch.FetchService
 import parse.{Parser, WeatherData}
-import server.ValidateRoutes.{AggKey, CityList, DateTimeRange, ValidDate}
+import server.ValidateRoutes.{AggKey, CityList, DateTimeRange, Granularity, ValidDate}
 import io.circe.{Json, Printer}
 import org.http4s._
 import org.http4s.dsl.io._
@@ -23,6 +23,7 @@ import org.http4s.circe.jsonEncoder
 import org.typelevel.log4cats.Logger
 import org.typelevel.log4cats.slf4j.Slf4jLogger
 
+import java.time.temporal.ChronoUnit
 import scala.concurrent.duration.DurationInt
 
 
@@ -47,9 +48,9 @@ class Server(dataService: DataService, fetch: FetchService, log: Logger[IO]) {
   private val apiRoutes = HttpRoutes.of[IO] {
 
     // http://0.0.0.0:8080/api/query/20230414_2200-20230501_1230/Liepāja,Rēzekne/tempMax/max
-    case GET -> Root / "query" / DateTimeRange(from, to) / CityList(cities) / field / AggKey(key) =>
+    case GET -> Root / "query" / DateTimeRange(from, to) / Granularity(granularity) / CityList(cities) / field / AggKey(key) =>
       dataService.getInRange(from, to)
-        .map(Parser.queryData(UserQuery(cities, field, key), _))
+        .map(Parser.queryData(UserQuery(cities, field, key, granularity), _))
         .flatMap(result => Ok(result.asJson.pretty))
 
     // http://0.0.0.0:8080/api/fetch/date/20230514

src/main/scala/server/ValidateRoutes.scala

@@ -4,6 +4,7 @@ import parse.Aggregate.AggregateKey
 
 import java.time.{LocalDate, LocalDateTime}
 import java.time.format.DateTimeFormatter
+import java.time.temporal.ChronoUnit
 import scala.util.Try
 
 object ValidateRoutes {
@@ -40,4 +41,16 @@ object ValidateRoutes {
       AggregateKey.fromString(str)
     }
   }
+
+  object Granularity {
+    def unapply(str: String): Option[ChronoUnit] = {
+      str match {
+        case "hour" => Some(ChronoUnit.HOURS)
+        case "day" => Some(ChronoUnit.DAYS)
+        case "month" => Some(ChronoUnit.MONTHS)
+        case "year" => Some(ChronoUnit.YEARS)
+        case _ => None
+      }
+    }
+  }
 }

src/test/scala/parse/ParserSpec.scala

@@ -8,6 +8,7 @@ import parse.Aggregate.{AggregateKey, DoubleValue, TimeDoubleList, UserQuery}
 
 import java.time.LocalDateTime
 import java.time.format.DateTimeFormatter
+import java.time.temporal.ChronoUnit
 import scala.collection.immutable.HashMap
 
 class ParserSpec extends AnyFunSuite with Matchers {
@@ -16,7 +17,7 @@ class ParserSpec extends AnyFunSuite with Matchers {
     val formatter = DateTimeFormatter.ofPattern("yyyyMMdd_HHmm")
     val from = LocalDateTime.parse("20230515_0905", formatter)
     val to = LocalDateTime.parse("20230516_0942", formatter)
-    val userQuery = UserQuery(List("Bauska", "Dagda", "Daugavgrīva", "Rīga"), "precipitation", AggregateKey.Sum)
+    val userQuery = UserQuery(List("Bauska", "Dagda", "Daugavgrīva", "Rīga"), "precipitation", AggregateKey.Sum, ChronoUnit.HOURS)
 
     val dbService = DBService.of.unsafeRunSync()
 
@@ -35,7 +36,7 @@ class ParserSpec extends AnyFunSuite with Matchers {
     val formatter = DateTimeFormatter.ofPattern("yyyyMMdd_HHmm")
     val from = LocalDateTime.parse("20230516_0400", formatter)
     val to = LocalDateTime.parse("20230516_0800", formatter)
-    val userQuery = UserQuery(List("Rīga"), "precipitation", AggregateKey.List)
+    val userQuery = UserQuery(List("Rīga"), "precipitation", AggregateKey.List, ChronoUnit.HOURS)
 
     val dbService = DBService.of.unsafeRunSync()
 
@@ -45,10 +46,10 @@ class ParserSpec extends AnyFunSuite with Matchers {
     parsed shouldBe HashMap(
       "Rīga" ->
         Some(TimeDoubleList(List(
-          (LocalDateTime.parse("2023-05-16T04:00"), Some(1.9)),
+          ("2023-05-16T04:00", Some(1.9)),
-          (LocalDateTime.parse("2023-05-16T05:00"), Some(4.5)),
+          ("2023-05-16T05:00", Some(4.5)),
-          (LocalDateTime.parse("2023-05-16T06:00"), Some(1.5)),
+          ("2023-05-16T06:00", Some(1.5)),
-          (LocalDateTime.parse("2023-05-16T07:00"), Some(0.0)),
+          ("2023-05-16T07:00", Some(0.0)),
         )))
     )
   }

web/src/aggregator/Aggregator.tsx

@@ -4,6 +4,7 @@ import { createSignal } from "solid-js";
 import { Result } from "./Result";
 import { SelectCity } from "./SelectCity";
 import { SelectField } from "./SelectField";
+import { SelectGranularity } from "./SelectGranularity";
 import { SelectKey } from "./SelectKey";
 import { SelectTimeRange } from "./SelectTimeRange";
 
@@ -16,6 +17,7 @@ export function Aggregator() {
     const [getEnd, setEnd] = createSignal(nowRounded);
     const [getField, setField] = createSignal("tempMax");
     const [getKey, setKey] = createSignal("max");
+    const [getGranularity, setGranularity] = createSignal("hour");
 
     return (
         <div class="aggregator">
@@ -36,6 +38,7 @@ export function Aggregator() {
                     <div>
                         <SelectField getField={getField} setField={setField} />
                         <SelectKey getKey={getKey} setKey={setKey} />
+                        <SelectGranularity getGranularity={getGranularity} setGranularity={setGranularity} />
                     </div>
                 </div>
                 <div class="column">
@@ -45,6 +48,7 @@ export function Aggregator() {
                         getEnd={getEnd}
                         getField={getField}
                         getKey={getKey}
+                        getGranularity={getGranularity}
                     />
                 </div>
             </div>

web/src/aggregator/CityChart.tsx

@@ -5,11 +5,25 @@ import moment from "moment";
 // Register the controllers, elements, scales, and plugins we'll be using
 Chart.register(LineController, LinearScale, PointElement, LineElement, Title, CategoryScale);
 
+function formatDateString(str: string): string {
+  const date = moment(new Date(str));
+  switch (str.length) {
+    // year: 2023
+    case 4: return date.format("yyyy");
+    // year-month: 2023-06
+    case 7: return date.format("yyyy-MM");
+    // year-month-day: 2023-06-23
+    case 10: return date.format("yyyy-MM-DD");
+    // year-month-day hour:minute 2023-06-23T23:59
+    default: return date.format("HH:mm");
+  }
+}
+
 export const CityChart: Component<{ data: [string, number | null][] }> = (props) => {
   const [canvas, setCanvas] = createSignal<HTMLCanvasElement>();
   let chart: Chart;
   const data: [string, number | null][] = props.data.map(([dateStr, value]) => [
-    moment(new Date(dateStr)).format("HH:mm"),
+    formatDateString(dateStr),
     value,
   ]);
 

web/src/aggregator/Result.tsx

@@ -12,6 +12,7 @@ export const Result: Component<{
     getEnd: Accessor<Date>,
     getField: Accessor<string>,
     getKey: Accessor<string>,
+    getGranularity: Accessor<string>,
 }> = (props) => {
     const cities = () => [...props.getCities()].join(",");
     const queryStart = () => moment(props.getStart()).format("YYYYMMDD_HHmm");
@@ -21,7 +22,7 @@ export const Result: Component<{
     const fetchQuery = async (timestamp: number) => {
         if (cities() === "") return new Error("ERROR: Select cities!");
         await new Promise(resolve => setTimeout(resolve, 500))
-        const response = await fetch(`${apiHost}/api/query/${queryStart()}-${queryEnd()}/${cities()}/${props.getField()}/${props.getKey()}`);
+        const response = await fetch(`${apiHost}/api/query/${queryStart()}-${queryEnd()}/${props.getGranularity()}/${cities()}/${props.getField()}/${props.getKey()}`);
         const json = await response.json();
         return json;
     }

web/src/aggregator/SelectGranularity.tsx

@@ -0,0 +1,22 @@
+import { Accessor, Component, Setter } from "solid-js";
+
+import { aggregateGranularity } from "../consts";
+
+export const SelectGranularity: Component<{
+    getGranularity: Accessor<string>,
+    setGranularity: Setter<string>,
+}> = ({ getGranularity, setGranularity }) => {
+    return (
+        <div>
+            <h3>Select granularity</h3>
+            <select onChange={(e) => setGranularity(e.target.value)}>
+                { aggregateGranularity.map(granularity =>
+                    <option
+                        value={granularity}
+                        selected={granularity === getGranularity() ? true : false}
+                    >{granularity}</option>
+                )}
+            </select>
+        </div>
+    );
+}

web/src/consts.ts

@@ -6,6 +6,8 @@ export const weatherField = ["tempMax", "tempMin", "tempAvg", "precipitation", "
 
 export const aggregateKey = ["min", "max", "avg", "sum", "list", "distinct"];
 
+export const aggregateGranularity = ["hour", "day", "month", "year"];
+
 export type ResultKeyVal = [key: string, value: number | Array<any>];
 
 export const resultOrder = {