Browse Source

first commit

placidenduwayo 1 year ago
parent
commit
2c06acafc5

+ 3
- 2
src/main/java/fr/natan/akkastreamfileprocessingapi/controller/MovieController.java View File

8
 import org.springframework.web.bind.annotation.RequestMethod;
8
 import org.springframework.web.bind.annotation.RequestMethod;
9
 import org.springframework.web.bind.annotation.RestController;
9
 import org.springframework.web.bind.annotation.RestController;
10
 
10
 
11
+@SuppressWarnings("SpellCheckingInspection")
11
 @RestController
12
 @RestController
12
 public class MovieController {
13
 public class MovieController {
13
 
14
 
18
     }
19
     }
19
 
20
 
20
     @RequestMapping(value = "/persons", method = RequestMethod.GET)
21
     @RequestMapping(value = "/persons", method = RequestMethod.GET)
21
-    private ResponseEntity<String> getallPersons(){
22
+    private ResponseEntity<String> getAllPersons(){
22
         akkaStreamFilesProcessing.getAllPersons();
23
         akkaStreamFilesProcessing.getAllPersons();
23
         return new ResponseEntity<>("is running", HttpStatus.OK);
24
         return new ResponseEntity<>("is running", HttpStatus.OK);
24
     }
25
     }
49
     }
50
     }
50
 
51
 
51
     @RequestMapping(value = "/tvseries/title/{tvSerieTitle}", method = RequestMethod.GET)
52
     @RequestMapping(value = "/tvseries/title/{tvSerieTitle}", method = RequestMethod.GET)
52
-    private ResponseEntity<String> getListOfPersonsIDByTvSerieID(@PathVariable(name = "tvSerieTitle") String tvSerieTitle){
53
+    private ResponseEntity<String> getListOfPersonsByTvSerieitle(@PathVariable(name = "tvSerieTitle") String tvSerieTitle){
53
       akkaStreamFilesProcessing.getPersonsTeamForTvSerie(tvSerieTitle);
54
       akkaStreamFilesProcessing.getPersonsTeamForTvSerie(tvSerieTitle);
54
        return new ResponseEntity<>("is running", HttpStatus.OK);
55
        return new ResponseEntity<>("is running", HttpStatus.OK);
55
     }
56
     }

+ 1
- 1
src/main/resources/application.yml View File

1
 server:
1
 server:
2
-  port: 8090
2
+  port: 8091
3
   servlet:
3
   servlet:
4
     context-path: /path
4
     context-path: /path
5
 spring:
5
 spring:

+ 13
- 32
src/main/scala/fr/natan/akkastreamfileprocessingapi/service/AkkaStreamComponents.scala View File

12
 
12
 
13
 import java.io.File
13
 import java.io.File
14
 import java.nio.file.Paths
14
 import java.nio.file.Paths
15
-import java.util
16
 import scala.concurrent.Future
15
 import scala.concurrent.Future
17
 
16
 
17
+//noinspection SpellCheckingInspection
18
 object AkkaStreamComponents {
18
 object AkkaStreamComponents {
19
 
19
 
20
   implicit val actor: ActorSystem = ActorSystem("AkkaStreamActor")
20
   implicit val actor: ActorSystem = ActorSystem("AkkaStreamActor")
21
 
21
 
22
   //flows building
22
   //flows building
23
 
23
 
24
-  //flow1
25
   def buildPersonFlow(): Flow[Map[String, String], Person, NotUsed] = {
24
   def buildPersonFlow(): Flow[Map[String, String], Person, NotUsed] = {
26
     val personFlow: Flow[Map[String, String], Person, NotUsed] =
25
     val personFlow: Flow[Map[String, String], Person, NotUsed] =
27
       Flow[Map[String, String]]
26
       Flow[Map[String, String]]
32
     personFlow
31
     personFlow
33
   }
32
   }
34
 
33
 
35
-  //flow2
36
-
37
-  def buildTvSerieFlow(): Flow[Map[String, String], TvSeries, NotUsed] = {
38
-
39
-    val tvFlow: Flow[Map[String, String], TvSeries, NotUsed] =
40
-      Flow[Map[String, String]]
41
-        .map(row => {
42
-          buildTvSerieModel(row)
43
-        })
44
-    tvFlow
45
-  }
46
-
47
-  //flow3
48
   def filterByTvSeriePrimaryTitleFlow(tvSeriePrimaryTitle: String): Flow[Map[String, String], TvSeries, NotUsed] = {
34
   def filterByTvSeriePrimaryTitleFlow(tvSeriePrimaryTitle: String): Flow[Map[String, String], TvSeries, NotUsed] = {
49
     val filterFlow: Flow[Map[String, String], TvSeries, NotUsed] = Flow[Map[String, String]]
35
     val filterFlow: Flow[Map[String, String], TvSeries, NotUsed] = Flow[Map[String, String]]
50
         .filter((rows: Map[String, String]) => {
36
         .filter((rows: Map[String, String]) => {
58
     filterFlow
44
     filterFlow
59
   }
45
   }
60
 
46
 
61
-  //flow4
62
   def filterByPersonIdFlow(nconst: String): Flow[Map[String, String], Person, NotUsed]={
47
   def filterByPersonIdFlow(nconst: String): Flow[Map[String, String], Person, NotUsed]={
63
     val personFilter: Flow[Map[String, String], Person, NotUsed]=
48
     val personFilter: Flow[Map[String, String], Person, NotUsed]=
64
       Flow[Map[String, String]]
49
       Flow[Map[String, String]]
72
     personFilter
57
     personFilter
73
   }
58
   }
74
 
59
 
75
-  //flow5
76
-  def filterByPersonNameFlow(personName: String): Flow[Map[String, String], Person, NotUsed] ={
60
+  def filterByPersonNameFlow(primaryName: String): Flow[Map[String, String], Person, NotUsed] ={
77
     val personFilter: Flow[Map[String, String], Person, NotUsed] =
61
     val personFilter: Flow[Map[String, String], Person, NotUsed] =
78
       Flow[Map[String, String]]
62
       Flow[Map[String, String]]
79
         .filter((rowMap: Map[String, String]) =>{
63
         .filter((rowMap: Map[String, String]) =>{
80
-          rowMap.getOrElse(key = "primaryName", default = "")==personName
64
+          rowMap.getOrElse(key = "primaryName", default = "")==primaryName
81
         })
65
         })
82
         .map((rowMap: Map[String, String])=>{
66
         .map((rowMap: Map[String, String])=>{
83
           buildPersonModel(personMap = rowMap)
67
           buildPersonModel(personMap = rowMap)
86
     personFilter
70
     personFilter
87
   }
71
   }
88
 
72
 
73
+  def buildTvSerieFlow(): Flow[Map[String, String], TvSeries, NotUsed] = {
74
+
75
+    val tvFlow: Flow[Map[String, String], TvSeries, NotUsed] =
76
+      Flow[Map[String, String]]
77
+        .map((rowMap: Map[String, String]) => {
78
+          buildTvSerieModel(tvSerieMap = rowMap)
79
+        })
80
+    tvFlow
81
+  }
89
   //source building
82
   //source building
90
 
83
 
91
   def buildSource(inputFile: File): Source[Map[String, String], NotUsed] = {
84
   def buildSource(inputFile: File): Source[Map[String, String], NotUsed] = {
103
       }
96
       }
104
       )
97
       )
105
       .via(Compression.gunzip())
98
       .via(Compression.gunzip())
106
-      .via(CsvParsing.lineScanner(CsvParsing.Tab))
99
+      .via(CsvParsing.lineScanner(CsvParsing.Tab, CsvParsing.DoubleQuote, CsvParsing.DoubleQuote))
107
       .via(CsvToMap.toMapAsStrings())
100
       .via(CsvToMap.toMapAsStrings())
108
 
101
 
109
     datasource
102
     datasource
119
   }
112
   }
120
 
113
 
121
   //sinks building
114
   //sinks building
122
-  //sink1
123
-  def buildTvSeriesSink(logger: Logger): Sink[TvSeries, Future[Done]] = {
124
-    val tvSeriesSink : Sink[TvSeries, Future[Done]]= Sink
125
-      .foreach((tvSerie: TvSeries)=> logger.info(s"${tvSerie.toString}"))
126
-    tvSeriesSink
127
-  }
128
 
115
 
129
   def buildAllTvSeriesSink(logger: Logger): Sink[TvSeries, Future[Done]] = {
116
   def buildAllTvSeriesSink(logger: Logger): Sink[TvSeries, Future[Done]] = {
130
     val tvSeriesSink: Sink[TvSeries, Future[Done]] = Sink
117
     val tvSeriesSink: Sink[TvSeries, Future[Done]] = Sink
131
       .foreach((tvSerie: TvSeries)=>logger.info(s"${tvSerie.toString}"))
118
       .foreach((tvSerie: TvSeries)=>logger.info(s"${tvSerie.toString}"))
132
     tvSeriesSink
119
     tvSeriesSink
133
   }
120
   }
134
-  //sink2
135
   def buildAllPersonsSink(logger: Logger): Sink[Person,Future[Done]] = {
121
   def buildAllPersonsSink(logger: Logger): Sink[Person,Future[Done]] = {
136
     val listPersonsSink: Sink[Person, Future[Done]]=
122
     val listPersonsSink: Sink[Person, Future[Done]]=
137
       Sink.foreach((person: Person)=>logger.info(s"${person.toString}"))
123
       Sink.foreach((person: Person)=>logger.info(s"${person.toString}"))
138
     listPersonsSink
124
     listPersonsSink
139
   }
125
   }
140
-
141
-  //sink3
142
-  def buildPersonSink(): Sink[Person, Future[Person]] = {
143
-    Sink.head
144
-  }
145
 }
126
 }

+ 2
- 2
src/main/scala/fr/natan/akkastreamfileprocessingapi/service/AkkaStreamFileProcessing.scala View File

1
 package fr.natan.akkastreamfileprocessingapi.service
1
 package fr.natan.akkastreamfileprocessingapi.service
2
 
2
 
3
-import akka.stream.scaladsl.Source
4
 import fr.natan.akkastreamfileprocessingapi.models.Models.{Person, TvSeries}
3
 import fr.natan.akkastreamfileprocessingapi.models.Models.{Person, TvSeries}
5
 
4
 
6
 import scala.concurrent.Future
5
 import scala.concurrent.Future
7
 
6
 
7
+//noinspection SpellCheckingInspection
8
 trait AkkaStreamFileProcessing {
8
 trait AkkaStreamFileProcessing {
9
   def getAllPersons(): Unit
9
   def getAllPersons(): Unit
10
 
10
 
11
   def getPersonById(nconst: String): Future[Person]
11
   def getPersonById(nconst: String): Future[Person]
12
-  def getPersonByName(primaryName: String): Unit
12
+  def getPersonByName(primaryName: String): Future[IndexedSeq[Person]]
13
 
13
 
14
   def getAllTvSeries(): Unit
14
   def getAllTvSeries(): Unit
15
 
15
 

+ 50
- 62
src/main/scala/fr/natan/akkastreamfileprocessingapi/service/AkkaStreamFileProcessingImpl.scala View File

6
 import com.typesafe.scalalogging.slf4j.Logger
6
 import com.typesafe.scalalogging.slf4j.Logger
7
 import fr.natan.akkastreamfileprocessingapi.datasource.Datasource.{nameBasics, titleBasics, titlePrincipalsBasics}
7
 import fr.natan.akkastreamfileprocessingapi.datasource.Datasource.{nameBasics, titleBasics, titlePrincipalsBasics}
8
 import fr.natan.akkastreamfileprocessingapi.models.Models.{Person, TvSeries}
8
 import fr.natan.akkastreamfileprocessingapi.models.Models.{Person, TvSeries}
9
-import fr.natan.akkastreamfileprocessingapi.models.ModelsBuilder.{buildPersonModel, buildTvSerieModel}
10
-import fr.natan.akkastreamfileprocessingapi.service.AkkaStreamComponents.{buildAllPersonsSink, buildAllTvSeriesSink, buildAndValidateSource, buildPersonFlow, buildPersonSink, buildSource, buildTvSerieFlow, buildTvSeriesSink, filterByPersonIdFlow, filterByPersonNameFlow, filterByTvSeriePrimaryTitleFlow}
9
+import fr.natan.akkastreamfileprocessingapi.models.ModelsBuilder.buildPersonModel
10
+import fr.natan.akkastreamfileprocessingapi.service.AkkaStreamComponents.{
11
+  buildAllPersonsSink, buildAllTvSeriesSink, buildAndValidateSource, buildPersonFlow,
12
+  buildSource, buildTvSerieFlow, filterByPersonIdFlow, filterByPersonNameFlow, filterByTvSeriePrimaryTitleFlow}
11
 import org.slf4j.LoggerFactory
13
 import org.slf4j.LoggerFactory
12
 import org.springframework.stereotype.Component
14
 import org.springframework.stereotype.Component
13
 
15
 
14
 import scala.concurrent.ExecutionContext.Implicits.global
16
 import scala.concurrent.ExecutionContext.Implicits.global
15
-import scala.concurrent.duration.DurationInt
16
-import scala.concurrent.{Await, Future}
17
+import scala.concurrent.Future
17
 import scala.language.postfixOps
18
 import scala.language.postfixOps
18
 import scala.util.{Failure, Success}
19
 import scala.util.{Failure, Success}
19
 
20
 
21
+//noinspection SpellCheckingInspection
20
 @Component
22
 @Component
21
 class AkkaStreamFileProcessingImpl extends AkkaStreamFileProcessing {
23
 class AkkaStreamFileProcessingImpl extends AkkaStreamFileProcessing {
22
 
24
 
23
   implicit val actorSystem: ActorSystem = ActorSystem("AkkaStreamActor")
25
   implicit val actorSystem: ActorSystem = ActorSystem("AkkaStreamActor")
24
   implicit val logger: Logger = Logger(LoggerFactory.getLogger(this.getClass))
26
   implicit val logger: Logger = Logger(LoggerFactory.getLogger(this.getClass))
25
 
27
 
26
-  override def getAllPersons() = {
28
+  override def getAllPersons(): Unit = {
27
     val personSource: Source[Map[String, String], _] = buildSource(inputFile = nameBasics)
29
     val personSource: Source[Map[String, String], _] = buildSource(inputFile = nameBasics)
28
     //graph
30
     //graph
29
     val startTime: Long = System.currentTimeMillis()
31
     val startTime: Long = System.currentTimeMillis()
35
       case Failure(exception) => logger.error(s"$exception")
37
       case Failure(exception) => logger.error(s"$exception")
36
       case Success(value) =>
38
       case Success(value) =>
37
         logger.info(s"$value")
39
         logger.info(s"$value")
38
-        val time: Long =(System.currentTimeMillis()-startTime)/100
40
+        val time: Long = (System.currentTimeMillis() - startTime) / 100
39
         logger.info(s"elapsed time: $time")
41
         logger.info(s"elapsed time: $time")
40
     }
42
     }
41
   }
43
   }
43
   override def getPersonById(nconst: String): Future[Person] = {
45
   override def getPersonById(nconst: String): Future[Person] = {
44
     val source: Source[Map[String, String], _] = buildSource(inputFile = nameBasics)
46
     val source: Source[Map[String, String], _] = buildSource(inputFile = nameBasics)
45
 
47
 
46
-    val startTime: Long = System.currentTimeMillis()
48
+
47
     val res = source
49
     val res = source
48
       .via(flow = filterByPersonIdFlow(nconst = nconst))
50
       .via(flow = filterByPersonIdFlow(nconst = nconst))
49
-      .runWith(sink = buildPersonSink())
51
+      .runWith(Sink.head)
50
 
52
 
51
     res.andThen {
53
     res.andThen {
52
-      case Failure(exception) => logger.info(s"error!!!!!!!!!!!!!$exception")
54
+      case Failure(exception) => logger.info(s"$exception")
53
       case Success(value) => logger.info(s"$value")
55
       case Success(value) => logger.info(s"$value")
54
     }
56
     }
55
 
57
 
56
     res
58
     res
57
   }
59
   }
58
 
60
 
59
-  override def getPersonByName(primaryName: String): Unit = {
61
+  override def getPersonByName(primaryName: String): Future[IndexedSeq[Person]] = {
60
     val source: Source[Map[String, String], _] = buildSource(inputFile = nameBasics)
62
     val source: Source[Map[String, String], _] = buildSource(inputFile = nameBasics)
61
-    source
62
-      .via(
63
-        flow = filterByPersonNameFlow(personName = primaryName)
64
-      )
65
-      .runWith(sink = buildPersonSink())
66
-      .andThen {
67
-        case Failure(exception) => logger.info(s"$exception")
68
-        case Success(value) => logger.info(s"$value: Successufully processing")
69
-      }
63
+    val persons: Future[IndexedSeq[Person]] = source
64
+      .via(flow = filterByPersonNameFlow(primaryName = primaryName))
65
+      .runWith(Sink.collection)
66
+
67
+    persons
70
   }
68
   }
71
 
69
 
72
   override def getAllTvSeries(): Unit = {
70
   override def getAllTvSeries(): Unit = {
107
 
105
 
108
   }
106
   }
109
 
107
 
110
-  private def getTvSerieIdByPrimaryTitle(primaryTitle: String): String = {
108
+  private def getTvSerieIdByPrimaryTitle(primaryTitle: String): Future[Option[String]] = {
111
     val source: Source[Map[String, String], _] = buildSource(inputFile = titleBasics)
109
     val source: Source[Map[String, String], _] = buildSource(inputFile = titleBasics)
112
 
110
 
113
-    var tvSerie: TvSeries = null
114
     val res: Future[Option[String]] = source
111
     val res: Future[Option[String]] = source
115
       .filter((rowMap: Map[String, String]) => {
112
       .filter((rowMap: Map[String, String]) => {
116
         rowMap.getOrElse(key = "primaryTitle", "") == primaryTitle
113
         rowMap.getOrElse(key = "primaryTitle", "") == primaryTitle
117
       })
114
       })
118
       .map((rowMap: Map[String, String]) => {
115
       .map((rowMap: Map[String, String]) => {
119
-        tvSerie = buildTvSerieModel(rowMap)
120
         rowMap.get("tconst")
116
         rowMap.get("tconst")
121
       })
117
       })
122
       .runWith(Sink.head)
118
       .runWith(Sink.head)
123
 
119
 
124
-    Await.result(res, 1 minutes)
125
-    val tconst: String = res.value.get.get.get
126
-    logger.info(s"TvSerie ID: $tconst, $tvSerie")
127
-    tconst
120
+    res
121
+
128
   }
122
   }
129
 
123
 
130
-  private def getListOfPersonsIDByTvSerieID(tconst: String): List[Option[String]]={
124
+  private def getListOfPersonsIDByTvSerieID(tvSerieID: Future[Option[String]]): Future[IndexedSeq[Option[String]]]={
131
     val source: Source[Map[String, String], _] = buildSource(inputFile = titlePrincipalsBasics)
125
     val source: Source[Map[String, String], _] = buildSource(inputFile = titlePrincipalsBasics)
132
 
126
 
133
-    val startTime : Long = System.currentTimeMillis()
134
     val res: Future[IndexedSeq[Option[String]]] = source
127
     val res: Future[IndexedSeq[Option[String]]] = source
135
       .filter((rowMaps: Map[String, String])=>{
128
       .filter((rowMaps: Map[String, String])=>{
136
-        rowMaps.getOrElse(key = "tconst", default = "")==tconst
129
+        rowMaps.getOrElse(key = "tconst", default = "")==tvSerieID.value.get.get.get
137
       })
130
       })
138
       .map((rowMap: Map[String, String])=>{
131
       .map((rowMap: Map[String, String])=>{
139
         rowMap.get(key = "nconst")
132
         rowMap.get(key = "nconst")
140
       })
133
       })
141
       .runWith(Sink.collection)
134
       .runWith(Sink.collection)
142
-      .andThen {
143
-        case Success(value) =>
144
-          logger.info(s"$value")
145
-          val elapsedTime : Long = (System.currentTimeMillis() - startTime)/1000
146
-          logger.info(s"success, elapsed time: $elapsedTime")
147
-
148
-        case Failure(error: Error) => logger.error(s"$error")
149
-      }
150
-
151
-    Await.result(res, 3 minutes)
152
-    val listPersonsID: List[Option[String]] = res.value.get.get.toList
153
 
135
 
154
-    listPersonsID
136
+    res
155
 
137
 
156
   }
138
   }
157
 
139
 
158
-  private def getListOfPersonsForTvSerie(nconstList: List[Option[String]]): List[Person] = {
140
+  private def getListOfPersonsForTvSerie(listPersonsIDs: Future[IndexedSeq[Option[String]]]): Future[IndexedSeq[Person]] = {
159
     val source: Source[Map[String, String], _] = buildSource(inputFile = nameBasics)
141
     val source: Source[Map[String, String], _] = buildSource(inputFile = nameBasics)
160
 
142
 
161
-    val startTime: Long = System.currentTimeMillis()
162
-
163
     val res : Future[IndexedSeq[Person]]=
143
     val res : Future[IndexedSeq[Person]]=
164
       source
144
       source
165
         .filter((rowMaps: Map[String, String])=>{
145
         .filter((rowMaps: Map[String, String])=>{
166
-          nconstList.contains(rowMaps.get(key = "nconst"))
146
+          listPersonsIDs.value.get.get.contains(rowMaps.get(key = "nconst"))
167
         })
147
         })
168
       .map((rowMap: Map[String, String])=>{
148
       .map((rowMap: Map[String, String])=>{
169
        buildPersonModel(rowMap)
149
        buildPersonModel(rowMap)
170
       })
150
       })
171
       .runWith(Sink.collection)
151
       .runWith(Sink.collection)
172
-      .andThen {
173
-        case Failure(exception) => logger.error(s"$exception")
174
-        case Success(value) =>
175
-          val elapsedTime: Long = (System.currentTimeMillis()-startTime)/1000
176
-          logger.info(s"END: Successfully, elapsed time: $elapsedTime")
177
-      }
178
-
179
-    Await.result(res, 4 minutes)
180
-    val listPersons: List[Person] = res.value.get.get.toList
181
-
182
-    listPersons
183
 
152
 
153
+    res
184
   }
154
   }
185
   override def getPersonsTeamForTvSerie(tvSeriePrimaryTitle: String):Unit={
155
   override def getPersonsTeamForTvSerie(tvSeriePrimaryTitle: String):Unit={
186
 
156
 
187
-    val tvSerieID: String = getTvSerieIdByPrimaryTitle(primaryTitle = tvSeriePrimaryTitle)
188
-    val listPersonIDs: List[Option[String]] = getListOfPersonsIDByTvSerieID(tconst = tvSerieID)
189
-    val personsTeam: List[Person] = getListOfPersonsForTvSerie(nconstList = listPersonIDs)
157
+    logger.info("STEP 1/3")
158
+    val tvSerieID = getTvSerieIdByPrimaryTitle(primaryTitle = tvSeriePrimaryTitle)
159
+    tvSerieID.onComplete({
160
+      case Failure(exception) => logger.error(s"$exception")
161
+      case Success(value: Option[String]) =>
162
+        logger.info(s"TvSerie ID: $value")
163
+        logger.info("END STEP 1/3")
164
+    })
165
+    logger.info("STEP 2/3")
166
+    val listPersonIDs = getListOfPersonsIDByTvSerieID(tvSerieID = tvSerieID)
167
+    listPersonIDs.onComplete({
168
+      case Failure(exception) => logger.error(s"$exception")
169
+      case Success(value) =>
170
+        value.toList.foreach((personID: Option[String])=>logger.info(s"Person ID:$personID"))
171
+        logger.info("END STEP 2/3")
172
+    })
190
 
173
 
191
-    logger.info(s"Team size:${personsTeam.size}")
192
-    personsTeam.foreach((person: Person)=>{
193
-      logger.info(s"${person.toString}")
174
+    logger.info("STEP 3/3")
175
+    val personsTeam= getListOfPersonsForTvSerie(listPersonsIDs = listPersonIDs)
176
+    personsTeam.onComplete({
177
+      case Failure(exception) => logger.error(s"$exception")
178
+      case Success(value) =>
179
+        logger.info(s"${value.toList}")
180
+        value.toList.foreach((person:Person)=>logger.info(s"${person.toString}"))
181
+        logger.info("END STEP 3/3")
194
     })
182
     })
195
 
183
 
196
   }
184
   }

+ 8
- 0
src/main/scala/fr/natan/akkastreamfileprocessingapi/service/CompleteFuture.scala View File

1
+package fr.natan.akkastreamfileprocessingapi.service
2
+
3
+import org.springframework.stereotype.Component
4
+
5
+@Component
6
+class CompleteFuture {
7
+
8
+}

Powered by TurnKey Linux.