Merge pull request #147 from hasadna/containerize-siri-collector

Containerize siri collector
hasadna · Mar 4, 2019 · 5fd2b16 · 5fd2b16
2 parents 8bb130b + 3eff01b
commit 5fd2b16
Show file tree

Hide file tree

Showing 27 changed files with 796 additions and 646 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -1,12 +1,33 @@
-language: python
-python: 
-  - "3.6"
-install:
-  - pip install boto3 codecov coverage
-# commands to run tests
-script: 
-  - cd ./gtfs/gtfs_utils
-  - python setup.py test
-  - cd ../retriever/
-  - coverage run s3_wrapper_setup.py pytest
-  - codecov
+matrix:
+  include:
+    - language: python
+      python:
+        - "3.6"
+      install:
+        - pip install boto3 codecov coverage
+      # commands to run tests
+      script:
+        - cd ./gtfs/gtfs_utils
+        - python setup.py test
+        - cd ../retriever/
+        - coverage run s3_wrapper_setup.py pytest
+        - codecov
+    - language: java
+      jdk: oraclejdk8
+      script:
+        - cd ./siri/siri_retriever/siri-0.1
+        - ./mvnw install -DskipTests=true -Dmaven.javadoc.skip=true -B -V
+        - docker login -u ${DOCKER_HUB_USERNAME} -p ${DOCKER_HUB_PASSWORD}
+        - docker build -t siri-collector .
+        - docker tag siri-collector openbus1/siri-collector:${TRAVIS_BRANCH}
+        - docker push openbus1/siri-collector:${TRAVIS_BRANCH}
+    - language: java
+      jdk: oraclejdk8
+      script:
+        - docker login -u ${DOCKER_HUB_USERNAME} -p ${DOCKER_HUB_PASSWORD}
+        - cd ./siri/gtfs_reader
+        - mvn install -DskipTests=true
+        - docker build -t gtfs-collector .
+        - docker tag gtfs-collector openbus1/gtfs-collector:${TRAVIS_BRANCH}
+        - docker push openbus1/gtfs-collector:${TRAVIS_BRANCH}
+
diff --git a/siri/docker-compose.debug.yml b/siri/docker-compose.debug.yml
@@ -0,0 +1,6 @@
+version: "3"
+
+services:
+  siri-collector:
+    extra_hosts:
+      - "siri.motrealtime.co.il:172.17.0.1"
diff --git a/siri/docker-compose.yml b/siri/docker-compose.yml
@@ -0,0 +1,29 @@
+version: "3"
+
+services:
+  siri-collector:
+    container_name: siri-collector
+    image: openbus1/siri-collector:containerize-siri-collector
+    environment:
+      - spring.profiles.active=production
+      - scheduler.data.file=/opt/hasadna/data/
+      - gtfs.schedules.location=/opt/hasadna/data/
+      - gtfs.dir.location=/opt/hasadna
+      - SIRI_OUTPUT_PATH=/opt/hasadna/data/siri_output
+    volumes:
+      - ${SCHEDULE_DATA_HOST_PATH}:/opt/hasadna/data
+    restart: always
+  gtfs-collector:
+    container_name: gtfs-collector
+    image: openbus1/gtfs-collector:containerize-siri-collector
+    environment:
+      - gtfs.agencies=[3,4,5,7,8,14,15,16,18,19,25,30,31,32]
+      - gtfs.dateOfLastDownload=2018-11-17
+      - gtfs.reschedule.url=http://siri-collector:8080/data/schedules/read/all
+      - gtfs.dir.location=/opt/hasadna/data/
+      - gtfs.schedules.location=/opt/hasadna/data/
+      - gtfs.RawFilesBackupDirectory=/opt/hasadna/data/
+    volumes:
+      - ${SCHEDULE_DATA_HOST_PATH}:/opt/hasadna/data
+    restart: always
+
diff --git a/siri/gtfs_reader/.classpath b/siri/gtfs_reader/.classpath
diff --git a/siri/gtfs_reader/.gitignore b/siri/gtfs_reader/.gitignore
@@ -1 +1,2 @@
 /target/
+.idea/
diff --git a/siri/gtfs_reader/.project b/siri/gtfs_reader/.project
diff --git a/siri/gtfs_reader/Dockerfile b/siri/gtfs_reader/Dockerfile
@@ -0,0 +1,13 @@
+FROM java:openjdk-8-jre-alpine
+
+LABEL vendor="hasadna"
+LABEL type="gtfs-collector"
+
+EXPOSE 8080
+RUN mkdir -p /opt/hasadna/data/schedule_data
+RUN mkdir -p /opt/hasadna/data/gtfs_schedules
+RUN mkdir -p /opt/hasadna/data/siri_output
+
+ADD target/siri-client-0.0.1-SNAPSHOT.jar /opt/hasadna/siri-client.jar
+
+CMD ["java", "-Xmx2g", "-jar", "/opt/hasadna/siri-client.jar"]
diff --git a/siri/gtfs_reader/pom.xml b/siri/gtfs_reader/pom.xml
@@ -9,13 +9,37 @@
 
     <name>siri-client</name>
 
+  <description>OpenBus Java Data Retriever</description>
+
+  <parent>
+    <groupId>org.springframework.boot</groupId>
+    <artifactId>spring-boot-starter-parent</artifactId>
+    <version>2.0.2.RELEASE</version>
+    <relativePath/> <!-- lookup parent from repository -->
+  </parent>
+
     <properties>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
         <maven.compiler.source>1.8</maven.compiler.source>
         <maven.compiler.target>1.8</maven.compiler.target>
+      <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+      <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
+      <java.version>1.8</java.version>
     </properties>
 
     <dependencies>
+      <dependency>
+        <groupId>org.springframework.boot</groupId>
+        <artifactId>spring-boot-starter-actuator</artifactId>
+      </dependency>
+
+      <!-- https://mvnrepository.com/artifact/commons-net/commons-net -->
+      <dependency>
+        <groupId>commons-net</groupId>
+        <artifactId>commons-net</artifactId>
+        <version>3.6</version>
+      </dependency>
+
         <dependency>
             <groupId>junit</groupId>
             <artifactId>junit</artifactId>
@@ -41,13 +65,35 @@
             <version>1.2.3</version>
         </dependency>
 
+
+      <dependency>
+        <groupId>org.springframework.boot</groupId>
+        <artifactId>spring-boot-starter-web</artifactId>
+      </dependency>
+
+      <dependency>
+        <groupId>org.springframework.retry</groupId>
+        <artifactId>spring-retry</artifactId>
+        <version>1.2.2.RELEASE</version>
+      </dependency>
+      <dependency>	<!--  required for Spring-Retry -->
+        <groupId>org.springframework.boot</groupId>
+        <artifactId>spring-boot-starter-aop</artifactId>
+      </dependency>
+
         <dependency>
             <groupId>com.fasterxml.jackson.core</groupId>
             <artifactId>jackson-databind</artifactId>
             <version>2.9.6</version>
         </dependency>
 
-        <dependency>
+
+  <dependency>
+      <groupId>org.springframework.boot</groupId>
+      <artifactId>spring-boot-starter-test</artifactId>
+      <scope>test</scope>
+    </dependency>
+      <dependency>
             <groupId>net.jodah</groupId>
             <artifactId>failsafe</artifactId>
             <version>1.1.0</version>
@@ -56,55 +102,14 @@
 
     </dependencies>
 
+  <build>
+    <plugins>
+      <plugin>
+        <groupId>org.springframework.boot</groupId>
+        <artifactId>spring-boot-maven-plugin</artifactId>
+      </plugin>
+    </plugins>
+  </build>
+
 
-    <build>
-        <plugins>
-            <plugin>
-                <groupId>org.apache.maven.plugins</groupId>
-                <artifactId>maven-jar-plugin</artifactId>
-                <version>3.1.0</version>
-                <configuration>
-                    <archive>
-                        <manifest>
-                            <addClasspath>true</addClasspath>
-                            <classpathPrefix>lib/</classpathPrefix>
-                            <mainClass>il.org.hasadna.siri_client.gtfs.main.DefaultGtfsQueryBasedOnFtp</mainClass>
-                        </manifest>
-                    </archive>
-                </configuration>
-            </plugin>
-
-
-
-            <plugin>
-                <artifactId>maven-assembly-plugin</artifactId>
-                <version>3.1.0</version>
-                <configuration>
-                    <descriptorRefs>
-                        <descriptorRef>jar-with-dependencies</descriptorRef>
-                    </descriptorRefs>
-
-                    <archive>
-                        <manifest>
-                            <mainClass>il.org.hasadna.siri_client.gtfs.main.DefaultGtfsQueryBasedOnFtp</mainClass>
-                        </manifest>
-
-                        <manifestEntries>
-                            <Class-Path>.</Class-Path>
-                        </manifestEntries>
-                    </archive>
-                </configuration>
-                <executions>
-                    <execution>
-                        <id>make-assembly</id> <!-- this is used for inheritance merges -->
-                        <phase>package</phase> <!-- bind to the packaging phase -->
-                        <goals>
-                            <goal>single</goal>
-                        </goals>
-                    </execution>
-                </executions>
-            </plugin>
-
-        </plugins>
-    </build>
 </project>
diff --git a/.../siri_client/gtfs/analysis/MakatFile.java → .../siri_client/gtfs/analysis/MakatFile.java b/.../siri_client/gtfs/analysis/MakatFile.java → .../siri_client/gtfs/analysis/MakatFile.java
diff --git a/..._reader/src/main/java/il/org/hasadna/siri_client/gtfs/analysis/SchedulingDataCreator.java b/..._reader/src/main/java/il/org/hasadna/siri_client/gtfs/analysis/SchedulingDataCreator.java
@@ -4,9 +4,6 @@
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.SerializationFeature;
 import il.org.hasadna.siri_client.gtfs.crud.Route;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
 import java.io.IOException;
 import java.nio.charset.Charset;
 import java.nio.file.Files;
@@ -16,10 +13,23 @@
 import java.time.LocalTime;
 import java.time.format.DateTimeFormatter;
 import java.time.format.TextStyle;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.TreeMap;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 public class SchedulingDataCreator {