Siri/results file v2

siri/gtfs_reader/src/main/java/il/org/hasadna/siri_client/gtfs/main/GtfsCollectorConfiguration.java

@@ -12,12 +12,16 @@
 import java.util.Collections;
 import java.util.List;
 import java.util.concurrent.TimeUnit;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Component;
 
 @Component
 public class GtfsCollectorConfiguration {
 
+  private static Logger logger = LoggerFactory.getLogger(GtfsCollectorConfiguration.class);
+
   private static final int DEFAULT_BACKUP_FILES_STORE_TIME_IN_DAYS = 5;
   private final LocalTime DEFAULT_WHEN_TO_DOWNLOAD = LocalTime.of(3, 30);
   private final LocalDate DEFAULT_DATE_OF_LAST_DOWNLOAD = LocalDate.of(2000, 1, 1);
@@ -59,6 +63,7 @@ public void setGtfsRawFilesBackupDirectory(String gtfsRawFilesBackupDirectory) {
     }
 
     GtfsCollectorConfiguration.gtfsRawFilesBackupDirectory = gtfsRawFilesBackupDirectory;
+    logger.info("set gtfsRawFilesBackupDirectory={}", gtfsRawFilesBackupDirectory);
   }
 
   @Value("${gtfs.gtfsDownloadDisabled:false}")
@@ -187,4 +192,4 @@ private List<String> parseList(String s) {
     String[] items = s.split(",");
     return new ArrayList<>(Arrays.asList(items));
   }
-}
+}

siri/siri_retriever/siri-0.1/src/main/java/org/hasadna/bus/service/SiriConsumeServiceImpl.java

@@ -19,6 +19,7 @@
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
+import org.springframework.context.annotation.ImportResource;
 import org.springframework.context.annotation.Profile;
 import org.springframework.http.HttpEntity;
 import org.springframework.http.HttpHeaders;
@@ -27,7 +28,7 @@
 import org.springframework.web.client.RestTemplate;
 
 @Component
-@Profile({"production", "integrationTests"})
+@Profile({"production", "test2"})
 public class SiriConsumeServiceImpl implements SiriConsumeService {
 
     @Value("${number.of.intervals:12}")
@@ -43,6 +44,7 @@ public class SiriConsumeServiceImpl implements SiriConsumeService {
     private DatadogMeterRegistry registry;
 
     protected final Logger logger = LoggerFactory.getLogger(this.getClass());
+    protected final Logger responseLogger = LoggerFactory.getLogger("RESPONSE_LOGGER");
 
     final String SIRI_SERVICES_URL = "http://siri.motrealtime.co.il:8081/Siri/SiriServices";
 
@@ -97,11 +99,17 @@ public GetStopMonitoringServiceResponse retrieveSiri(String stopCode, String pre
         if (content == null) {
             return null;
         }
-        logger.debug("lineRef={}, stopCode={}, previewInterval={}", lineRef, stopCode, previewInterval);
-        logger.trace(" response={}", content);
+        try {
+            logger.debug("lineRef={}, stopCode={}, previewInterval={}", lineRef, stopCode, previewInterval);
+            //logger.trace(" response={}", content);
+            //responseLogger.warn("{},{}", lineRef, content.replace("\n", ""));
+        }
+        catch (Exception ex) {
+            // absorb
+        }
 
         content = SoapUtils.removeSoapEnvelope(content);
-        logger.trace(content);
+        //logger.trace(content);
 
         //unmarshall XML to object
         GetStopMonitoringServiceResponse response = unmarshalXml(content);

siri/siri_retriever/siri-0.1/src/main/java/org/hasadna/bus/service/SiriParseServiceImpl.java

@@ -5,6 +5,7 @@
 import org.slf4j.LoggerFactory;
 import org.springframework.stereotype.Component;
 import uk.org.siri.siri.MonitoredStopVisitStructure;
+import uk.org.siri.siri.MonitoredVehicleJourneyStructure;
 import uk.org.siri.siri.StopMonitoringDeliveriesStructure;
 import uk.org.siri.siri.StopMonitoringDeliveryStructure;
 
@@ -43,7 +44,7 @@ public Optional<String> parseShortSummary(GetStopMonitoringServiceResponse sm) {
                     }
                     Date departureTime = visit.getMonitoredVehicleJourney().getOriginAimedDepartureTime();
                     licensePlates.add(licensePlate);
-                    visits.put(formatDate(departureTime) + "/" + licensePlate, visit);
+                    visits.putIfAbsent(((departureTime==null)?"99:99":formatDate(departureTime)) + "/" + licensePlate, visit);
                 }
             }
             String s = "";
@@ -53,7 +54,8 @@ public Optional<String> parseShortSummary(GetStopMonitoringServiceResponse sm) {
                 MonitoredStopVisitStructure visit = visits.get(key);
                 String licensePlate = visit.getMonitoredVehicleJourney().getVehicleRef().getValue();
                 String lineRef = visit.getMonitoredVehicleJourney().getLineRef().getValue();
-                visit.getMonitoredVehicleJourney().getMonitoredCall().isVehicleAtStop();
+                // isVehicleAtStop = visit.getMonitoredVehicleJourney().getMonitoredCall().isVehicleAtStop();
+                String vehicleAtStopStr = parseVehicleAtStop(visit.getMonitoredVehicleJourney());   // "0"=false, "1"=true, "2"=did not appear (or null)
                 Date expectedArrivalTime = visit.getMonitoredVehicleJourney().getMonitoredCall().getExpectedArrivalTime();
                 BigDecimal lon = BigDecimal.ZERO;
                 BigDecimal lat = BigDecimal.ZERO;
@@ -69,7 +71,9 @@ public Optional<String> parseShortSummary(GetStopMonitoringServiceResponse sm) {
                 Date departureTime = visit.getMonitoredVehicleJourney().getOriginAimedDepartureTime();
                 String operatorRef = visit.getMonitoredVehicleJourney().getOperatorRef().getValue();
                 String journeyRef = visit.getMonitoredVehicleJourney().getFramedVehicleJourneyRef().getDatedVehicleJourneyRef();
-                String rep = stringRepresentation(lineRef, lineName, recordedAt, expectedArrivalTime, licensePlate, lat, lon, departureTime, operatorRef, journeyRef, responseTimestamp);
+                String dataFrameRef = visit.getMonitoredVehicleJourney().getFramedVehicleJourneyRef().getDataFrameRef().getValue();
+                String stopPointRef = visit.getMonitoredVehicleJourney().getMonitoredCall().getStopPointRef().getValue();
+                String rep = stringRepresentation(lineRef, lineName, recordedAt, expectedArrivalTime, licensePlate, lat, lon, departureTime, operatorRef, journeyRef, responseTimestamp, dataFrameRef, stopPointRef, vehicleAtStopStr);
                 s = s + rep + "\n";
             }
             if (!visits.isEmpty()) {
@@ -86,16 +90,50 @@ public Optional<String> parseShortSummary(GetStopMonitoringServiceResponse sm) {
         }
     }
 
+    /**
+     * r
+     * @param monitoredVehicleJourney
+     * @return  isVehicleAtStop: 0=false, 1=true, 2=did not appear (or null, or some unknown exception during parsing)
+     */
+    private String parseVehicleAtStop(MonitoredVehicleJourneyStructure monitoredVehicleJourney) {
+        if ((monitoredVehicleJourney == null) || (monitoredVehicleJourney.getMonitoredCall() == null)) {
+            return "2";
+        }
+        String defaultVal = "2";
+        try {
+            Boolean isVehicleAtStop = monitoredVehicleJourney.getMonitoredCall().isVehicleAtStop();
+            String vehicleAtStopStr = defaultVal;
+            if (isVehicleAtStop != null) {
+                if (isVehicleAtStop) {
+                    vehicleAtStopStr = "1";
+                } else {
+                    vehicleAtStopStr = "0";
+                }
+            } else {
+                vehicleAtStopStr = "2";
+            }
+            return vehicleAtStopStr;
+        }
+        catch (Exception ex) {
+            // silently absorb (logging here might cause several millions lines in log file every day)
+            return defaultVal;
+        }
+    }
 
-    private String stringRepresentation(String lineRef, String lineName, Date recordedAt, Date expectedArrivalTime, String licensePlate, BigDecimal lon, BigDecimal lat, Date departureTime, String operatorRef, String journeyRef, String responseTimestamp) {
-        String s = MessageFormat.format("{10},[line {0} v {1} oad {6} ea {2}],{7},{8},{0},{9},{6},{1},{2},{3},{4},{5}",
+    private String stringRepresentation(String lineRef, String lineName, Date recordedAt, Date expectedArrivalTime,
+                                        String licensePlate, BigDecimal lon, BigDecimal lat, Date departureTime,
+                                        String operatorRef, String journeyRef, String responseTimestamp,
+                                        String dataFrameRef, String stopPointRef, String vehicleAtStopStr) {
+        String s = MessageFormat.format("{10},[line {0} v {1} oad {15} ea {11}],{7},{8},{0},{9},{6},{1},{2},{3},{4},{5},{12},{13},{14},v2",
                 lineName, licensePlate,
-                formatTimeHHMM(expectedArrivalTime),
-                formatTime(recordedAt),
+                formatDate(expectedArrivalTime),    // expectedArrivalTime should include both date and time - // <ns3:ExpectedArrivalTime>2019-04-01T21:14:00.000+03:00</ns3:ExpectedArrivalTime>
+                formatDate(recordedAt),             // recordedAt should include both date and time
                 lon.toString(), lat.toString(),
-                formatTimeHHMM(departureTime),
-                operatorRef, lineRef, journeyRef, responseTimestamp
-                );
+                formatDate(departureTime),          // OriginAimedDeparture should include both date and time - // <ns3:OriginAimedDepartureTime>2019-04-01T20:00:00.000+03:00</ns3:OriginAimedDepartureTime>
+                operatorRef, lineRef, journeyRef, responseTimestamp,
+                formatTimeHHMM(expectedArrivalTime),    // ea as time only, for the free text part
+                dataFrameRef, stopPointRef, vehicleAtStopStr, formatTimeHHMM(departureTime)
+        );
         return s ;
     }
 
@@ -116,3 +154,30 @@ private String formatTimeHHMM(Date date) {
     }
 
 }
+
+/*
+<ns3:MonitoredVehicleJourney><ns3:LineRef>5962</ns3:LineRef><ns3:DirectionRef>1</ns3:DirectionRef><ns3:FramedVehicleJourneyRef><ns3:DataFrameRef>2019-04-01</ns3:DataFrameRef><ns3:DatedVehicleJourneyRef>31439164</ns3:DatedVehicleJourneyRef></ns3:FramedVehicleJourneyRef><ns3:PublishedLineName>174</ns3:PublishedLineName><ns3:OperatorRef>4</ns3:OperatorRef><ns3:DestinationRef>60664</ns3:DestinationRef><ns3:OriginAimedDepartureTime>2019-04-01T20:36:00.000+03:00</ns3:OriginAimedDepartureTime><ns3:VehicleLocation><ns3:Longitude>35.29014587402344</ns3:Longitude><ns3:Latitude>31.76877212524414</ns3:Latitude></ns3:VehicleLocation><ns3:VehicleRef>7558569</ns3:VehicleRef><ns3:MonitoredCall><ns3:StopPointRef>60664</ns3:StopPointRef><ns3:ExpectedArrivalTime>2019-04-01T21:18:00.000+03:00</ns3:ExpectedArrivalTime></ns3:MonitoredCall></ns3:MonitoredVehicleJourney>
+
+<ns3:MonitoredVehicleJourney>
+    <ns3:LineRef>5962</ns3:LineRef>
+    <ns3:DirectionRef>1</ns3:DirectionRef>
+    <ns3:FramedVehicleJourneyRef>
+        <ns3:DataFrameRef>2019-04-01</ns3:DataFrameRef>
+        <ns3:DatedVehicleJourneyRef>31439164</ns3:DatedVehicleJourneyRef>
+    </ns3:FramedVehicleJourneyRef>
+    <ns3:PublishedLineName>174</ns3:PublishedLineName>
+    <ns3:OperatorRef>4</ns3:OperatorRef>
+    <ns3:DestinationRef>60664</ns3:DestinationRef>
+    <ns3:OriginAimedDepartureTime>2019-04-01T20:36:00.000+03:00</ns3:OriginAimedDepartureTime>
+    <ns3:VehicleLocation>
+        <ns3:Longitude>35.29014587402344</ns3:Longitude>
+        <ns3:Latitude>31.76877212524414</ns3:Latitude>
+    </ns3:VehicleLocation>
+    <ns3:VehicleRef>7558569</ns3:VehicleRef>
+    <ns3:MonitoredCall>
+        <ns3:StopPointRef>60664</ns3:StopPointRef>
+        <ns3:ExpectedArrivalTime>2019-04-01T21:18:00.000+03:00</ns3:ExpectedArrivalTime>
+    </ns3:MonitoredCall>
+</ns3:MonitoredVehicleJourney>
+
+*/

siri/siri_retriever/siri-0.1/src/main/java/org/hasadna/bus/service/SiriPersistFileService.java

@@ -2,7 +2,12 @@
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.boot.web.client.RestTemplateBuilder;
+import org.springframework.http.HttpEntity;
 import org.springframework.stereotype.Component;
+import org.springframework.util.StringUtils;
 
 @Component
 public class SiriPersistFileService implements SiriPersistService {
@@ -12,10 +17,33 @@ public class SiriPersistFileService implements SiriPersistService {
     // this logger should be defined to a specific file - see logback.xml
     protected final Logger fileLogger = LoggerFactory.getLogger("SiriRealTimeData");
 
+    @Value("${external.send:false}")
+    boolean sendExternal;
+
+    @Value("${external.url}")
+    String url;
+
+    @Autowired
+    RestTemplateBuilder restTemplateBuilder;
+
     @Override
     public void persistShortSummary(String summary) {
         logger.trace("persisting...");
         fileLogger.info(summary);
+        sendToExternal(summary);
         logger.trace("          ...Done");
     }
+
+    private void sendToExternal(String str) {
+        try {
+            if (sendExternal && !StringUtils.isEmpty(str)) {
+                restTemplateBuilder.build().postForEntity(url, new HttpEntity<String>(str, null), String.class);
+            }
+        }
+        catch (Exception ex) {
+            logger.debug("absorbing ", ex);
+        }
+
+    }
 }
+

siri/siri_retriever/siri-0.1/src/main/resources/application-production.properties

@@ -12,4 +12,10 @@ scheduler.data.file=/home/evyatar/logs/
 scheduler.enable=true
 
 # change to false if you want to disable DataDog Monitoring
-management.metrics.export.datadog.enabled=true
+management.metrics.export.datadog.enabled=true
+
+#change to error to disable writing response xml to debug log
+logging.level.RESPONSE_LOGGER=error
+
+external.url=https://gtfs-web-app.appspot.com/siri/raw/data
+external.send=false

siri/siri_retriever/siri-0.1/src/main/resources/logback-ignore-test.xml

@@ -29,14 +29,14 @@
 
     <!-- This file is machine readable -->
     <appender name="FILE-SiriRealTimeData" class="ch.qos.logback.core.rolling.RollingFileAppender">
-        <file>${DEV_HOME}/siri_rt_data.log</file>
+        <file>${DEV_HOME}/siri_rt_data_v2.log</file>
         <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
             <Pattern>%msg%n</Pattern>
         </encoder>
 
         <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
             <!-- rollover daily + zip -->
-            <fileNamePattern>${DEV_HOME}/archived/siri_rt_data.%d{yyyy-MM-dd}.%i.log.gz</fileNamePattern>
+            <fileNamePattern>${DEV_HOME}/archived/siri_rt_data_v2.%d{yyyy-MM-dd}.%i.log.gz</fileNamePattern>
             <timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
                 <maxFileSize>100MB</maxFileSize>
             </timeBasedFileNamingAndTriggeringPolicy>

siri/siri_retriever/siri-0.1/src/main/resources/logback.xml

@@ -27,16 +27,16 @@
 
   </appender>
 
-  <!-- This file is machine readable -->
+  <!-- This file is the siri_rt_log! if you change, make sure it is still written correctly -->
   <appender name="FILE-SiriRealTimeData" class="ch.qos.logback.core.rolling.RollingFileAppender">
-    <file>${SIRI_OUTPUT_PATH}/siri_rt_data.log</file>
+    <file>${SIRI_OUTPUT_PATH}/siri_rt_data_v2.log</file>
     <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
       <Pattern>%msg%n</Pattern>
     </encoder>
 
     <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
       <!-- rollover daily + zip -->
-      <fileNamePattern>${SIRI_OUTPUT_PATH}/archived/siri_rt_data.%d{yyyy-MM-dd}.%i.log.gz</fileNamePattern>
+      <fileNamePattern>${SIRI_OUTPUT_PATH}/archived/siri_rt_data_v2.%d{yyyy-MM-dd}.%i.log.gz</fileNamePattern>
       <timeBasedFileNamingAndTriggeringPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedFNATP">
         <maxFileSize>100MB</maxFileSize>
       </timeBasedFileNamingAndTriggeringPolicy>
@@ -49,13 +49,16 @@
   <logger name="SiriRealTimeData" level="info" additivity="false">
     <appender-ref ref="FILE-SiriRealTimeData"/>
   </logger>
-  <logger name="org.hasadna" level="info" additivity="false">
+  <logger name="RESPONSE_LOGGER" level="error" additivity="false">
     <appender-ref ref="FILE-debug"/>
-    <appender-ref ref="STDOUT"/>
+  </logger>
+  <logger name="org.hasadna" level="warn" additivity="false">
+    <appender-ref ref="FILE-debug"/>
+    <!--<appender-ref ref="STDOUT"/>-->
   </logger>
 
   <root level="info">
-    <appender-ref ref="STDOUT"/>
+    <appender-ref ref="FILE-debug"/>
   </root>
 
 </configuration>

siri/siri_retriever/siri-0.1/src/test/java/org/hasadna/bus/service/NullClock.java

@@ -0,0 +1,16 @@
+package org.hasadna.bus.service;
+
+import org.springframework.stereotype.Component;
+
+@Component
+public class NullClock implements io.micrometer.core.instrument.Clock {
+    @Override
+    public long wallTime() {
+        return 0;
+    }
+
+    @Override
+    public long monotonicTime() {
+        return 0;
+    }
+}

siri/siri_retriever/siri-0.1/src/test/java/org/hasadna/bus/service/NullDatadogConfig.java

@@ -0,0 +1,42 @@
+package org.hasadna.bus.service;
+
+import io.micrometer.datadog.DatadogConfig;
+import org.springframework.stereotype.Component;
+
+@Component
+public class NullDatadogConfig implements DatadogConfig {
+    @Override
+    public String prefix() {
+        return null;
+    }
+
+    @Override
+    public String get(String s) {
+        return null;
+    }
+
+    @Override
+    public String apiKey() {
+        return "";
+    }
+
+    @Override
+    public String applicationKey() {
+        return null;
+    }
+
+    @Override
+    public String hostTag() {
+        return null;
+    }
+
+    @Override
+    public String uri() {
+        return "http://www.example.com";
+    }
+
+    @Override
+    public boolean descriptions() {
+        return false;
+    }
+}

siri/siri_retriever/siri-0.1/src/test/java/org/hasadna/bus/service/NullDatadogMeterRegistry.java

@@ -0,0 +1,13 @@
+package org.hasadna.bus.service;
+
+import io.micrometer.core.instrument.Clock;
+import io.micrometer.datadog.DatadogConfig;
+import io.micrometer.datadog.DatadogMeterRegistry;
+import org.springframework.stereotype.Component;
+
+@Component
+public class NullDatadogMeterRegistry extends DatadogMeterRegistry {
+    public NullDatadogMeterRegistry(DatadogConfig config, Clock clock) {
+        super(config, clock);
+    }
+}