Skip to content

Commit

Permalink
fix conflicts with master
Browse files Browse the repository at this point in the history
  • Loading branch information
eldadru committed Mar 4, 2019
2 parents 6e693be + cdba00a commit 7d5d2d6
Show file tree
Hide file tree
Showing 15 changed files with 1,233 additions and 259 deletions.
1 change: 1 addition & 0 deletions complains/README.md
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
foo
47 changes: 47 additions & 0 deletions complains/frontend/index.html
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
<!DOCTYPE html>
<html lang="en">
<head>
<title>תלונות תחבורה ציבורית</title>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.7/css/bootstrap.min.css">
<link rel="stylesheet" href="https://unpkg.com/[email protected]/dist/leaflet.css"
integrity="sha512-puBpdR0798OZvTTbP4A8Ix/l+A4dHDD0DGqYW6RQ+9jxkRFclaxxQb/SJAWZfWAkuyeQUytO7+7N4QKrDh+drA=="
crossorigin=""/>
<script src="https://unpkg.com/[email protected]/dist/leaflet.js"
integrity="sha512-nMMmRyTVoLYqjP9hrbed9S+FzjZHW5gY1TWCHA5ckwXZBadntCNs8kEqAWdrb9O7rxbCaA4lKTIWjDXZxflOcA=="
crossorigin=""></script>

<link rel="stylesheet" type="text/css" href="style.css">
</head>
<body dir="rtl">
<div class="container">
<div class="jumbotron">
<h1>אתר תלונות תחבורה ציבורית</h1>
<p>כאן תוכלו להתלונן על התנהלות של התחבורה הציבורית בישראל</p>
</div>

<div class="row">
<div id="mapid"></div>
</div>
<div class="row">
<div class="col-sm-4">
<h3>Column 1</h3>
<p>Lorem ipsum dolor sit amet, consectetur adipisicing elit...</p>
<p>Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris...</p>
</div>
<div class="col-sm-4">
<h3>Column 2</h3>
<p>Lorem ipsum dolor sit amet, consectetur adipisicing elit...</p>
<p>Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris...</p>
</div>
<div class="col-sm-4">
<h3>Column 3</h3>
<p>Lorem ipsum dolor sit amet, consectetur adipisicing elit...</p>
<p>Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris...</p>
</div>
</div>
</div>
</body>
<script src="script.js"></script>
</html>
14 changes: 14 additions & 0 deletions complains/frontend/script.js
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
var mymap = L.map('mapid').setView([32.073608, 34.790128], 18);

mymap.scrollWheelZoom.disable();

// TODO: Create new `access_token`
L.tileLayer('https://api.tiles.mapbox.com/v4/{id}/{z}/{x}/{y}.png?access_token=pk.eyJ1IjoibWFwYm94IiwiYSI6ImNpejY4NXVycTA2emYycXBndHRqcmZ3N3gifQ.rJcFIG214AriISLbB6B5aw', {
maxZoom: 18,
attribution: 'Map data &copy; <a href="https://www.openstreetmap.org/">OpenStreetMap</a> contributors, ' +
'<a href="https://creativecommons.org/licenses/by-sa/2.0/">CC-BY-SA</a>, ' +
'Imagery © <a href="https://www.mapbox.com/">Mapbox</a>',
id: 'mapbox.streets'
}).addTo(mymap);

L.marker([32.073539, 34.789106]).addTo(mymap);
4 changes: 4 additions & 0 deletions complains/frontend/style.css
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
#mapid {
height: 400px;
width: 100%;
}
67 changes: 67 additions & 0 deletions pipeline/create_siri_csv_splunk.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
from glob import glob
import os
import pandas as pd
import datetime
import re
from os.path import join

def timestr_to_seconds(x, *, only_mins=False):
try:
hms = x.str.split(':', expand=True)
if not only_mins:
result = hms.iloc[:,0].astype(int) * 3600 + hms.iloc[:,1].astype(int) * 60 + hms.iloc[:,2].astype(int)
else:
result = hms.iloc[:,0].astype(int) * 3600 + hms.iloc[:,1].astype(int) * 60
except:
result = np.nan

return result

def create_trip_df(path, drop=['timestamp', 'desc'],
convert_timestr_to_seconds=True, add_date=True,
add_trailing_zeros=True):
header = ["timestamp", "desc", "agency_id",
"route_id", "route_short_name", "service_id",
"planned_start_time", "bus_id", "predicted_end_time",
"time_recorded", "lat", "lon"]
date = datetime.datetime.strptime(re.findall('siri_rt_data\\.([^\\.]+)\\.\\d+\\.log', path)[0], '%Y-%m-%d')
df = pd.read_csv(path, header=None, error_bad_lines=False)
df.columns = header
if drop is not None:
df = df.drop(drop, axis=1)
df = (df.assign(agency_id = lambda x: x.agency_id.astype(int))
.assign(service_id = lambda x: x.service_id.astype(int))
.assign(route_id = lambda x: x.route_id.astype(int))
.assign(lat = lambda x: x.lat.astype(float))
.assign(lon = lambda x: x.lon.astype(float)))
if convert_timestr_to_seconds:
df = (df.assign(planned_start_time = lambda x: timestr_to_seconds(x.planned_start_time, only_mins=True))
.assign(predicted_end_time = lambda x: timestr_to_seconds(x.predicted_end_time, only_mins=True))
.assign(time_recorded = lambda x: timestr_to_seconds(x.time_recorded)))
if add_date:
df = (df.assign(date = date))
if add_trailing_zeros:
df = (df
.assign(planned_start_time = lambda x: x.planned_start_time+':00')
.assign(predicted_end_time = lambda x: x.predicted_end_time+':00'))

return df

FOLDER = '<SIRI_LOGS_FOLDER>'
out_folder = '<SPLUNK_SIRI_CSV_INPUTS_FOLDER>'
if not os.path.exists(out_folder):
os.mkdir(out_folder)

for file in glob(FOLDER+'/*2019*.log.gz'):
base = '.'.join(os.path.basename(file).split('.')[:-2])
out_path = os.path.join(out_folder, base+'.csv.gz')
if not os.path.exists(out_path):
#out_path = os.path.join(out_folder, base+'_FIXED.csv.gz')
print(file)
try:
df = create_trip_df(file, drop=['desc'], convert_timestr_to_seconds=False)
except Exception as e:
print(str(e))
#df.to_parquet(bn + '.parq')
#os.remove(file)
df.to_csv(out_path, compression='gzip', index=False)
139 changes: 83 additions & 56 deletions siri/gtfs_reader/pom.xml
View file
Original file line number Diff line number Diff line change
@@ -1,13 +1,13 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>

<groupId>il.org.hasadna</groupId>
<artifactId>siri-client</artifactId>
<version>0.0.1-SNAPSHOT</version>
<packaging>jar</packaging>
<groupId>il.org.hasadna</groupId>
<artifactId>siri-client</artifactId>
<version>0.0.1-SNAPSHOT</version>
<packaging>jar</packaging>

<name>siri-client</name>
<name>siri-client</name>

<description>OpenBus Java Data Retriever</description>

Expand All @@ -18,62 +18,89 @@
<relativePath/> <!-- lookup parent from repository -->
</parent>

<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<java.version>1.8</java.version>
</properties>

<dependencies>

<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-actuator</artifactId>
</dependency>

<!-- https://mvnrepository.com/artifact/commons-net/commons-net -->
<dependency>
<groupId>commons-net</groupId>
<artifactId>commons-net</artifactId>
<version>3.6</version>
</dependency>

<!-- https://mvnrepository.com/artifact/nl.jqno.equalsverifier/equalsverifier -->
<dependency>
<groupId>nl.jqno.equalsverifier</groupId>
<artifactId>equalsverifier</artifactId>
<version>3.0.3</version>
<scope>test</scope>
</dependency>

<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
</dependency>

<dependency>
<groupId>org.springframework.retry</groupId>
<artifactId>spring-retry</artifactId>
<version>1.2.2.RELEASE</version>
</dependency>
<dependency> <!-- required for Spring-Retry -->
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-aop</artifactId>
</dependency>
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<maven.compiler.source>1.8</maven.compiler.source>
<maven.compiler.target>1.8</maven.compiler.target>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<java.version>1.8</java.version>
</properties>

<dependencies>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-actuator</artifactId>
</dependency>

<!-- https://mvnrepository.com/artifact/commons-net/commons-net -->
<dependency>
<groupId>commons-net</groupId>
<artifactId>commons-net</artifactId>
<version>3.6</version>
</dependency>

<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.11</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>nl.jqno.equalsverifier</groupId>
<artifactId>equalsverifier</artifactId>
<version>2.4.6</version>
<scope>test</scope>
</dependency>

<dependency>
<groupId>commons-net</groupId>
<artifactId>commons-net</artifactId>
<version>3.6</version>
</dependency>

<dependency>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
<version>1.2.3</version>
</dependency>


<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
</dependency>

<dependency>
<groupId>org.springframework.retry</groupId>
<artifactId>spring-retry</artifactId>
<version>1.2.2.RELEASE</version>
</dependency>
<dependency> <!-- required for Spring-Retry -->
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-aop</artifactId>
</dependency>

<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
<version>2.9.5</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
<version>2.9.6</version>
</dependency>


<dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<dependency>
<groupId>net.jodah</groupId>
<artifactId>failsafe</artifactId>
<version>1.1.0</version>
</dependency>


</dependencies>

<build>
<plugins>
Expand Down
15 changes: 7 additions & 8 deletions ..._reader/src/main/java/il/org/hasadna/siri_client/gtfs/analysis/GtfsDataManipulations.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -108,21 +108,20 @@ private Map<String,Route> filterRoutes(Map<String, Trip> trips) throws IOExcepti
filter(r -> routesOfTrips.contains(r.getRouteId())).
collect(Collectors.toMap(Route::getRouteId, r -> r));
return routesByRouteId;
// Map<String, List<Trip>> tripsByRouteId = trips.values().stream().collect(Collectors.groupingBy(Trip::getRouteId));
// Map<String, Route> routesByTripId = new HashMap<>();
// for (String routeId : tripsByRouteId.keySet()) {
// tripsByRouteId.get(routeId).forEach( trip ->
// routesByTripId.put(trip.getTripId(), routesByRouteId.get(routeId))
// );
// }
// return routesByTripId;
}

public Collection<GtfsRecord> combine(LocalDate date) throws IOException {
filterGtfs(date);
return getTrips().values().stream().map(this::createGtfsRecord).collect(Collectors.toList());
}

public Collection<GtfsRecord> combineForSpecificRoute(LocalDate date, String routeId) throws IOException {
filterGtfs(date);
return getTrips().values().stream()
.filter(trip-> trip.getRouteId().equals(routeId))
.map(this::createGtfsRecord).collect(Collectors.toList());
}

private GtfsRecord createGtfsRecord(Trip currTrip) {
Calendar currCalendar = getCalendars().get(currTrip.getServiceId());

Expand Down
Loading

0 comments on commit 7d5d2d6

Please sign in to comment.