-
Notifications
You must be signed in to change notification settings - Fork 0
/
fetch.rb
executable file
·76 lines (61 loc) · 1.61 KB
/
fetch.rb
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
#!/usr/bin/env ruby
require 'net/http'
require 'uri'
require 'json'
require 'cgi'
require 'nokogiri'
def get_item(id)
count = 0
begin
puts "Getting item with ID #{id}."
result = Net::HTTP.get(URI("https://hacker-news.firebaseio.com/v0/item/#{id}.json?print=pretty"))
obj = JSON.parse(result)
obj['text'] = Nokogiri::HTML::fragment(obj['text']).to_html
obj['by'] = CGI::escape(obj['by']) if obj['by'] != nil
obj
rescue => e
puts "Error getting item with ID #{id}."
puts e
puts result
count += 1
retry if count <= 3
nil
end
end
def get_comments(id)
comments = []
comment = get_item(id)
return comments if ! comment || comment['dead'] || comment['deleted']
comments << comment
if comment['kids']
comment['comments'] = []
comment['kids'].each do |kid|
comment['comments'] += get_comments(kid)
end
end
comments
end
def get_comments_for_story(id, fresh = false)
cached_filename = "story-and-comments-for-#{id}.json"
if File.exist?(cached_filename) && ! fresh
return JSON.parse(File.read(cached_filename))
end
# TODO: Maybe fix me up. Shouldn't need to recurse twice? Or should I...
story = get_item(id)
comments = []
story['kids'].each do |kid|
comments += get_comments(kid)
end
story['comments'] = comments
puts "Generated data file for #{id}."
File.write(cached_filename, JSON.pretty_generate(story))
story
end
if $0 == __FILE__
unless ARGV.length == 1
puts "usage: #{$0} story_id"
puts ' where story_id is the ID of a "Who\'s Hiring?" post'
exit 1
end
get_comments_for_story(ARGV[0])
end