-
Notifications
You must be signed in to change notification settings - Fork 0
/
fetch_cases.js
81 lines (77 loc) · 2.35 KB
/
fetch_cases.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
const bluebird = require('bluebird');
const request = require('request');
const getHrefs = require('get-hrefs');
const fs = require('fs');
const base_url = 'http://www.paho.org';
const config = require('./config');
const XLSX = require('xlsx');
request(
'http://www.paho.org/hq/index.php?option=com_content&view=article&id=12390&Itemid=42090&lang=en',
(error, response, body) => {
let urls = getHrefs(body).filter(l => {
return l.match('doc_view')
})
fs.readdir(config.dir_raw, (err, files) => {
bluebird.each(urls, url => {
console.log(url)
return fetch_doc(url, files);
}, {concurrency: 1})
.then(() => {
console.log('Done!');
process.exit();
})
});
});
/**
* Verifies if user has required level of authorisation
* @param {string} url
* @param {array} files auth and security definations from swagger file
* @return {Promise}
*/
function fetch_doc(url, files) {
return new Promise((resolve, reject) => {
let options = {
url: base_url + url,
headers: {
'Content-Type': 'application/vnd.openxmlformats-officedocument.' +
'spreadsheetml.sheet'
},
encoding: null
};
request.get(options, (err, res, body) => {
let title = res.headers['content-disposition'].match(
/filename="(.+?)";/
)[1];
if (title.match(/xls$/)) {
let arraybuffer = body;
/* convert data to binary string */
let data = arraybuffer;
// var data = new Uint8Array(arraybuffer);
let arr = [];
for (let i = 0;
i != data.length;
++i) arr[i] = String.fromCharCode(data[i]);
let bstr = arr.join('');
let workbook = XLSX.read(bstr, {type: 'binary'});
let worksheet = workbook.Sheets['2015 ENG'];
let file = title.replace('xls', 'json');
if (files.indexOf(file) === -1) {
// This file has not been downloaded before
console.log('download new', file)
fs.writeFile(config.dir_raw + title.replace('xls', 'json'),
JSON.stringify(worksheet),
(error, response) => {
return resolve();
});
} else {
console.log('already have', file)
// Already have file, move along.
resolve();
}
} else {
// Not an xls file, likely pdf.
return resolve();
}
});
})
}