-
Notifications
You must be signed in to change notification settings - Fork 2
/
povcalnet.ado
779 lines (620 loc) · 22.7 KB
/
povcalnet.ado
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
/*=======================================================
Program Name: povcalnet.ado
Author:
R.Andres Castaneda Aguilar
Jorge Soler Lopez
Espen Beer Prydz
Christoph Lakner
Ruoxuan Wu
Qinghua Zhao
World Bank Group
project: Stata package to easily query the [PovcalNet API](http://iresearch.worldbank.org/PovcalNet/docs/PovcalNet%20API.pdf)
Dependencies: The World Bank - DEC
-----------------------------------------------------------------------
Creation Date: Sept 2018
References:
Output: dta file
=======================================================*/
program def povcalnet, rclass
set checksum off //temporarily bypasses control of files taken from internet
version 11.0
syntax [anything(name=subcommand)] ///
[, ///
COUNtry(string) ///
REGion(string) ///
YEAR(string) ///
POVline(numlist) ///
POPShare(numlist) ///
PPP(numlist) ///
AGGregate ///
CLEAR ///
INFOrmation ///
coverage(string) ///
ISO /// Standard ISO codes
SERVER(string) /// internal use
pause /// debugging
FILLgaps ///
N2disp(integer 15) ///
noDIPSQuery ///
querytimes(integer 5) ///
]
if ("`pause'" == "pause") pause on
else pause off
qui {
dis as text "{hline}"
noi disp in red "WARNING:"
noi dis as text `"{p 6 4 0 80}The povcalnet command and website have been replaced with the new pip command and the Poverty and Inequality Portal, {browse "https://pip.worldbank.org/"}. Povcalnet will remain functional until the end of 2022, but its data is now {bf:outdated}. Once retired, you will no longer be able to access these data. If you anticipate needing to reproduce existing analysis based on PovcalNet data, you will want to save a version of the data. Moving forward, we encourage you to use the new PIP site and its corresponding {browse "https://github.com/worldbank/pip": pip} Stata command, with the latest data and many new features.{p_end}"'
dis as text "{hline}" _n
//========================================================
// Conditions
//========================================================
if ("`aggregate'" != "" & "`fillgaps'" != "") {
noi disp in red "options {it:aggregate} and {it:fillgaps} are mutually exclusive." _n /*
*/ "Please select only one."
error
}
if ("`popshare'" != "" & (lower("`subcommand'") == "wb" | "`aggregate'" != "")) {
noi disp in red "option {it:popshare} can't be combined with option {it:aggregate}" _c /*
*/ " or with subcommand {it:wb}" _n
error
}
// ------------------------------------------------------------------------
// New session procedure
// ------------------------------------------------------------------------
if ("${pcn_cmds_ssc}" == "") {
// ---------------------------------------------------------------
// Update PovcalNet
// ---------------------------------------------------------------
* mata: povcalnet_source("povcalnet") // creates local src
_pcn_find_src povcalnet
local src = "`r(src)'"
* If povcalnet was installed from github
if (regexm("`src'", "github")) {
local git_cmds povcalnet
foreach cmd of local git_cmds {
* Check repository of files
* mata: povcalnet_source("`cmd'")
_pcn_find_src `cmd'
local src = "`r(src)'"
if regexm("`src'", "\.io/") { // if site
if regexm("`src'", "://([^ ]+)\.github") {
local repo = regexs(1) + "/`cmd'"
}
}
else { // if branch
if regexm("`src'", "\.com/([^ ]+)(/`cmd')") {
local repo = regexs(1) + regexs(2)
}
}
qui github query `repo'
local latestversion = "`r(latestversion)'"
if regexm("`r(latestversion)'", "v([0-9]+)\.([0-9]+)\.([0-9]+)"){
local lastMajor = regexs(1)
local lastMinor = regexs(2)
local lastPatch = regexs(3)
}
qui github version `cmd'
local crrtversion = "`r(version)'"
if regexm("`r(version)'", "v([0-9]+)\.([0-9]+)\.([0-9]+)"){
local crrMajor = regexs(1)
local crrMinor = regexs(2)
local crrPatch = regexs(3)
}
foreach x in repo cmd {
local `x' : subinstr local `x' "." "", all
local `x' : subinstr local `x' "-" ".", all
if regexm("``x''", "v([0-9]+)(\.)?([a-z]+)?([0-9]?)") {
disp regexs(1) regexs(2) regexs(4)
}
}
* force installation
if ("`crrtversion'" == "") {
github install `repo', replace
cap window stopbox note "github command has been reinstalled to " ///
"keep record of new updates. Please type discard and retry."
global pcn_cmds_ssc = ""
exit
}
if (`lastMajor' > `crrMajor' | `lastMinor' > `crrMinor' | `lastPatch' > `crrPatch') {
* if (`lastMajor'`lastMinor'`lastPatch' > `crrMajor'`crrMinor'`crrPatch') {
cap window stopbox rusure "There is a new version of `cmd' in Github (`latestversion')." ///
"Would you like to install it now?"
if (_rc == 0) {
cap github install `repo', replace
if (_rc == 0) {
cap window stopbox note "Installation complete. please type" ///
"discard in your command window to finish"
local bye "exit"
}
else {
noi disp as err "there was an error in the installation. " _n ///
"please run the following to retry, " _n(2) ///
"{stata github install `repo', replace}"
local bye "error"
}
}
else local bye ""
} // end of checking github update
else {
noi disp as result "Github version of {cmd:`cmd'} is up to date."
local bye ""
}
} // end of loop
} // end if installed from github
else if (regexm("`src'", "repec")) { // if povcalnet was installed from SSC
qui adoupdate povcalnet, ssconly
if ("`r(pkglist)'" == "povcalnet") {
cap window stopbox rusure "There is a new version of povcalnet in SSC." ///
"Would you like to install it now?"
if (_rc == 0) {
cap ado update povcalnet, ssconly update
if (_rc == 0) {
cap window stopbox note "Installation complete. please type" ///
"discard in your command window to finish"
local bye "exit"
}
else {
noi disp as err "there was an error in the installation. " _n ///
"please run the following to retry, " _n(2) ///
"{stata ado update povcalnet, ssconly update}"
local bye "error"
}
}
else local bye ""
} // end of checking SSC update
else {
noi disp as result "SSC version of {cmd:povcalnet} is up to date."
local bye ""
}
} // Finish checking povcalnet update
else {
noi disp as result "Source of {cmd:povcalnet} package not found." _n ///
"You won't be able to benefit from latest updates."
local bye ""
}
/*==================================================
Dependencies
==================================================*/
*---------- check SSC commands
local ssc_cmds missings
noi disp in y "Note: " in w "{cmd:povcalnet} requires the packages " ///
"below from SSC: " _n in g "`ssc_cmds'"
foreach cmd of local ssc_cmds {
capture which `cmd'
if (_rc != 0) {
ssc install `cmd'
noi disp in g "{cmd:`cmd'} " in w _col(15) "installed"
}
}
adoupdate `ssc_cmds', ssconly
if ("`r(pkglist)'" != "") adoupdate `r(pkglist)', update ssconly
global pcn_cmds_ssc = 1 // make sure it does not execute again per session
`bye'
}
/*==================================================
Defaults
==================================================*/
*---------- API defaults
if "`server'"!="" {
if !inlist(lower("`server'"), "int", "testing", "ar") {
noi disp in red "the server requested does not exist"
error
}
if (lower("`server'") == "int") {
local server "${pcn_svr_in}"
}
if (lower("`server'") == "testing") {
local server "${pcn_svr_ts}"
}
if (upper("`server'") == "AR") {
local server "${pcn_svr_ar}"
}
if ("`server'" == "") {
noi disp in red "You don't have access to internal servers" _n /*
*/ "You're being redirected to public server"
local server "http://iresearch.worldbank.org/PovcalNet"
}
}
else {
local server "http://iresearch.worldbank.org/PovcalNet"
}
local base = "`server'/PovcalNetAPI.ashx"
return local server = "`server'"
//------------ Check internet connection
scalar tpage = fileread(`"`server'/js/common_NET.js"')
if regexm(tpage, "error") {
noi disp in red "You may not have Internet connections. Please verify"
error
}
*---------- lower case subcommand
local subcommand = lower("`subcommand'")
*---------- Test
if ("`subcommand'" == "test") {
if ("${pcn_query}" == "") {
noi di as err "global pcn_query does not exist. You cannot test the query."
error
}
local fq = "`base'?${pcn_query}"
noi disp in y "querying" in w _n "`fq'"
noi view browse "`fq'"
exit
}
*---------- Modify country(all) with aggregate
if (lower("`country'") == "all" & "`aggregate'" != "") {
local country ""
local aggregate ""
local subcommand "wb"
local wb_change 1
noi disp as err "Warning: " as text " {cmd:povcalnet, country(all) aggregate} " /*
*/ "is equivalent to {cmd:povcalnet wb}. " _n /*
*/ " if you want to aggregate all countries by survey years, " /*
*/ "you need to parse the list of countries in {it:country()} option. See " /*
*/ "{help povcalnet##options:aggregate option description} for an example on how to do it"
}
else {
local wb_change 0
}
if ("`year'" == "") local year "all"
*
*---------- Coverage
if ("`coverage'" == "") local coverage = "all"
local coverage = lower("`coverage'")
foreach c of local coverage {
if !inlist(lower("`c'"), "national", "rural", "urban", "all") {
noi disp in red `"option {it:coverage()} must be "national", "rural", "urban" or "all" "'
error
}
}
*---------- Poverty line/population share
// Blank popshare and blank povline = default povline 1.9
if ("`popshare'" == "" & "`povline'" == "") {
local povline = 1.9
local pcall = "povline"
}
// defined popshare and defined povline = error
else if ("`popshare'" != "" & "`povline'" != "") {
noi disp as err "povline and popshare cannot be used at the same time"
error
}
// blank popshare and defined povline
else if ("`popshare'" == "" & "`povline'" != "") {
local pcall = "povline"
}
// defined popshare and blank povline
else {
local pcall = "popshare"
}
*---------- Info
if regexm("`subcommand'", "^info") {
local information = "information"
local subcommand = "information"
}
*---------- Subcommand consistency
if !inlist("`subcommand'", "wb", "information", "cl", "") {
noi disp as err "subcommand must be either {it:wb}, {it:cl}, or {it:info}"
error
}
*---------- One-on-one execution
if ("`subcommand'" == "cl" & lower("`country'") == "all") {
noi disp in red "you cannot use option {it:countr(all)} with subcommand {it:cl}"
error 197
}
*---------- PPP
if (lower("`country'") == "all" & "`ppp'" != "") {
noi disp as err "Option {it:ppp()} is not allowed with {it:country(all)}"
error
}
*---------- WB aggregate
if ("`subcommand'" == "wb") {
if ("`country'" != "") {
noi disp as err "option {it:country()} is not allowed with subcommand {it:wb}"
error
}
noi disp as res "Note: " as txt "subcommand {it:wb} only accepts options " _n /*
*/ "{it:region()} and {it:year()}"
}
*---------- Country
if ("`country'" == "" & "`region'" == "") local country "all"
if ("`country'" != "") {
if (lower("`country'") != "all") local country = upper("`country'")
else local country "all"
}
/*==================================================
Main conditions
==================================================*/
if ("`information'" == "") {
if (c(N) != 0 & "`clear'" == "" & /*
*/ "`information'" == "") {
noi di as err "You must start with an empty dataset; or enable the option {it:clear}."
error 4
}
drop _all
}
*---------- Country and region
if ("`country'" != "") & ("`region'" != "") {
noi disp in r "options {it:country()} and {it:region()} are mutually exclusive"
error
}
if ("`aggregate'" != "") {
if ("`ppp'" != ""){
noi di as err "Option PPP cannot be combined with aggregate."
error 198
}
noi disp as res "Note: " as text "Aggregation is only possible over reference years."
local agg_display = "Aggregation in base year(s) `year'"
}
if (wordcount("`country'")>2) {
if ("`ppp'" != ""){
noi di as err "Option PPP can only be used with one country."
error 198
}
}
/*==================================================
Execution
==================================================*/
pause povcalnet - before execution
*---------- Information
if ("`information'" != ""){
noi povcalnet_info, `clear' `pause' server(`server')
return add
exit
}
*---------- Country Level (one-on-one query)
if ("`subcommand'" == "cl") {
noi povcalnet_cl, country("`country'") ///
year("`year'") ///
povline("`povline'") ///
ppp("`ppp'") ///
server("`server'") ///
coverage(`coverage') ///
`clear' ///
`iso' ///
`pause'
return add
exit
}
*---------- Regular query and Aggregate Query
if ("`subcommand'" == "wb") {
local wb "wb"
}
else local wb ""
tempfile povcalf
save `povcalf', empty
local f = 0
if ("`pcall'" == "povline") loc i_call "i_povline"
else loc i_call "i_popshare"
foreach `i_call' of local `pcall' {
local ++f
/*==================================================
Create Query
==================================================*/
povcalnet_query, country("`country'") ///
region("`region'") ///
year("`year'") ///
povline("`i_povline'") ///
popshare("`i_popshare'") ///
ppp("`ppp'") ///
coverage(`coverage') ///
server(`server') ///
`clear' ///
`information' ///
`iso' ///
`fillgaps' ///
`aggregate' ///
`wb' ///
`pause' ///
`groupedby' //
local query_ys = "`r(query_ys)'"
local query_ct = "`r(query_ct)'"
local query_pl = "`r(query_pl)'"
local query_ds = "`r(query_ds)'"
local query_pp = "`r(query_pp)'"
local query_ps = "`r(query_ps)'"
return local query_ys_`f' = "`query_ys'"
return local query_ct_`f' = "`query_ct'"
return local query_pl_`f' = "`query_pl'"
return local query_ds_`f' = "`query_ds'"
return local query_pp_`f' = "`query_pp'"
return local query_ps_`f' = "`query_ps'"
return local base = "`base'"
*---------- Query
if ("`popshare'" == ""){
local query = "`query_ys'&`query_ct'&`query_pl'`query_pp'`query_ds'&format=csv"
}
else{
local query = "`query_ys'&`query_ct'&`query_ps'`query_pp'`query_ds'&format=csv"
}
return local query_`f' "`query'"
global pcn_query = "`query'"
*---------- Base + query
local queryfull "`base'?`query'"
return local queryfull_`f' = "`queryfull'"
/*==================================================
Download and clean data
==================================================*/
*---------- download data
local rc = 0
local qr = 1 // query round
while (`qr' <= `querytimes') {
tempfile clfile
cap copy "`queryfull'" `clfile'
if (_rc == 0) {
cap insheet using `clfile', clear name
if (_rc != 0) local rc "in"
continue, break
}
else {
local rc "copy"
local ++qr
}
}
* global qr = `qr'
if ("`aggregate'" == "" & "`wb'" == "") {
local rtype 1
}
else {
local rtype 2
}
pause after download
*---------- Clean data
povcalnet_clean `rtype', year("`year'") `iso' /*
*/ rc(`rc') region(`region') `pause' `wb'
pause after cleaning
/*==================================================
Display Query
==================================================*/
if ("`dipsquery'" == "" & "`rc'" == "0") {
noi di as res _n "{ul: Query at \$`i_povline' poverty line}"
noi di as res "{hline}"
if ("`query_ys'" != "") noi di as res "Year:" as txt "{p 4 6 2} `query_ys' {p_end}"
if ("`query_ct'" != "") noi di as res "Country:" as txt "{p 4 6 2} `query_ct' {p_end}"
if ("`query_pl'" != "") noi di as res "Poverty line:" as txt "{p 4 6 2} `query_pl' {p_end}"
if ("`query_ps'" != "") noi di as res "Population share:" as txt "{p 4 6 2} `query_ps' {p_end}"
if ("`query_ds'" != "") noi di as res "Aggregation:" as txt "{p 4 6 2} `query_ds' {p_end}"
if ("`query_pp'" != "") noi di as res "PPP:" as txt "{p 4 6 2} `query_pp' {p_end}"
noi di as res _dup(20) "-"
noi di as res "No. Obs:" as txt _col(20) c(N)
noi di as res "{hline}"
}
/*==================================================
Append data
==================================================*/
if (`wb_change' == 1) {
keep if regioncode == "WLD"
}
append using `povcalf'
save `povcalf', replace
} // end of povline loop
return local npl = `f'
// ------------------------------
// display results
// ------------------------------
local n2disp = min(`c(N)', `n2disp')
noi di as res _n "{ul: first `n2disp' observations}"
if ("`subcommand'" == "wb") {
sort year regioncode
noi list region year povertyline headcount mean in 1/`n2disp', /*
*/ abbreviate(12) sepby(year)
}
else {
if ("`aggregate'" == "") {
sort countrycode year regioncode
noi list countrycode year povertyline headcount mean median datatype /*
*/ in 1/`n2disp', abbreviate(12) sepby(countrycode)
}
else {
sort year
noi list year povertyline headcount mean , /*
*/ abbreviate(12) sepby(povertyline)
}
}
//========================================================
// Create notes
//========================================================
local pllabel ""
foreach p of local povline {
local pllabel "`pllabel' \$`p'"
}
local pllabel = trim("`pllabel'")
local pllabel: subinstr local pllabel " " ", ", all
if ("`wb'" == "") {
if ("`aggregate'" == "" & "`fillgaps'" == "") {
local lvlabel "country level"
}
else if ("`aggregate'" != "" & "`fillgaps'" == "") {
local lvlabel "aggregated level"
}
else if ("`aggregate'" == "" & "`fillgaps'" != "") {
local lvlabel "Country level (lined up)"
}
else {
local lvlabel ""
}
}
else {
local lvlabel "regional and global level"
}
local datalabel "WB poverty at `lvlabel' using `pllabel'"
local datalabel = substr("`datalabel'", 1, 80)
label data "`datalabel' (`c(current_date)')"
* citations
local cite `"Please cite as: Castaneda Aguilar, R. A., C. Lakner, E. B. Prydz, J. S. Lopez, R. Wu and Q. Zhao (2019) "povcalnet: Stata module to access World Bank’s Global Poverty and Inequality data," Statistical Software Components 2019, Boston College Department of Economics."'
notes: `cite'
dis as text "{hline}"
noi disp in red "WARNING:"
noi dis as text `"{p 6 4 0 80}The povcalnet command and website have been replaced with the new pip command and the Poverty and Inequality Portal, {browse "https://pip.worldbank.org/"}. Povcalnet will remain functional until the end of 2022, but its data is now {bf:outdated}. Once retired, you will no longer be able to access these data. If you anticipate needing to reproduce existing analysis based on PovcalNet data, you will want to save a version of the data. Moving forward, we encourage you to use the new PIP site and its corresponding {browse "https://github.com/worldbank/pip": pip} Stata command, with the latest data and many new features.{p_end}"'
dis as text "{hline}" _n
noi disp in y _n `"`cite'"'
return local cite `"`cite'"'
} // end of qui
end
//========================================================
// Aux programs
//========================================================
program define _pcn_find_src, rclass
syntax anything(name=cmd id="Package name")
qui {
preserve
drop _all
// find stata.trk file
findfile stata.trk
local fn = "`r(fn)'"
// create copy
tempfile statatrk
copy "`r(fn)'" "`statatrk'"
// import copy into stata
import delimited using `statatrk', bindquote(nobind)
gen n = _n // line number
// find line where the package is used
levelsof n if regexm(v1, "`cmd'.pkg"), sep(,) loca(pklines)
if (`"`pklines'"' == `""') local src = "NotInstalled"
else {
// the latest source and subtract which refers to the source
local sourceline = max(0, `pklines') - 1
// get the source without the initial S
if regexm(v1[`sourceline'], "S (.*)") local src = regexs(1)
}
// return info
return local src = "`src'"
} // end of qui
end
// ------------------------------------------------------------------------
// MATA functions
// ------------------------------------------------------------------------
* findfile stata.trk
* local fn = "`r(fn)'"
cap mata: mata drop povcalnet_*()
mata:
// function to look for source of code
void povcalnet_source(string scalar cmd) {
cmd = cmd :+ "\.pkg"
fh = _fopen("`fn'", "r")
pos_a = ftell(fh)
pos_b = 0
while ((line=strtrim(fget(fh)))!=J(0,0,"")) {
if (regexm(strtrim(line), cmd)) {
fseek(fh, pos_b, -1)
break
}
pos_b = pos_a
pos_a = ftell(fh)
}
src = strtrim(fget(fh))
if (rows(src) > 0) {
src = substr(src, 3)
st_local("src", src)
}
else {
st_local("src", "NotFound")
}
fclose(fh)
}
end
exit
/* End of do-file */
><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><
Notes:
Version Control:
*! version 1.2.0 <Apr2022>
*! version 1.1.8 <Oct2021>
*! version 1.0.0 <sept2018>