2016i update
* Zone Changes
** Split Cyprus into 2 zones. The existing Asia/Nicosia now ends at
the northern boundary of the United Nations Buffer Zone and the new
zone Asia/Famagusta contains everything north of the buffer zone.
** Add missing data definitions: Congo-Kinshasa and South Sudan
** Old Crimea boundary no longer exists in OSM, use combination of
Crimea + Sevastopol
** Typo of extra space in Harrison County fixed in OSM
** Taishan City now has invalid geometry in OSM, use Xinhui district
instead when making boundaries
** Update to latest OSM data
* Other changes
** Add download throttling of publicly available Overpass API
** Remove old dist files if they exist so ogr2ogr can work
** Update README to note change in Overpass API querying
* Issues
** #4
diff --git a/README.md b/README.md
index 54799a6..1b84b7a 100644
--- a/README.md
+++ b/README.md
@@ -8,7 +8,7 @@
### Special note regarding the Overpass API
-Although the code currently queries the publicly available overpass API, this is not recommended because you will get throttled really quickly! I was able to get away with this because I worked on extracting pieces of data over a few months. In the future, I hope to make a docker file that downloads a planet dump and starts a local version of the Overpass API to extract the data.
+The code does query the publicly available overpass API, but it self-throttles the making of requests to have a minimum of 4 seconds gap between requests. If the Overpass API throttles the download, then the gap will be increased exponentionally.
### Running the project
diff --git a/index.js b/index.js
index ada5c81..78b2f80 100644
--- a/index.js
+++ b/index.js
@@ -3,6 +3,7 @@
var asynclib = require('async')
var jsts = require('jsts')
+var rimraf = require('rimraf')
var multiPolygon = require('turf-multipolygon')
var overpass = require('query-overpass')
var polygon = require('turf-polygon')
@@ -12,6 +13,8 @@
var geoJsonReader = new jsts.io.GeoJSONReader()
var geoJsonWriter = new jsts.io.GeoJSONWriter()
var distZones = {}
+var minRequestGap = 4
+var curRequestGap = 4
var safeMkdir = function (dirname, callback) {
fs.mkdir(dirname, function (err) {
@@ -109,7 +112,25 @@
downloadFromOverpass: function (cb) {
console.log('downloading from overpass')
fetchIfNeeded(boundaryFilename, boundaryCallback, function () {
- overpass(query, cb, { flatProperties: true })
+ var overpassResponseHandler = function (err, data) {
+ if (err) {
+ console.log(err)
+ console.log('Increasing overpass request gap')
+ curRequestGap *= 2
+ makeQuery()
+ } else {
+ console.log('Success, decreasing overpass request gap')
+ curRequestGap = Math.max(minRequestGap, curRequestGap / 2)
+ cb(null, data)
+ }
+ }
+ var makeQuery = function () {
+ console.log('waiting ' + curRequestGap + ' seconds')
+ setTimeout(function () {
+ overpass(query, overpassResponseHandler, { flatProperties: true })
+ }, curRequestGap * 1000)
+ }
+ makeQuery()
})
},
validateOverpassResult: ['downloadFromOverpass', function (results, cb) {
@@ -301,6 +322,8 @@
}],
makeShapefile: ['mergeZones', function (results, cb) {
console.log('convert from geojson to shapefile')
+ rimraf.sync('dist/dist')
+ rimraf.sync('dist/combined_shapefile.*')
exec('ogr2ogr -nlt MULTIPOLYGON dist/combined_shapefile.shp dist/combined.json OGRGeoJSON', function (err, stdout, stderr) {
if (err) { return cb(err) }
exec('zip dist/timezones.shapefile.zip dist/combined_shapefile.*', cb)
diff --git a/osmBoundarySources.json b/osmBoundarySources.json
index ba4890d..b9cef8c 100644
--- a/osmBoundarySources.json
+++ b/osmBoundarySources.json
@@ -251,6 +251,9 @@
"Congo-Brazzaville": {
"ISO3166-1": "CG"
},
+ "Congo-Kinshasa": {
+ "ISO3166-1": "CD"
+ },
"Cook Islands": {
"ISO3166-1": "CK"
},
@@ -277,7 +280,7 @@
"timezone": "America/Creston"
},
"Crimea": {
- "name:en": "Crimean Federal District"
+ "name:en": "Republic of Crimea"
},
"Croatia": {
"ISO3166-1": "HR"
@@ -466,7 +469,7 @@
"nist:state_fips": "21"
},
"Harrison County, IN": {
- "name": "Harrison County",
+ "name": "Harrison County",
"nist:state_fips": "18"
},
"Honduras": {
@@ -1035,6 +1038,9 @@
"name": "Sequatchie County",
"nist:state_fips": "47"
},
+ "Sevastopol": {
+ "name:en": "Sevastopol"
+ },
"Seychelles": {
"ISO3166-1": "SC"
},
@@ -1081,6 +1087,9 @@
"South Korea": {
"ISO3166-1": "KR"
},
+ "South Sudan": {
+ "ISO3166-1": "SS"
+ },
"Spain": {
"ISO3166-1": "ES"
},
@@ -1125,9 +1134,6 @@
"Tajikistan": {
"ISO3166-1": "TJ"
},
- "Taishan City": {
- "name:en": "Taishan City"
- },
"Tanzania": {
"name:en": "Tanzania"
},
@@ -1217,6 +1223,9 @@
"United Kingdom": {
"ISO3166-1": "GB"
},
+ "United Nations Buffer Zone": {
+ "name": "United Nations Buffer Zone"
+ },
"United States of America": {
"ISO3166-1": "US"
},
@@ -1281,6 +1290,9 @@
"Winnipeg-tz": {
"timezone": "America/Winnipeg"
},
+ "Xinhui District": {
+ "name:en": "Xinhui District"
+ },
"Xinxing County": {
"name:en": "Xinxing County"
},
diff --git a/package.json b/package.json
index ea3aee7..afb7e9c 100644
--- a/package.json
+++ b/package.json
@@ -16,6 +16,7 @@
"async": "^2.1.2",
"jsts": "^1.2.1",
"query-overpass": "^1.1.0",
+ "rimraf": "^2.5.4",
"shpjs": "^3.3.2",
"turf-multipolygon": "^1.0.1",
"turf-polygon": "^1.0.3"
diff --git a/timezones.json b/timezones.json
index 7395847..adbd99f 100644
--- a/timezones.json
+++ b/timezones.json
@@ -2240,7 +2240,7 @@
}, {
"op": "intersect",
"source": "manual-multipolygon",
- "data": [[[[110,44],[106,18],[87,18],[94,45],[110,44]]],[[[113.03,22.12],[113.05,22.097],[113.22,20.9],[113,19],[111.5,21.35],[111.45,21.48],[111.455,21.53],[111.441,21.5457],[111.4455,21.554],[111.4398,21.5638],[111.4391,21.574],[111.4364,21.5797],[111.4305,21.5802],[111.4311892,21.5843946],[111.6,21.6],[111.3,22.1],[110.5,22.3],[110,23],[111.9,23.2],[112.4,22.5],[112.8,22.45],[112.8,22.2],[113.0154615,22.1175344],[113.03,22.12]]]]
+ "data": [[[[110,44],[106,18],[87,18],[94,45],[110,44]]],[[[113.03,22.12],[113.05,22.097],[113.22,20.9],[113,19],[111.5,21.35],[111.45,21.48],[111.455,21.53],[111.441,21.5457],[111.4455,21.554],[111.4398,21.5638],[111.4391,21.574],[111.4364,21.5797],[111.4305,21.5802],[111.4311892,21.5843946],[111.6,21.6],[111.3,22.1],[110.5,22.3],[110,23],[111.9,23.2],[112.4,22.5],[112.8,22.45],[113,22.4],[113.0154615,22.1175344],[113.03,22.12]]]]
}, {
"op": "union",
"source": "overpass",
@@ -2278,9 +2278,9 @@
"source": "overpass",
"id": "Kaiping City"
}, {
- "op": "union",
+ "op": "difference",
"source": "overpass",
- "id": "Taishan City"
+ "id": "Xinhui District"
}, {
"op": "union",
"source": "overpass",
@@ -2337,6 +2337,25 @@
"id": "Uzbekistan"
}
],
+ "Asia/Famagusta": [
+ {
+ "op": "init",
+ "source": "overpass",
+ "id": "Cyprus"
+ }, {
+ "op": "intersect",
+ "source": "manual-polygon",
+ "data": [[[32.609482,35.166161],[32.621326,35.176544],[32.6283528,35.1893165],[32.640381,35.229355],[32.659264,35.244498],[32.686043,35.244217],[32.701836,35.227672],[32.6968982,35.1834922],[32.703896,35.179982],[32.703037,35.140125],[32.764664,35.122576],[32.792988,35.118504],[32.811527,35.095614],[32.839851,35.080163],[32.866888,35.081287],[32.86397,35.101232],[32.886114,35.104181],[32.897272,35.105445],[32.906799,35.101162],[32.920532,35.090277],[32.959242,35.1006],[33.012371,35.147144],[33.122749,35.156407],[33.164978,35.189803],[33.211327,35.183349],[33.267288,35.160056],[33.307028,35.166231],[33.315933,35.172844],[33.31869,35.172116],[33.320729,35.172247],[33.321351,35.176167],[33.319076,35.176957],[33.319774,35.180561],[33.323979,35.181227],[33.325567,35.182016],[33.334258,35.181946],[33.335073,35.182508],[33.338056,35.181824],[33.348613,35.183121],[33.348312,35.179456],[33.349857,35.177421],[33.352985,35.178452],[33.353135,35.178618],[33.353725,35.178842],[33.355844,35.178176],[33.355855,35.175878],[33.355002,35.174878],[33.355168,35.17462],[33.356161,35.173813],[33.35689,35.174049],[33.358929,35.174036],[33.360474,35.174891],[33.362303,35.174808],[33.363773,35.174611],[33.365752,35.175264],[33.367361,35.175953],[33.367522,35.17601],[33.367442,35.176382],[33.367769,35.176584],[33.368048,35.177009],[33.368397,35.177119],[33.368493,35.177597],[33.368826,35.177763],[33.36933,35.178114],[33.369963,35.17782],[33.372281,35.179158],[33.372624,35.185506],[33.37198,35.186497],[33.374727,35.187734],[33.374566,35.188567],[33.373922,35.189452],[33.374491,35.190689],[33.388739,35.197799],[33.403759,35.186261],[33.412471,35.174054],[33.408952,35.15967],[33.424401,35.156267],[33.424273,35.139143],[33.459463,35.109026],[33.469849,35.048268],[33.447189,35.022968],[33.444614,35.004128],[33.458347,35.001914],[33.464913,35.004796],[33.485126,35.005886],[33.488731,35.060493],[33.576279,35.042927],[33.593616,35.032808],[33.705025,35.028591],[33.794975,35.050235],[33.821239,35.072577],[33.847847,35.063304],[33.888702,35.088591],[33.907499,35.067448],[33.931961,35.067167],[33.94136,35.070048],[33.960886,35.073525],[33.966765,35.072436],[33.977666,35.064919],[33.990326,35.064744],[33.9988932,35.069797],[33.999531,35.069626],[34.246445,35.178298],[35.134277,35.554574],[34.799194,36.160053],[32.382202,35.572449],[32.542877,35.396887],[32.5987035,35.1791626],[32.597401,35.175001],[32.609482,35.166161]]]
+ }, {
+ "op": "difference",
+ "source": "overpass",
+ "id": "British Sovereign Base Areas"
+ }, {
+ "op": "difference",
+ "source": "overpass",
+ "id": "United Nations Buffer Zone"
+ }
+ ],
"Asia/Gaza": [
{
"op": "init",
@@ -2633,9 +2652,17 @@
"source": "overpass",
"id": "Cyprus"
}, {
+ "op": "difference",
+ "source": "manual-polygon",
+ "data": [[[32.609482,35.166161],[32.621326,35.176544],[32.6283528,35.1893165],[32.640381,35.229355],[32.659264,35.244498],[32.686043,35.244217],[32.701836,35.227672],[32.6968982,35.1834922],[32.703896,35.179982],[32.703037,35.140125],[32.764664,35.122576],[32.792988,35.118504],[32.811527,35.095614],[32.839851,35.080163],[32.866888,35.081287],[32.86397,35.101232],[32.886114,35.104181],[32.897272,35.105445],[32.906799,35.101162],[32.920532,35.090277],[32.959242,35.1006],[33.012371,35.147144],[33.122749,35.156407],[33.164978,35.189803],[33.211327,35.183349],[33.267288,35.160056],[33.307028,35.166231],[33.315933,35.172844],[33.31869,35.172116],[33.320729,35.172247],[33.321351,35.176167],[33.319076,35.176957],[33.319774,35.180561],[33.323979,35.181227],[33.325567,35.182016],[33.334258,35.181946],[33.335073,35.182508],[33.338056,35.181824],[33.348613,35.183121],[33.348312,35.179456],[33.349857,35.177421],[33.352985,35.178452],[33.353135,35.178618],[33.353725,35.178842],[33.355844,35.178176],[33.355855,35.175878],[33.355002,35.174878],[33.355168,35.17462],[33.356161,35.173813],[33.35689,35.174049],[33.358929,35.174036],[33.360474,35.174891],[33.362303,35.174808],[33.363773,35.174611],[33.365752,35.175264],[33.367361,35.175953],[33.367522,35.17601],[33.367442,35.176382],[33.367769,35.176584],[33.368048,35.177009],[33.368397,35.177119],[33.368493,35.177597],[33.368826,35.177763],[33.36933,35.178114],[33.369963,35.17782],[33.372281,35.179158],[33.372624,35.185506],[33.37198,35.186497],[33.374727,35.187734],[33.374566,35.188567],[33.373922,35.189452],[33.374491,35.190689],[33.388739,35.197799],[33.403759,35.186261],[33.412471,35.174054],[33.408952,35.15967],[33.424401,35.156267],[33.424273,35.139143],[33.459463,35.109026],[33.469849,35.048268],[33.447189,35.022968],[33.444614,35.004128],[33.458347,35.001914],[33.464913,35.004796],[33.485126,35.005886],[33.488731,35.060493],[33.576279,35.042927],[33.593616,35.032808],[33.705025,35.028591],[33.794975,35.050235],[33.821239,35.072577],[33.847847,35.063304],[33.888702,35.088591],[33.907499,35.067448],[33.931961,35.067167],[33.94136,35.070048],[33.960886,35.073525],[33.966765,35.072436],[33.977666,35.064919],[33.990326,35.064744],[33.9988932,35.069797],[33.999531,35.069626],[34.246445,35.178298],[35.134277,35.554574],[34.799194,36.160053],[32.382202,35.572449],[32.542877,35.396887],[32.5987035,35.1791626],[32.597401,35.175001],[32.609482,35.166161]]]
+ }, {
"op": "union",
"source": "overpass",
"id": "British Sovereign Base Areas"
+ }, {
+ "op": "union",
+ "source": "overpass",
+ "id": "United Nations Buffer Zone"
}
],
"Asia/Novokuznetsk": [
@@ -2768,7 +2795,7 @@
}, {
"op": "difference",
"source": "manual-polygon",
- "data": [[[113.03,22.12],[113.05,22.097],[113.22,20.9],[113,19],[107,19],[110,23],[111.9,23.2],[112.4,22.5],[112.8,22.45],[112.8,22.2],[113.0154615,22.1175344],[113.03,22.12]]]
+ "data": [[[113.03,22.12],[113.05,22.097],[113.22,20.9],[113,19],[107,19],[110,23],[111.9,23.2],[112.4,22.5],[112.8,22.45],[113,22.4],[113.0154615,22.1175344],[113.03,22.12]]]
}, {
"op": "difference",
"source": "overpass",
@@ -2786,9 +2813,9 @@
"source": "overpass",
"id": "Kaiping City"
}, {
- "op": "difference",
+ "op": "union",
"source": "overpass",
- "id": "Taishan City"
+ "id": "Xinhui District"
}, {
"op": "difference",
"source": "overpass",
@@ -3455,6 +3482,10 @@
}, {
"op": "difference",
"source": "overpass",
+ "id": "Sevastopol"
+ }, {
+ "op": "difference",
+ "source": "overpass",
"id": "Zakarpattia"
}
],
@@ -3568,6 +3599,10 @@
}, {
"op": "difference",
"source": "overpass",
+ "id": "Sevastopol"
+ }, {
+ "op": "difference",
+ "source": "overpass",
"id": "Yekaterinburg-tz"
}
],
@@ -3655,6 +3690,10 @@
"op": "init",
"source": "overpass",
"id": "Crimea"
+ }, {
+ "op": "union",
+ "source": "overpass",
+ "id": "Sevastopol"
}
],
"Europe/Skopje": [