blob: e6f50f11a629a82396229351700988df0b8862ea [file] [log] [blame]
Evan Siroky7891a6e2016-11-05 11:50:50 -07001var exec = require('child_process').exec
2var fs = require('fs')
evansirokyd401c892016-06-16 00:05:14 -07003
Evan Siroky477ece62017-08-01 07:08:51 -07004var area = require('@mapbox/geojson-area')
evansiroky70b35fe2018-04-01 21:06:36 -07005var geojsonhint = require('@mapbox/geojsonhint')
evansiroky0ea1d1e2018-10-30 22:30:51 -07006var bbox = require('@turf/bbox').default
Evan Siroky8326cf02017-03-02 08:27:55 -08007var helpers = require('@turf/helpers')
Evan Siroky070bbb92017-03-07 23:48:29 -08008var multiPolygon = helpers.multiPolygon
Evan Siroky8326cf02017-03-02 08:27:55 -08009var polygon = helpers.polygon
Evan Siroky7891a6e2016-11-05 11:50:50 -070010var asynclib = require('async')
11var jsts = require('jsts')
Evan Sirokyb57a5b92016-11-07 10:22:34 -080012var rimraf = require('rimraf')
Evan Siroky7891a6e2016-11-05 11:50:50 -070013var overpass = require('query-overpass')
Neil Fullerc4ae49b2020-04-30 18:08:43 +010014var yargs = require('yargs')
evansirokyd401c892016-06-16 00:05:14 -070015
evansiroky5348a6f2019-01-05 15:39:28 -080016const ProgressStats = require('./progressStats')
17
Evan Siroky7891a6e2016-11-05 11:50:50 -070018var osmBoundarySources = require('./osmBoundarySources.json')
19var zoneCfg = require('./timezones.json')
evansiroky0ea1d1e2018-10-30 22:30:51 -070020var expectedZoneOverlaps = require('./expectedZoneOverlaps.json')
Evan Siroky081648a2017-07-04 09:53:36 -070021
Neil Fullerc4ae49b2020-04-30 18:08:43 +010022const argv = yargs
23 .option('included_zones', {
24 description: 'Include specified zones',
25 type: 'array'
26 })
27 .option('no_validation', {
28 description: 'Skip validation',
29 type: 'boolean'
30 })
31 .help()
32 .strict()
33 .alias('help', 'h')
34 .argv
35
Evan Siroky081648a2017-07-04 09:53:36 -070036// allow building of only a specified zones
Neil Fullerc4ae49b2020-04-30 18:08:43 +010037let includedZones = []
38if (argv.included_zones) {
Evan Siroky081648a2017-07-04 09:53:36 -070039 var newZoneCfg = {}
Neil Fullerc4ae49b2020-04-30 18:08:43 +010040 includedZones = argv.included_zones
41 includedZones.forEach((zoneName) => {
Evan Siroky081648a2017-07-04 09:53:36 -070042 newZoneCfg[zoneName] = zoneCfg[zoneName]
43 })
44 zoneCfg = newZoneCfg
45
46 // filter out unneccessary downloads
47 var newOsmBoundarySources = {}
48 Object.keys(zoneCfg).forEach((zoneName) => {
49 zoneCfg[zoneName].forEach((op) => {
50 if (op.source === 'overpass') {
51 newOsmBoundarySources[op.id] = osmBoundarySources[op.id]
52 }
53 })
54 })
55
56 osmBoundarySources = newOsmBoundarySources
57}
58
Evan Siroky7891a6e2016-11-05 11:50:50 -070059var geoJsonReader = new jsts.io.GeoJSONReader()
60var geoJsonWriter = new jsts.io.GeoJSONWriter()
Evan Siroky477ece62017-08-01 07:08:51 -070061var precisionModel = new jsts.geom.PrecisionModel(1000000)
62var precisionReducer = new jsts.precision.GeometryPrecisionReducer(precisionModel)
Evan Siroky7891a6e2016-11-05 11:50:50 -070063var distZones = {}
Evan Sirokyb57a5b92016-11-07 10:22:34 -080064var minRequestGap = 4
65var curRequestGap = 4
evansirokyd401c892016-06-16 00:05:14 -070066
Evan Siroky7891a6e2016-11-05 11:50:50 -070067var safeMkdir = function (dirname, callback) {
68 fs.mkdir(dirname, function (err) {
69 if (err && err.code === 'EEXIST') {
evansiroky4be1c7a2016-06-16 18:23:34 -070070 callback()
71 } else {
72 callback(err)
73 }
74 })
75}
76
Evan Sirokyb173fd42017-03-08 15:16:27 -080077var debugGeo = function (op, a, b, reducePrecision) {
evansirokybecb56e2016-07-06 12:42:35 -070078 var result
79
Evan Sirokyb173fd42017-03-08 15:16:27 -080080 if (reducePrecision) {
Evan Sirokyb173fd42017-03-08 15:16:27 -080081 a = precisionReducer.reduce(a)
82 b = precisionReducer.reduce(b)
83 }
84
evansiroky6f9d8f72016-06-21 16:27:54 -070085 try {
Evan Siroky7891a6e2016-11-05 11:50:50 -070086 switch (op) {
evansiroky6f9d8f72016-06-21 16:27:54 -070087 case 'union':
evansirokybecb56e2016-07-06 12:42:35 -070088 result = a.union(b)
evansiroky6f9d8f72016-06-21 16:27:54 -070089 break
90 case 'intersection':
evansirokybecb56e2016-07-06 12:42:35 -070091 result = a.intersection(b)
evansiroky6f9d8f72016-06-21 16:27:54 -070092 break
Evan Siroky070bbb92017-03-07 23:48:29 -080093 case 'intersects':
94 result = a.intersects(b)
95 break
evansiroky6f9d8f72016-06-21 16:27:54 -070096 case 'diff':
Evan Sirokyb173fd42017-03-08 15:16:27 -080097 result = a.difference(b)
evansiroky6f9d8f72016-06-21 16:27:54 -070098 break
99 default:
100 var err = new Error('invalid op: ' + op)
101 throw err
102 }
Evan Siroky7891a6e2016-11-05 11:50:50 -0700103 } catch (e) {
Evan Sirokyb173fd42017-03-08 15:16:27 -0800104 if (e.name === 'TopologyException') {
105 console.log('Encountered TopologyException, retry with GeometryPrecisionReducer')
106 return debugGeo(op, a, b, true)
107 }
evansiroky6f9d8f72016-06-21 16:27:54 -0700108 console.log('op err')
evansirokybecb56e2016-07-06 12:42:35 -0700109 console.log(e)
110 console.log(e.stack)
111 fs.writeFileSync('debug_' + op + '_a.json', JSON.stringify(geoJsonWriter.write(a)))
112 fs.writeFileSync('debug_' + op + '_b.json', JSON.stringify(geoJsonWriter.write(b)))
evansiroky6f9d8f72016-06-21 16:27:54 -0700113 throw e
114 }
evansiroky6f9d8f72016-06-21 16:27:54 -0700115
evansirokybecb56e2016-07-06 12:42:35 -0700116 return result
evansiroky4be1c7a2016-06-16 18:23:34 -0700117}
118
Evan Siroky070bbb92017-03-07 23:48:29 -0800119var fetchIfNeeded = function (file, superCallback, downloadCallback, fetchFn) {
120 // check for file that got downloaded
Evan Siroky7891a6e2016-11-05 11:50:50 -0700121 fs.stat(file, function (err) {
Evan Siroky070bbb92017-03-07 23:48:29 -0800122 if (!err) {
123 // file found, skip download steps
124 return superCallback()
125 }
126 // check for manual file that got fixed and needs validation
127 var fixedFile = file.replace('.json', '_fixed.json')
128 fs.stat(fixedFile, function (err) {
129 if (!err) {
130 // file found, return fixed file
131 return downloadCallback(null, require(fixedFile))
132 }
133 // no manual fixed file found, download from overpass
134 fetchFn()
135 })
evansiroky50216c62016-06-16 17:41:47 -0700136 })
137}
138
Evan Siroky7891a6e2016-11-05 11:50:50 -0700139var geoJsonToGeom = function (geoJson) {
Evan Siroky8326cf02017-03-02 08:27:55 -0800140 try {
141 return geoJsonReader.read(JSON.stringify(geoJson))
142 } catch (e) {
143 console.error('error converting geojson to geometry')
144 fs.writeFileSync('debug_geojson_read_error.json', JSON.stringify(geoJson))
145 throw e
146 }
Evan Siroky5669adc2016-07-07 17:25:31 -0700147}
148
Evan Siroky8b47abe2016-10-02 12:28:52 -0700149var geomToGeoJson = function (geom) {
150 return geoJsonWriter.write(geom)
151}
152
Evan Siroky7891a6e2016-11-05 11:50:50 -0700153var geomToGeoJsonString = function (geom) {
Evan Siroky5669adc2016-07-07 17:25:31 -0700154 return JSON.stringify(geoJsonWriter.write(geom))
155}
156
evansiroky5348a6f2019-01-05 15:39:28 -0800157const downloadProgress = new ProgressStats(
158 'Downloading',
159 Object.keys(osmBoundarySources).length
160)
161
Evan Siroky7891a6e2016-11-05 11:50:50 -0700162var downloadOsmBoundary = function (boundaryId, boundaryCallback) {
163 var cfg = osmBoundarySources[boundaryId]
Evan Siroky1bcd4772017-10-14 23:47:21 -0700164 var query = '[out:json][timeout:60];('
165 if (cfg.way) {
166 query += 'way'
167 } else {
168 query += 'relation'
169 }
Evan Siroky7891a6e2016-11-05 11:50:50 -0700170 var boundaryFilename = './downloads/' + boundaryId + '.json'
171 var debug = 'getting data for ' + boundaryId
172 var queryKeys = Object.keys(cfg)
evansiroky63d35e12016-06-16 10:08:15 -0700173
Evan Siroky5669adc2016-07-07 17:25:31 -0700174 for (var i = queryKeys.length - 1; i >= 0; i--) {
Evan Siroky7891a6e2016-11-05 11:50:50 -0700175 var k = queryKeys[i]
Evan Siroky1bcd4772017-10-14 23:47:21 -0700176 if (k === 'way') continue
Evan Siroky7891a6e2016-11-05 11:50:50 -0700177 var v = cfg[k]
Evan Siroky5669adc2016-07-07 17:25:31 -0700178
179 query += '["' + k + '"="' + v + '"]'
evansiroky63d35e12016-06-16 10:08:15 -0700180 }
181
evansiroky283ebbc2018-07-16 15:13:07 -0700182 query += ';);out body;>;out meta qt;'
evansiroky4be1c7a2016-06-16 18:23:34 -0700183
evansiroky5348a6f2019-01-05 15:39:28 -0800184 downloadProgress.beginTask(debug, true)
evansiroky63d35e12016-06-16 10:08:15 -0700185
Evan Siroky7891a6e2016-11-05 11:50:50 -0700186 asynclib.auto({
187 downloadFromOverpass: function (cb) {
evansiroky5348a6f2019-01-05 15:39:28 -0800188 console.log('downloading from overpass')
Evan Siroky070bbb92017-03-07 23:48:29 -0800189 fetchIfNeeded(boundaryFilename, boundaryCallback, cb, function () {
Evan Sirokyb57a5b92016-11-07 10:22:34 -0800190 var overpassResponseHandler = function (err, data) {
191 if (err) {
192 console.log(err)
193 console.log('Increasing overpass request gap')
194 curRequestGap *= 2
195 makeQuery()
196 } else {
197 console.log('Success, decreasing overpass request gap')
198 curRequestGap = Math.max(minRequestGap, curRequestGap / 2)
199 cb(null, data)
200 }
201 }
202 var makeQuery = function () {
203 console.log('waiting ' + curRequestGap + ' seconds')
204 setTimeout(function () {
205 overpass(query, overpassResponseHandler, { flatProperties: true })
206 }, curRequestGap * 1000)
207 }
208 makeQuery()
evansiroky63d35e12016-06-16 10:08:15 -0700209 })
210 },
Evan Siroky7891a6e2016-11-05 11:50:50 -0700211 validateOverpassResult: ['downloadFromOverpass', function (results, cb) {
evansiroky63d35e12016-06-16 10:08:15 -0700212 var data = results.downloadFromOverpass
evansiroky70b35fe2018-04-01 21:06:36 -0700213 if (!data.features) {
Evan Siroky7891a6e2016-11-05 11:50:50 -0700214 var err = new Error('Invalid geojson for boundary: ' + boundaryId)
evansiroky63d35e12016-06-16 10:08:15 -0700215 return cb(err)
216 }
evansiroky70b35fe2018-04-01 21:06:36 -0700217 if (data.features.length === 0) {
218 console.error('No data for the following query:')
219 console.error(query)
220 console.error('To read more about this error, please visit https://git.io/vxKQL')
evansiroky0ea1d1e2018-10-30 22:30:51 -0700221 return cb(new Error('No data found for from overpass query'))
evansiroky70b35fe2018-04-01 21:06:36 -0700222 }
evansiroky63d35e12016-06-16 10:08:15 -0700223 cb()
224 }],
Evan Siroky7891a6e2016-11-05 11:50:50 -0700225 saveSingleMultiPolygon: ['validateOverpassResult', function (results, cb) {
226 var data = results.downloadFromOverpass
227 var combined
evansiroky63d35e12016-06-16 10:08:15 -0700228
229 // union all multi-polygons / polygons into one
230 for (var i = data.features.length - 1; i >= 0; i--) {
Evan Siroky5669adc2016-07-07 17:25:31 -0700231 var curOsmGeom = data.features[i].geometry
evansiroky92c15c42018-11-15 20:58:18 -0800232 const curOsmProps = data.features[i].properties
233 if (
234 (curOsmGeom.type === 'Polygon' || curOsmGeom.type === 'MultiPolygon') &&
235 curOsmProps.type === 'boundary' // need to make sure enclaves aren't unioned
236 ) {
evansiroky63d35e12016-06-16 10:08:15 -0700237 console.log('combining border')
evansiroky70b35fe2018-04-01 21:06:36 -0700238 let errors = geojsonhint.hint(curOsmGeom)
239 if (errors && errors.length > 0) {
240 const stringifiedGeojson = JSON.stringify(curOsmGeom, null, 2)
241 errors = geojsonhint.hint(stringifiedGeojson)
242 console.error('Invalid geojson received in Overpass Result')
243 console.error('Overpass query: ' + query)
244 const problemFilename = boundaryId + '_convert_to_geom_error.json'
245 fs.writeFileSync(problemFilename, stringifiedGeojson)
246 console.error('saved problem file to ' + problemFilename)
247 console.error('To read more about this error, please visit https://git.io/vxKQq')
248 return cb(errors)
249 }
Evan Siroky070bbb92017-03-07 23:48:29 -0800250 try {
251 var curGeom = geoJsonToGeom(curOsmGeom)
252 } catch (e) {
253 console.error('error converting overpass result to geojson')
Evan Siroky477ece62017-08-01 07:08:51 -0700254 console.error(e)
evansiroky70b35fe2018-04-01 21:06:36 -0700255
Evan Siroky1bcd4772017-10-14 23:47:21 -0700256 fs.writeFileSync(boundaryId + '_convert_to_geom_error-all-features.json', JSON.stringify(data))
evansiroky70b35fe2018-04-01 21:06:36 -0700257 return cb(e)
Evan Siroky070bbb92017-03-07 23:48:29 -0800258 }
Evan Siroky7891a6e2016-11-05 11:50:50 -0700259 if (!combined) {
evansiroky63d35e12016-06-16 10:08:15 -0700260 combined = curGeom
261 } else {
Evan Siroky5669adc2016-07-07 17:25:31 -0700262 combined = debugGeo('union', curGeom, combined)
evansiroky63d35e12016-06-16 10:08:15 -0700263 }
264 }
265 }
Evan Siroky081c8e42017-05-29 14:53:52 -0700266 try {
267 fs.writeFile(boundaryFilename, geomToGeoJsonString(combined), cb)
268 } catch (e) {
269 console.error('error writing combined border to geojson')
270 fs.writeFileSync(boundaryId + '_combined_border_convert_to_geom_error.json', JSON.stringify(data))
evansiroky70b35fe2018-04-01 21:06:36 -0700271 return cb(e)
Evan Siroky081c8e42017-05-29 14:53:52 -0700272 }
evansiroky63d35e12016-06-16 10:08:15 -0700273 }]
274 }, boundaryCallback)
275}
evansirokyd401c892016-06-16 00:05:14 -0700276
Evan Siroky4fc596c2016-09-25 19:52:30 -0700277var getTzDistFilename = function (tzid) {
278 return './dist/' + tzid.replace(/\//g, '__') + '.json'
279}
280
281/**
282 * Get the geometry of the requested source data
283 *
284 * @return {Object} geom The geometry of the source
285 * @param {Object} source An object representing the data source
286 * must have `source` key and then either:
287 * - `id` if from a file
288 * - `id` if from a file
289 */
Evan Siroky7891a6e2016-11-05 11:50:50 -0700290var getDataSource = function (source) {
evansirokybecb56e2016-07-06 12:42:35 -0700291 var geoJson
Evan Siroky7891a6e2016-11-05 11:50:50 -0700292 if (source.source === 'overpass') {
evansirokybecb56e2016-07-06 12:42:35 -0700293 geoJson = require('./downloads/' + source.id + '.json')
Evan Siroky7891a6e2016-11-05 11:50:50 -0700294 } else if (source.source === 'manual-polygon') {
evansirokybecb56e2016-07-06 12:42:35 -0700295 geoJson = polygon(source.data).geometry
Evan Siroky7891a6e2016-11-05 11:50:50 -0700296 } else if (source.source === 'manual-multipolygon') {
Evan Siroky8e30a2e2016-08-06 19:55:35 -0700297 geoJson = multiPolygon(source.data).geometry
Evan Siroky7891a6e2016-11-05 11:50:50 -0700298 } else if (source.source === 'dist') {
Evan Siroky4fc596c2016-09-25 19:52:30 -0700299 geoJson = require(getTzDistFilename(source.id))
evansiroky4be1c7a2016-06-16 18:23:34 -0700300 } else {
301 var err = new Error('unknown source: ' + source.source)
302 throw err
303 }
Evan Siroky5669adc2016-07-07 17:25:31 -0700304 return geoJsonToGeom(geoJson)
evansiroky4be1c7a2016-06-16 18:23:34 -0700305}
306
Evan Siroky477ece62017-08-01 07:08:51 -0700307/**
308 * Post process created timezone boundary.
309 * - remove small holes and exclaves
310 * - reduce geometry precision
311 *
312 * @param {Geometry} geom The jsts geometry of the timezone
evansiroky26325842018-04-03 14:10:42 -0700313 * @param {boolean} returnAsObject if true, return as object, otherwise return stringified
314 * @return {Object|String} geojson as object or stringified
Evan Siroky477ece62017-08-01 07:08:51 -0700315 */
evansiroky26325842018-04-03 14:10:42 -0700316var postProcessZone = function (geom, returnAsObject) {
Evan Siroky477ece62017-08-01 07:08:51 -0700317 // reduce precision of geometry
318 const geojson = geomToGeoJson(precisionReducer.reduce(geom))
319
320 // iterate through all polygons
321 const filteredPolygons = []
322 let allPolygons = geojson.coordinates
323 if (geojson.type === 'Polygon') {
324 allPolygons = [geojson.coordinates]
325 }
326
327 allPolygons.forEach((curPolygon, idx) => {
328 // remove any polygon with very small area
329 const polygonFeature = polygon(curPolygon)
330 const polygonArea = area.geometry(polygonFeature.geometry)
331
332 if (polygonArea < 1) return
333
334 // find all holes
335 const filteredLinearRings = []
336
337 curPolygon.forEach((curLinearRing, lrIdx) => {
338 if (lrIdx === 0) {
339 // always keep first linearRing
340 filteredLinearRings.push(curLinearRing)
341 } else {
342 const polygonFromLinearRing = polygon([curLinearRing])
343 const linearRingArea = area.geometry(polygonFromLinearRing.geometry)
344
345 // only include holes with relevant area
346 if (linearRingArea > 1) {
347 filteredLinearRings.push(curLinearRing)
348 }
349 }
350 })
351
352 filteredPolygons.push(filteredLinearRings)
353 })
354
355 // recompile to geojson string
356 const newGeojson = {
357 type: geojson.type
358 }
359
360 if (geojson.type === 'Polygon') {
361 newGeojson.coordinates = filteredPolygons[0]
362 } else {
363 newGeojson.coordinates = filteredPolygons
364 }
365
evansiroky26325842018-04-03 14:10:42 -0700366 return returnAsObject ? newGeojson : JSON.stringify(newGeojson)
Evan Siroky477ece62017-08-01 07:08:51 -0700367}
368
evansiroky5348a6f2019-01-05 15:39:28 -0800369const buildingProgress = new ProgressStats(
370 'Building',
371 Object.keys(zoneCfg).length
372)
373
Evan Siroky7891a6e2016-11-05 11:50:50 -0700374var makeTimezoneBoundary = function (tzid, callback) {
evansiroky3046a3d2019-01-05 21:19:14 -0800375 buildingProgress.beginTask(`makeTimezoneBoundary for ${tzid}`, true)
evansiroky35f64342016-06-16 22:17:04 -0700376
Evan Siroky7891a6e2016-11-05 11:50:50 -0700377 var ops = zoneCfg[tzid]
378 var geom
evansiroky4be1c7a2016-06-16 18:23:34 -0700379
Evan Siroky7891a6e2016-11-05 11:50:50 -0700380 asynclib.eachSeries(ops, function (task, cb) {
evansiroky4be1c7a2016-06-16 18:23:34 -0700381 var taskData = getDataSource(task)
evansiroky6f9d8f72016-06-21 16:27:54 -0700382 console.log('-', task.op, task.id)
Evan Siroky7891a6e2016-11-05 11:50:50 -0700383 if (task.op === 'init') {
evansiroky4be1c7a2016-06-16 18:23:34 -0700384 geom = taskData
Evan Siroky7891a6e2016-11-05 11:50:50 -0700385 } else if (task.op === 'intersect') {
evansiroky6f9d8f72016-06-21 16:27:54 -0700386 geom = debugGeo('intersection', geom, taskData)
Evan Siroky7891a6e2016-11-05 11:50:50 -0700387 } else if (task.op === 'difference') {
evansiroky6f9d8f72016-06-21 16:27:54 -0700388 geom = debugGeo('diff', geom, taskData)
Evan Siroky7891a6e2016-11-05 11:50:50 -0700389 } else if (task.op === 'difference-reverse-order') {
Evan Siroky8ccaf0b2016-09-03 11:36:13 -0700390 geom = debugGeo('diff', taskData, geom)
Evan Siroky7891a6e2016-11-05 11:50:50 -0700391 } else if (task.op === 'union') {
evansiroky6f9d8f72016-06-21 16:27:54 -0700392 geom = debugGeo('union', geom, taskData)
Evan Siroky8ccaf0b2016-09-03 11:36:13 -0700393 } else {
394 var err = new Error('unknown op: ' + task.op)
395 return cb(err)
evansiroky4be1c7a2016-06-16 18:23:34 -0700396 }
evansiroky35f64342016-06-16 22:17:04 -0700397 cb()
Evan Siroky4fc596c2016-09-25 19:52:30 -0700398 },
Evan Siroky7891a6e2016-11-05 11:50:50 -0700399 function (err) {
400 if (err) { return callback(err) }
Evan Siroky4fc596c2016-09-25 19:52:30 -0700401 fs.writeFile(getTzDistFilename(tzid),
Evan Siroky477ece62017-08-01 07:08:51 -0700402 postProcessZone(geom),
evansirokybecb56e2016-07-06 12:42:35 -0700403 callback)
evansiroky4be1c7a2016-06-16 18:23:34 -0700404 })
405}
406
Evan Siroky4fc596c2016-09-25 19:52:30 -0700407var loadDistZonesIntoMemory = function () {
408 console.log('load zones into memory')
Evan Siroky7891a6e2016-11-05 11:50:50 -0700409 var zones = Object.keys(zoneCfg)
410 var tzid
Evan Siroky4fc596c2016-09-25 19:52:30 -0700411
412 for (var i = 0; i < zones.length; i++) {
413 tzid = zones[i]
414 distZones[tzid] = getDataSource({ source: 'dist', id: tzid })
415 }
416}
417
418var getDistZoneGeom = function (tzid) {
419 return distZones[tzid]
420}
421
evansiroky92c15c42018-11-15 20:58:18 -0800422var roundDownToTenth = function (n) {
423 return Math.floor(n * 10) / 10
424}
425
426var roundUpToTenth = function (n) {
427 return Math.ceil(n * 10) / 10
428}
429
430var formatBounds = function (bounds) {
431 let boundsStr = '['
432 boundsStr += roundDownToTenth(bounds[0]) + ', '
433 boundsStr += roundDownToTenth(bounds[1]) + ', '
434 boundsStr += roundUpToTenth(bounds[2]) + ', '
435 boundsStr += roundUpToTenth(bounds[3]) + ']'
436 return boundsStr
437}
438
Evan Siroky4fc596c2016-09-25 19:52:30 -0700439var validateTimezoneBoundaries = function () {
evansiroky5348a6f2019-01-05 15:39:28 -0800440 const numZones = Object.keys(zoneCfg).length
441 const validationProgress = new ProgressStats(
442 'Validation',
443 numZones * (numZones + 1) / 2
444 )
445
evansiroky26325842018-04-03 14:10:42 -0700446 console.log('do validation... this may take a few minutes')
Evan Siroky7891a6e2016-11-05 11:50:50 -0700447 var allZonesOk = true
448 var zones = Object.keys(zoneCfg)
evansiroky3046a3d2019-01-05 21:19:14 -0800449 var lastPct = 0
Evan Siroky7891a6e2016-11-05 11:50:50 -0700450 var compareTzid, tzid, zoneGeom
Evan Siroky4fc596c2016-09-25 19:52:30 -0700451
452 for (var i = 0; i < zones.length; i++) {
453 tzid = zones[i]
454 zoneGeom = getDistZoneGeom(tzid)
455
456 for (var j = i + 1; j < zones.length; j++) {
evansiroky3046a3d2019-01-05 21:19:14 -0800457 const curPct = Math.floor(validationProgress.getPercentage())
458 if (curPct % 10 === 0 && curPct !== lastPct) {
evansiroky5348a6f2019-01-05 15:39:28 -0800459 validationProgress.printStats('Validating zones', true)
evansiroky3046a3d2019-01-05 21:19:14 -0800460 lastPct = curPct
evansiroky5348a6f2019-01-05 15:39:28 -0800461 }
Evan Siroky4fc596c2016-09-25 19:52:30 -0700462 compareTzid = zones[j]
463
464 var compareZoneGeom = getDistZoneGeom(compareTzid)
Evan Siroky070bbb92017-03-07 23:48:29 -0800465
466 var intersects = false
467 try {
468 intersects = debugGeo('intersects', zoneGeom, compareZoneGeom)
469 } catch (e) {
470 console.warn('warning, encountered intersection error with zone ' + tzid + ' and ' + compareTzid)
471 }
472 if (intersects) {
Evan Siroky7891a6e2016-11-05 11:50:50 -0700473 var intersectedGeom = debugGeo('intersection', zoneGeom, compareZoneGeom)
474 var intersectedArea = intersectedGeom.getArea()
Evan Siroky4fc596c2016-09-25 19:52:30 -0700475
Evan Siroky7891a6e2016-11-05 11:50:50 -0700476 if (intersectedArea > 0.0001) {
evansiroky0ea1d1e2018-10-30 22:30:51 -0700477 // check if the intersected area(s) are one of the expected areas of overlap
478 const allowedOverlapBounds = expectedZoneOverlaps[`${tzid}-${compareTzid}`] || expectedZoneOverlaps[`${compareTzid}-${tzid}`]
479 const overlapsGeoJson = geoJsonWriter.write(intersectedGeom)
480
481 // these zones are allowed to overlap in certain places, make sure the
482 // found overlap(s) all fit within the expected areas of overlap
483 if (allowedOverlapBounds) {
484 // if the overlaps are a multipolygon, make sure each individual
485 // polygon of overlap fits within at least one of the expected
486 // overlaps
487 let overlapsPolygons
488 switch (overlapsGeoJson.type) {
evansiroky92c15c42018-11-15 20:58:18 -0800489 case 'MultiPolygon':
490 overlapsPolygons = overlapsGeoJson.coordinates.map(
491 polygonCoords => ({
492 coordinates: polygonCoords,
493 type: 'Polygon'
494 })
495 )
496 break
evansiroky0ea1d1e2018-10-30 22:30:51 -0700497 case 'Polygon':
498 overlapsPolygons = [overlapsGeoJson]
499 break
evansiroky92c15c42018-11-15 20:58:18 -0800500 case 'GeometryCollection':
501 overlapsPolygons = []
502 overlapsGeoJson.geometries.forEach(geom => {
503 if (geom.type === 'Polygon') {
504 overlapsPolygons.push(geom)
505 } else if (geom.type === 'MultiPolygon') {
506 geom.coordinates.forEach(polygonCoords => {
507 overlapsPolygons.push({
508 coordinates: polygonCoords,
509 type: 'Polygon'
510 })
511 })
512 }
513 })
514 break
evansiroky0ea1d1e2018-10-30 22:30:51 -0700515 default:
evansiroky92c15c42018-11-15 20:58:18 -0800516 console.error('unexpected geojson overlap type')
517 console.log(overlapsGeoJson)
evansiroky0ea1d1e2018-10-30 22:30:51 -0700518 break
519 }
520
521 let allOverlapsOk = true
522 overlapsPolygons.forEach((polygon, idx) => {
523 const bounds = bbox(polygon)
evansiroky92c15c42018-11-15 20:58:18 -0800524 const polygonArea = area.geometry(polygon)
evansiroky0ea1d1e2018-10-30 22:30:51 -0700525 if (
evansiroky92c15c42018-11-15 20:58:18 -0800526 polygonArea > 10 && // ignore small polygons
evansiroky0ea1d1e2018-10-30 22:30:51 -0700527 !allowedOverlapBounds.some(allowedBounds =>
evansiroky92c15c42018-11-15 20:58:18 -0800528 allowedBounds.bounds[0] <= bounds[0] && // minX
529 allowedBounds.bounds[1] <= bounds[1] && // minY
530 allowedBounds.bounds[2] >= bounds[2] && // maxX
531 allowedBounds.bounds[3] >= bounds[3] // maxY
evansiroky0ea1d1e2018-10-30 22:30:51 -0700532 )
533 ) {
evansiroky92c15c42018-11-15 20:58:18 -0800534 console.error(`Unexpected intersection (${polygonArea} area) with bounds: ${formatBounds(bounds)}`)
evansiroky0ea1d1e2018-10-30 22:30:51 -0700535 allOverlapsOk = false
536 }
537 })
538
539 if (allOverlapsOk) continue
540 }
541
evansiroky92c15c42018-11-15 20:58:18 -0800542 // at least one unexpected overlap found, output an error and write debug file
evansiroky70b35fe2018-04-01 21:06:36 -0700543 console.error('Validation error: ' + tzid + ' intersects ' + compareTzid + ' area: ' + intersectedArea)
evansiroky92c15c42018-11-15 20:58:18 -0800544 const debugFilename = tzid.replace(/\//g, '-') + '-' + compareTzid.replace(/\//g, '-') + '-overlap.json'
evansiroky70b35fe2018-04-01 21:06:36 -0700545 fs.writeFileSync(
546 debugFilename,
evansiroky0ea1d1e2018-10-30 22:30:51 -0700547 JSON.stringify(overlapsGeoJson)
evansiroky70b35fe2018-04-01 21:06:36 -0700548 )
549 console.error('wrote overlap area as file ' + debugFilename)
550 console.error('To read more about this error, please visit https://git.io/vx6nx')
Evan Siroky4fc596c2016-09-25 19:52:30 -0700551 allZonesOk = false
552 }
553 }
evansiroky5348a6f2019-01-05 15:39:28 -0800554 validationProgress.logNext()
Evan Siroky4fc596c2016-09-25 19:52:30 -0700555 }
556 }
557
558 return allZonesOk ? null : 'Zone validation unsuccessful'
Evan Siroky4fc596c2016-09-25 19:52:30 -0700559}
560
evansiroky26325842018-04-03 14:10:42 -0700561let oceanZoneBoundaries
evansiroky9fd50512019-07-07 12:06:28 -0700562let oceanZones = [
563 { tzid: 'Etc/GMT-12', left: 172.5, right: 180 },
564 { tzid: 'Etc/GMT-11', left: 157.5, right: 172.5 },
565 { tzid: 'Etc/GMT-10', left: 142.5, right: 157.5 },
566 { tzid: 'Etc/GMT-9', left: 127.5, right: 142.5 },
567 { tzid: 'Etc/GMT-8', left: 112.5, right: 127.5 },
568 { tzid: 'Etc/GMT-7', left: 97.5, right: 112.5 },
569 { tzid: 'Etc/GMT-6', left: 82.5, right: 97.5 },
570 { tzid: 'Etc/GMT-5', left: 67.5, right: 82.5 },
571 { tzid: 'Etc/GMT-4', left: 52.5, right: 67.5 },
572 { tzid: 'Etc/GMT-3', left: 37.5, right: 52.5 },
573 { tzid: 'Etc/GMT-2', left: 22.5, right: 37.5 },
574 { tzid: 'Etc/GMT-1', left: 7.5, right: 22.5 },
575 { tzid: 'Etc/GMT', left: -7.5, right: 7.5 },
576 { tzid: 'Etc/GMT+1', left: -22.5, right: -7.5 },
577 { tzid: 'Etc/GMT+2', left: -37.5, right: -22.5 },
578 { tzid: 'Etc/GMT+3', left: -52.5, right: -37.5 },
579 { tzid: 'Etc/GMT+4', left: -67.5, right: -52.5 },
580 { tzid: 'Etc/GMT+5', left: -82.5, right: -67.5 },
581 { tzid: 'Etc/GMT+6', left: -97.5, right: -82.5 },
582 { tzid: 'Etc/GMT+7', left: -112.5, right: -97.5 },
583 { tzid: 'Etc/GMT+8', left: -127.5, right: -112.5 },
584 { tzid: 'Etc/GMT+9', left: -142.5, right: -127.5 },
585 { tzid: 'Etc/GMT+10', left: -157.5, right: -142.5 },
586 { tzid: 'Etc/GMT+11', left: -172.5, right: -157.5 },
587 { tzid: 'Etc/GMT+12', left: -180, right: -172.5 }
588]
589
Neil Fullerc4ae49b2020-04-30 18:08:43 +0100590if (includedZones.length > 0) {
591 oceanZones = oceanZones.filter(oceanZone => includedZones.indexOf(oceanZone) > -1)
evansiroky9fd50512019-07-07 12:06:28 -0700592}
evansiroky26325842018-04-03 14:10:42 -0700593
594var addOceans = function (callback) {
595 console.log('adding ocean boundaries')
evansiroky26325842018-04-03 14:10:42 -0700596 const zones = Object.keys(zoneCfg)
597
evansiroky3046a3d2019-01-05 21:19:14 -0800598 const oceanProgress = new ProgressStats(
599 'Oceans',
600 oceanZones.length
601 )
602
evansiroky26325842018-04-03 14:10:42 -0700603 oceanZoneBoundaries = oceanZones.map(zone => {
evansiroky3046a3d2019-01-05 21:19:14 -0800604 oceanProgress.beginTask(zone.tzid, true)
evansiroky26325842018-04-03 14:10:42 -0700605 const geoJson = polygon([[
606 [zone.left, 90],
evansiroky0ea1d1e2018-10-30 22:30:51 -0700607 [zone.left, -90],
608 [zone.right, -90],
609 [zone.right, 90],
evansiroky26325842018-04-03 14:10:42 -0700610 [zone.left, 90]
611 ]]).geometry
612
613 let geom = geoJsonToGeom(geoJson)
614
615 // diff against every zone
616 zones.forEach(distZone => {
617 geom = debugGeo('diff', geom, getDistZoneGeom(distZone))
618 })
619
620 return {
621 geom: postProcessZone(geom, true),
622 tzid: zone.tzid
623 }
624 })
625
626 callback()
627}
628
Evan Siroky7891a6e2016-11-05 11:50:50 -0700629var combineAndWriteZones = function (callback) {
Evan Siroky8b47abe2016-10-02 12:28:52 -0700630 var stream = fs.createWriteStream('./dist/combined.json')
evansiroky26325842018-04-03 14:10:42 -0700631 var streamWithOceans = fs.createWriteStream('./dist/combined-with-oceans.json')
Evan Siroky8b47abe2016-10-02 12:28:52 -0700632 var zones = Object.keys(zoneCfg)
633
634 stream.write('{"type":"FeatureCollection","features":[')
evansiroky26325842018-04-03 14:10:42 -0700635 streamWithOceans.write('{"type":"FeatureCollection","features":[')
Evan Siroky8b47abe2016-10-02 12:28:52 -0700636
637 for (var i = 0; i < zones.length; i++) {
Evan Siroky7891a6e2016-11-05 11:50:50 -0700638 if (i > 0) {
Evan Siroky8b47abe2016-10-02 12:28:52 -0700639 stream.write(',')
evansiroky26325842018-04-03 14:10:42 -0700640 streamWithOceans.write(',')
Evan Siroky8b47abe2016-10-02 12:28:52 -0700641 }
642 var feature = {
643 type: 'Feature',
644 properties: { tzid: zones[i] },
645 geometry: geomToGeoJson(getDistZoneGeom(zones[i]))
646 }
evansiroky26325842018-04-03 14:10:42 -0700647 const stringified = JSON.stringify(feature)
648 stream.write(stringified)
649 streamWithOceans.write(stringified)
Evan Siroky8b47abe2016-10-02 12:28:52 -0700650 }
evansiroky26325842018-04-03 14:10:42 -0700651 oceanZoneBoundaries.forEach(boundary => {
652 streamWithOceans.write(',')
653 var feature = {
654 type: 'Feature',
655 properties: { tzid: boundary.tzid },
656 geometry: boundary.geom
657 }
658 streamWithOceans.write(JSON.stringify(feature))
659 })
660 asynclib.parallel([
661 cb => {
662 stream.end(']}', cb)
663 },
664 cb => {
665 streamWithOceans.end(']}', cb)
666 }
667 ], callback)
Evan Siroky8b47abe2016-10-02 12:28:52 -0700668}
669
evansiroky5348a6f2019-01-05 15:39:28 -0800670const autoScript = {
Evan Siroky7891a6e2016-11-05 11:50:50 -0700671 makeDownloadsDir: function (cb) {
evansiroky5348a6f2019-01-05 15:39:28 -0800672 overallProgress.beginTask('Creating downloads dir')
evansiroky4be1c7a2016-06-16 18:23:34 -0700673 safeMkdir('./downloads', cb)
674 },
Evan Siroky7891a6e2016-11-05 11:50:50 -0700675 makeDistDir: function (cb) {
evansiroky5348a6f2019-01-05 15:39:28 -0800676 overallProgress.beginTask('Creating dist dir')
evansiroky4be1c7a2016-06-16 18:23:34 -0700677 safeMkdir('./dist', cb)
evansirokyd401c892016-06-16 00:05:14 -0700678 },
Evan Siroky7891a6e2016-11-05 11:50:50 -0700679 getOsmBoundaries: ['makeDownloadsDir', function (results, cb) {
evansiroky5348a6f2019-01-05 15:39:28 -0800680 overallProgress.beginTask('Downloading osm boundaries')
Evan Siroky7891a6e2016-11-05 11:50:50 -0700681 asynclib.eachSeries(Object.keys(osmBoundarySources), downloadOsmBoundary, cb)
evansiroky63d35e12016-06-16 10:08:15 -0700682 }],
evansirokya48b3922020-04-27 15:29:06 -0700683 zipInputData: ['makeDistDir', 'getOsmBoundaries', function (results, cb) {
684 overallProgress.beginTask('Zipping up input data')
685 exec('zip dist/input-data.zip downloads/* timezones.json osmBoundarySources.json expectedZoneOverlaps.json', cb)
686 }],
Evan Siroky7891a6e2016-11-05 11:50:50 -0700687 createZones: ['makeDistDir', 'getOsmBoundaries', function (results, cb) {
evansiroky5348a6f2019-01-05 15:39:28 -0800688 overallProgress.beginTask('Creating timezone boundaries')
Evan Siroky7891a6e2016-11-05 11:50:50 -0700689 asynclib.each(Object.keys(zoneCfg), makeTimezoneBoundary, cb)
evansiroky50216c62016-06-16 17:41:47 -0700690 }],
Evan Siroky7891a6e2016-11-05 11:50:50 -0700691 validateZones: ['createZones', function (results, cb) {
evansiroky5348a6f2019-01-05 15:39:28 -0800692 overallProgress.beginTask('Validating timezone boundaries')
Evan Siroky4fc596c2016-09-25 19:52:30 -0700693 loadDistZonesIntoMemory()
Neil Fullerc4ae49b2020-04-30 18:08:43 +0100694 if (argv.no_validation) {
Evan Siroky081648a2017-07-04 09:53:36 -0700695 console.warn('WARNING: Skipping validation!')
696 cb()
697 } else {
698 cb(validateTimezoneBoundaries())
699 }
Evan Siroky4fc596c2016-09-25 19:52:30 -0700700 }],
evansiroky26325842018-04-03 14:10:42 -0700701 addOceans: ['validateZones', function (results, cb) {
evansiroky5348a6f2019-01-05 15:39:28 -0800702 overallProgress.beginTask('Adding oceans')
evansiroky26325842018-04-03 14:10:42 -0700703 addOceans(cb)
704 }],
705 mergeZones: ['addOceans', function (results, cb) {
evansiroky5348a6f2019-01-05 15:39:28 -0800706 overallProgress.beginTask('Merging zones')
Evan Siroky8b47abe2016-10-02 12:28:52 -0700707 combineAndWriteZones(cb)
708 }],
709 zipGeoJson: ['mergeZones', function (results, cb) {
evansiroky5348a6f2019-01-05 15:39:28 -0800710 overallProgress.beginTask('Zipping geojson')
Evan Siroky8b47abe2016-10-02 12:28:52 -0700711 exec('zip dist/timezones.geojson.zip dist/combined.json', cb)
712 }],
evansiroky26325842018-04-03 14:10:42 -0700713 zipGeoJsonWithOceans: ['mergeZones', function (results, cb) {
evansiroky5348a6f2019-01-05 15:39:28 -0800714 overallProgress.beginTask('Zipping geojson with oceans')
evansiroky26325842018-04-03 14:10:42 -0700715 exec('zip dist/timezones-with-oceans.geojson.zip dist/combined-with-oceans.json', cb)
716 }],
Evan Siroky8b47abe2016-10-02 12:28:52 -0700717 makeShapefile: ['mergeZones', function (results, cb) {
evansiroky5348a6f2019-01-05 15:39:28 -0800718 overallProgress.beginTask('Converting from geojson to shapefile')
evansiroky26325842018-04-03 14:10:42 -0700719 rimraf.sync('dist/combined-shapefile.*')
720 exec(
evansirokye3360f72018-11-16 09:24:26 -0800721 'ogr2ogr -f "ESRI Shapefile" dist/combined-shapefile.shp dist/combined.json',
evansiroky26325842018-04-03 14:10:42 -0700722 function (err, stdout, stderr) {
723 if (err) { return cb(err) }
724 exec('zip dist/timezones.shapefile.zip dist/combined-shapefile.*', cb)
725 }
726 )
727 }],
728 makeShapefileWithOceans: ['mergeZones', function (results, cb) {
evansiroky5348a6f2019-01-05 15:39:28 -0800729 overallProgress.beginTask('Converting from geojson with oceans to shapefile')
evansiroky26325842018-04-03 14:10:42 -0700730 rimraf.sync('dist/combined-shapefile-with-oceans.*')
731 exec(
evansirokye3360f72018-11-16 09:24:26 -0800732 'ogr2ogr -f "ESRI Shapefile" dist/combined-shapefile-with-oceans.shp dist/combined-with-oceans.json',
evansiroky26325842018-04-03 14:10:42 -0700733 function (err, stdout, stderr) {
734 if (err) { return cb(err) }
735 exec('zip dist/timezones-with-oceans.shapefile.zip dist/combined-shapefile-with-oceans.*', cb)
736 }
737 )
evansiroky9fd50512019-07-07 12:06:28 -0700738 }],
739 makeListOfTimeZoneNames: function (cb) {
740 overallProgress.beginTask('Writing timezone names to file')
741 let zoneNames = Object.keys(zoneCfg)
742 oceanZones.forEach(oceanZone => {
743 zoneNames.push(oceanZone.tzid)
744 })
Neil Fullerc4ae49b2020-04-30 18:08:43 +0100745 if (includedZones.length > 0) {
746 zoneNames = zoneNames.filter(zoneName => includedZones.indexOf(zoneName) > -1)
evansiroky9fd50512019-07-07 12:06:28 -0700747 }
748 fs.writeFile(
749 'dist/timezone-names.json',
750 JSON.stringify(zoneNames),
751 cb
752 )
753 }
evansiroky5348a6f2019-01-05 15:39:28 -0800754}
755
756const overallProgress = new ProgressStats('Overall', Object.keys(autoScript).length)
757
758asynclib.auto(autoScript, function (err, results) {
evansirokyd401c892016-06-16 00:05:14 -0700759 console.log('done')
Evan Siroky7891a6e2016-11-05 11:50:50 -0700760 if (err) {
evansirokyd401c892016-06-16 00:05:14 -0700761 console.log('error!', err)
evansirokyd401c892016-06-16 00:05:14 -0700762 }
Evan Siroky4fc596c2016-09-25 19:52:30 -0700763})