blob: 72b367b203e3c541f4d4cdf2b604dce13af7363f [file] [log] [blame]
Evan Siroky7891a6e2016-11-05 11:50:50 -07001var exec = require('child_process').exec
2var fs = require('fs')
evansirokyd401c892016-06-16 00:05:14 -07003
Evan Siroky477ece62017-08-01 07:08:51 -07004var area = require('@mapbox/geojson-area')
evansiroky70b35fe2018-04-01 21:06:36 -07005var geojsonhint = require('@mapbox/geojsonhint')
evansiroky0ea1d1e2018-10-30 22:30:51 -07006var bbox = require('@turf/bbox').default
Evan Siroky8326cf02017-03-02 08:27:55 -08007var helpers = require('@turf/helpers')
Evan Siroky070bbb92017-03-07 23:48:29 -08008var multiPolygon = helpers.multiPolygon
Evan Siroky8326cf02017-03-02 08:27:55 -08009var polygon = helpers.polygon
Evan Siroky7891a6e2016-11-05 11:50:50 -070010var asynclib = require('async')
11var jsts = require('jsts')
Evan Sirokyb57a5b92016-11-07 10:22:34 -080012var rimraf = require('rimraf')
Evan Siroky7891a6e2016-11-05 11:50:50 -070013var overpass = require('query-overpass')
evansirokyd401c892016-06-16 00:05:14 -070014
evansiroky5348a6f2019-01-05 15:39:28 -080015const ProgressStats = require('./progressStats')
16
Evan Siroky7891a6e2016-11-05 11:50:50 -070017var osmBoundarySources = require('./osmBoundarySources.json')
18var zoneCfg = require('./timezones.json')
evansiroky0ea1d1e2018-10-30 22:30:51 -070019var expectedZoneOverlaps = require('./expectedZoneOverlaps.json')
Evan Siroky081648a2017-07-04 09:53:36 -070020
21// allow building of only a specified zones
22var filteredIndex = process.argv.indexOf('--filtered-zones')
evansiroky9fd50512019-07-07 12:06:28 -070023let filteredZones = []
Evan Siroky081648a2017-07-04 09:53:36 -070024if (filteredIndex > -1 && process.argv[filteredIndex + 1]) {
evansiroky9fd50512019-07-07 12:06:28 -070025 filteredZones = process.argv[filteredIndex + 1].split(',')
Evan Siroky081648a2017-07-04 09:53:36 -070026 var newZoneCfg = {}
27 filteredZones.forEach((zoneName) => {
28 newZoneCfg[zoneName] = zoneCfg[zoneName]
29 })
30 zoneCfg = newZoneCfg
31
32 // filter out unneccessary downloads
33 var newOsmBoundarySources = {}
34 Object.keys(zoneCfg).forEach((zoneName) => {
35 zoneCfg[zoneName].forEach((op) => {
36 if (op.source === 'overpass') {
37 newOsmBoundarySources[op.id] = osmBoundarySources[op.id]
38 }
39 })
40 })
41
42 osmBoundarySources = newOsmBoundarySources
43}
44
Evan Siroky7891a6e2016-11-05 11:50:50 -070045var geoJsonReader = new jsts.io.GeoJSONReader()
46var geoJsonWriter = new jsts.io.GeoJSONWriter()
Evan Siroky477ece62017-08-01 07:08:51 -070047var precisionModel = new jsts.geom.PrecisionModel(1000000)
48var precisionReducer = new jsts.precision.GeometryPrecisionReducer(precisionModel)
Evan Siroky7891a6e2016-11-05 11:50:50 -070049var distZones = {}
Evan Sirokyb57a5b92016-11-07 10:22:34 -080050var minRequestGap = 4
51var curRequestGap = 4
evansirokyd401c892016-06-16 00:05:14 -070052
Evan Siroky7891a6e2016-11-05 11:50:50 -070053var safeMkdir = function (dirname, callback) {
54 fs.mkdir(dirname, function (err) {
55 if (err && err.code === 'EEXIST') {
evansiroky4be1c7a2016-06-16 18:23:34 -070056 callback()
57 } else {
58 callback(err)
59 }
60 })
61}
62
Evan Sirokyb173fd42017-03-08 15:16:27 -080063var debugGeo = function (op, a, b, reducePrecision) {
evansirokybecb56e2016-07-06 12:42:35 -070064 var result
65
Evan Sirokyb173fd42017-03-08 15:16:27 -080066 if (reducePrecision) {
Evan Sirokyb173fd42017-03-08 15:16:27 -080067 a = precisionReducer.reduce(a)
68 b = precisionReducer.reduce(b)
69 }
70
evansiroky6f9d8f72016-06-21 16:27:54 -070071 try {
Evan Siroky7891a6e2016-11-05 11:50:50 -070072 switch (op) {
evansiroky6f9d8f72016-06-21 16:27:54 -070073 case 'union':
evansirokybecb56e2016-07-06 12:42:35 -070074 result = a.union(b)
evansiroky6f9d8f72016-06-21 16:27:54 -070075 break
76 case 'intersection':
evansirokybecb56e2016-07-06 12:42:35 -070077 result = a.intersection(b)
evansiroky6f9d8f72016-06-21 16:27:54 -070078 break
Evan Siroky070bbb92017-03-07 23:48:29 -080079 case 'intersects':
80 result = a.intersects(b)
81 break
evansiroky6f9d8f72016-06-21 16:27:54 -070082 case 'diff':
Evan Sirokyb173fd42017-03-08 15:16:27 -080083 result = a.difference(b)
evansiroky6f9d8f72016-06-21 16:27:54 -070084 break
85 default:
86 var err = new Error('invalid op: ' + op)
87 throw err
88 }
Evan Siroky7891a6e2016-11-05 11:50:50 -070089 } catch (e) {
Evan Sirokyb173fd42017-03-08 15:16:27 -080090 if (e.name === 'TopologyException') {
91 console.log('Encountered TopologyException, retry with GeometryPrecisionReducer')
92 return debugGeo(op, a, b, true)
93 }
evansiroky6f9d8f72016-06-21 16:27:54 -070094 console.log('op err')
evansirokybecb56e2016-07-06 12:42:35 -070095 console.log(e)
96 console.log(e.stack)
97 fs.writeFileSync('debug_' + op + '_a.json', JSON.stringify(geoJsonWriter.write(a)))
98 fs.writeFileSync('debug_' + op + '_b.json', JSON.stringify(geoJsonWriter.write(b)))
evansiroky6f9d8f72016-06-21 16:27:54 -070099 throw e
100 }
evansiroky6f9d8f72016-06-21 16:27:54 -0700101
evansirokybecb56e2016-07-06 12:42:35 -0700102 return result
evansiroky4be1c7a2016-06-16 18:23:34 -0700103}
104
Evan Siroky070bbb92017-03-07 23:48:29 -0800105var fetchIfNeeded = function (file, superCallback, downloadCallback, fetchFn) {
106 // check for file that got downloaded
Evan Siroky7891a6e2016-11-05 11:50:50 -0700107 fs.stat(file, function (err) {
Evan Siroky070bbb92017-03-07 23:48:29 -0800108 if (!err) {
109 // file found, skip download steps
110 return superCallback()
111 }
112 // check for manual file that got fixed and needs validation
113 var fixedFile = file.replace('.json', '_fixed.json')
114 fs.stat(fixedFile, function (err) {
115 if (!err) {
116 // file found, return fixed file
117 return downloadCallback(null, require(fixedFile))
118 }
119 // no manual fixed file found, download from overpass
120 fetchFn()
121 })
evansiroky50216c62016-06-16 17:41:47 -0700122 })
123}
124
Evan Siroky7891a6e2016-11-05 11:50:50 -0700125var geoJsonToGeom = function (geoJson) {
Evan Siroky8326cf02017-03-02 08:27:55 -0800126 try {
127 return geoJsonReader.read(JSON.stringify(geoJson))
128 } catch (e) {
129 console.error('error converting geojson to geometry')
130 fs.writeFileSync('debug_geojson_read_error.json', JSON.stringify(geoJson))
131 throw e
132 }
Evan Siroky5669adc2016-07-07 17:25:31 -0700133}
134
Evan Siroky8b47abe2016-10-02 12:28:52 -0700135var geomToGeoJson = function (geom) {
136 return geoJsonWriter.write(geom)
137}
138
Evan Siroky7891a6e2016-11-05 11:50:50 -0700139var geomToGeoJsonString = function (geom) {
Evan Siroky5669adc2016-07-07 17:25:31 -0700140 return JSON.stringify(geoJsonWriter.write(geom))
141}
142
evansiroky5348a6f2019-01-05 15:39:28 -0800143const downloadProgress = new ProgressStats(
144 'Downloading',
145 Object.keys(osmBoundarySources).length
146)
147
Evan Siroky7891a6e2016-11-05 11:50:50 -0700148var downloadOsmBoundary = function (boundaryId, boundaryCallback) {
149 var cfg = osmBoundarySources[boundaryId]
Evan Siroky1bcd4772017-10-14 23:47:21 -0700150 var query = '[out:json][timeout:60];('
151 if (cfg.way) {
152 query += 'way'
153 } else {
154 query += 'relation'
155 }
Evan Siroky7891a6e2016-11-05 11:50:50 -0700156 var boundaryFilename = './downloads/' + boundaryId + '.json'
157 var debug = 'getting data for ' + boundaryId
158 var queryKeys = Object.keys(cfg)
evansiroky63d35e12016-06-16 10:08:15 -0700159
Evan Siroky5669adc2016-07-07 17:25:31 -0700160 for (var i = queryKeys.length - 1; i >= 0; i--) {
Evan Siroky7891a6e2016-11-05 11:50:50 -0700161 var k = queryKeys[i]
Evan Siroky1bcd4772017-10-14 23:47:21 -0700162 if (k === 'way') continue
Evan Siroky7891a6e2016-11-05 11:50:50 -0700163 var v = cfg[k]
Evan Siroky5669adc2016-07-07 17:25:31 -0700164
165 query += '["' + k + '"="' + v + '"]'
evansiroky63d35e12016-06-16 10:08:15 -0700166 }
167
evansiroky283ebbc2018-07-16 15:13:07 -0700168 query += ';);out body;>;out meta qt;'
evansiroky4be1c7a2016-06-16 18:23:34 -0700169
evansiroky5348a6f2019-01-05 15:39:28 -0800170 downloadProgress.beginTask(debug, true)
evansiroky63d35e12016-06-16 10:08:15 -0700171
Evan Siroky7891a6e2016-11-05 11:50:50 -0700172 asynclib.auto({
173 downloadFromOverpass: function (cb) {
evansiroky5348a6f2019-01-05 15:39:28 -0800174 console.log('downloading from overpass')
Evan Siroky070bbb92017-03-07 23:48:29 -0800175 fetchIfNeeded(boundaryFilename, boundaryCallback, cb, function () {
Evan Sirokyb57a5b92016-11-07 10:22:34 -0800176 var overpassResponseHandler = function (err, data) {
177 if (err) {
178 console.log(err)
179 console.log('Increasing overpass request gap')
180 curRequestGap *= 2
181 makeQuery()
182 } else {
183 console.log('Success, decreasing overpass request gap')
184 curRequestGap = Math.max(minRequestGap, curRequestGap / 2)
185 cb(null, data)
186 }
187 }
188 var makeQuery = function () {
189 console.log('waiting ' + curRequestGap + ' seconds')
190 setTimeout(function () {
191 overpass(query, overpassResponseHandler, { flatProperties: true })
192 }, curRequestGap * 1000)
193 }
194 makeQuery()
evansiroky63d35e12016-06-16 10:08:15 -0700195 })
196 },
Evan Siroky7891a6e2016-11-05 11:50:50 -0700197 validateOverpassResult: ['downloadFromOverpass', function (results, cb) {
evansiroky63d35e12016-06-16 10:08:15 -0700198 var data = results.downloadFromOverpass
evansiroky70b35fe2018-04-01 21:06:36 -0700199 if (!data.features) {
Evan Siroky7891a6e2016-11-05 11:50:50 -0700200 var err = new Error('Invalid geojson for boundary: ' + boundaryId)
evansiroky63d35e12016-06-16 10:08:15 -0700201 return cb(err)
202 }
evansiroky70b35fe2018-04-01 21:06:36 -0700203 if (data.features.length === 0) {
204 console.error('No data for the following query:')
205 console.error(query)
206 console.error('To read more about this error, please visit https://git.io/vxKQL')
evansiroky0ea1d1e2018-10-30 22:30:51 -0700207 return cb(new Error('No data found for from overpass query'))
evansiroky70b35fe2018-04-01 21:06:36 -0700208 }
evansiroky63d35e12016-06-16 10:08:15 -0700209 cb()
210 }],
Evan Siroky7891a6e2016-11-05 11:50:50 -0700211 saveSingleMultiPolygon: ['validateOverpassResult', function (results, cb) {
212 var data = results.downloadFromOverpass
213 var combined
evansiroky63d35e12016-06-16 10:08:15 -0700214
215 // union all multi-polygons / polygons into one
216 for (var i = data.features.length - 1; i >= 0; i--) {
Evan Siroky5669adc2016-07-07 17:25:31 -0700217 var curOsmGeom = data.features[i].geometry
evansiroky92c15c42018-11-15 20:58:18 -0800218 const curOsmProps = data.features[i].properties
219 if (
220 (curOsmGeom.type === 'Polygon' || curOsmGeom.type === 'MultiPolygon') &&
221 curOsmProps.type === 'boundary' // need to make sure enclaves aren't unioned
222 ) {
evansiroky63d35e12016-06-16 10:08:15 -0700223 console.log('combining border')
evansiroky70b35fe2018-04-01 21:06:36 -0700224 let errors = geojsonhint.hint(curOsmGeom)
225 if (errors && errors.length > 0) {
226 const stringifiedGeojson = JSON.stringify(curOsmGeom, null, 2)
227 errors = geojsonhint.hint(stringifiedGeojson)
228 console.error('Invalid geojson received in Overpass Result')
229 console.error('Overpass query: ' + query)
230 const problemFilename = boundaryId + '_convert_to_geom_error.json'
231 fs.writeFileSync(problemFilename, stringifiedGeojson)
232 console.error('saved problem file to ' + problemFilename)
233 console.error('To read more about this error, please visit https://git.io/vxKQq')
234 return cb(errors)
235 }
Evan Siroky070bbb92017-03-07 23:48:29 -0800236 try {
237 var curGeom = geoJsonToGeom(curOsmGeom)
238 } catch (e) {
239 console.error('error converting overpass result to geojson')
Evan Siroky477ece62017-08-01 07:08:51 -0700240 console.error(e)
evansiroky70b35fe2018-04-01 21:06:36 -0700241
Evan Siroky1bcd4772017-10-14 23:47:21 -0700242 fs.writeFileSync(boundaryId + '_convert_to_geom_error-all-features.json', JSON.stringify(data))
evansiroky70b35fe2018-04-01 21:06:36 -0700243 return cb(e)
Evan Siroky070bbb92017-03-07 23:48:29 -0800244 }
Evan Siroky7891a6e2016-11-05 11:50:50 -0700245 if (!combined) {
evansiroky63d35e12016-06-16 10:08:15 -0700246 combined = curGeom
247 } else {
Evan Siroky5669adc2016-07-07 17:25:31 -0700248 combined = debugGeo('union', curGeom, combined)
evansiroky63d35e12016-06-16 10:08:15 -0700249 }
250 }
251 }
Evan Siroky081c8e42017-05-29 14:53:52 -0700252 try {
253 fs.writeFile(boundaryFilename, geomToGeoJsonString(combined), cb)
254 } catch (e) {
255 console.error('error writing combined border to geojson')
256 fs.writeFileSync(boundaryId + '_combined_border_convert_to_geom_error.json', JSON.stringify(data))
evansiroky70b35fe2018-04-01 21:06:36 -0700257 return cb(e)
Evan Siroky081c8e42017-05-29 14:53:52 -0700258 }
evansiroky63d35e12016-06-16 10:08:15 -0700259 }]
260 }, boundaryCallback)
261}
evansirokyd401c892016-06-16 00:05:14 -0700262
Evan Siroky4fc596c2016-09-25 19:52:30 -0700263var getTzDistFilename = function (tzid) {
264 return './dist/' + tzid.replace(/\//g, '__') + '.json'
265}
266
267/**
268 * Get the geometry of the requested source data
269 *
270 * @return {Object} geom The geometry of the source
271 * @param {Object} source An object representing the data source
272 * must have `source` key and then either:
273 * - `id` if from a file
274 * - `id` if from a file
275 */
Evan Siroky7891a6e2016-11-05 11:50:50 -0700276var getDataSource = function (source) {
evansirokybecb56e2016-07-06 12:42:35 -0700277 var geoJson
Evan Siroky7891a6e2016-11-05 11:50:50 -0700278 if (source.source === 'overpass') {
evansirokybecb56e2016-07-06 12:42:35 -0700279 geoJson = require('./downloads/' + source.id + '.json')
Evan Siroky7891a6e2016-11-05 11:50:50 -0700280 } else if (source.source === 'manual-polygon') {
evansirokybecb56e2016-07-06 12:42:35 -0700281 geoJson = polygon(source.data).geometry
Evan Siroky7891a6e2016-11-05 11:50:50 -0700282 } else if (source.source === 'manual-multipolygon') {
Evan Siroky8e30a2e2016-08-06 19:55:35 -0700283 geoJson = multiPolygon(source.data).geometry
Evan Siroky7891a6e2016-11-05 11:50:50 -0700284 } else if (source.source === 'dist') {
Evan Siroky4fc596c2016-09-25 19:52:30 -0700285 geoJson = require(getTzDistFilename(source.id))
evansiroky4be1c7a2016-06-16 18:23:34 -0700286 } else {
287 var err = new Error('unknown source: ' + source.source)
288 throw err
289 }
Evan Siroky5669adc2016-07-07 17:25:31 -0700290 return geoJsonToGeom(geoJson)
evansiroky4be1c7a2016-06-16 18:23:34 -0700291}
292
Evan Siroky477ece62017-08-01 07:08:51 -0700293/**
294 * Post process created timezone boundary.
295 * - remove small holes and exclaves
296 * - reduce geometry precision
297 *
298 * @param {Geometry} geom The jsts geometry of the timezone
evansiroky26325842018-04-03 14:10:42 -0700299 * @param {boolean} returnAsObject if true, return as object, otherwise return stringified
300 * @return {Object|String} geojson as object or stringified
Evan Siroky477ece62017-08-01 07:08:51 -0700301 */
evansiroky26325842018-04-03 14:10:42 -0700302var postProcessZone = function (geom, returnAsObject) {
Evan Siroky477ece62017-08-01 07:08:51 -0700303 // reduce precision of geometry
304 const geojson = geomToGeoJson(precisionReducer.reduce(geom))
305
306 // iterate through all polygons
307 const filteredPolygons = []
308 let allPolygons = geojson.coordinates
309 if (geojson.type === 'Polygon') {
310 allPolygons = [geojson.coordinates]
311 }
312
313 allPolygons.forEach((curPolygon, idx) => {
314 // remove any polygon with very small area
315 const polygonFeature = polygon(curPolygon)
316 const polygonArea = area.geometry(polygonFeature.geometry)
317
318 if (polygonArea < 1) return
319
320 // find all holes
321 const filteredLinearRings = []
322
323 curPolygon.forEach((curLinearRing, lrIdx) => {
324 if (lrIdx === 0) {
325 // always keep first linearRing
326 filteredLinearRings.push(curLinearRing)
327 } else {
328 const polygonFromLinearRing = polygon([curLinearRing])
329 const linearRingArea = area.geometry(polygonFromLinearRing.geometry)
330
331 // only include holes with relevant area
332 if (linearRingArea > 1) {
333 filteredLinearRings.push(curLinearRing)
334 }
335 }
336 })
337
338 filteredPolygons.push(filteredLinearRings)
339 })
340
341 // recompile to geojson string
342 const newGeojson = {
343 type: geojson.type
344 }
345
346 if (geojson.type === 'Polygon') {
347 newGeojson.coordinates = filteredPolygons[0]
348 } else {
349 newGeojson.coordinates = filteredPolygons
350 }
351
evansiroky26325842018-04-03 14:10:42 -0700352 return returnAsObject ? newGeojson : JSON.stringify(newGeojson)
Evan Siroky477ece62017-08-01 07:08:51 -0700353}
354
evansiroky5348a6f2019-01-05 15:39:28 -0800355const buildingProgress = new ProgressStats(
356 'Building',
357 Object.keys(zoneCfg).length
358)
359
Evan Siroky7891a6e2016-11-05 11:50:50 -0700360var makeTimezoneBoundary = function (tzid, callback) {
evansiroky3046a3d2019-01-05 21:19:14 -0800361 buildingProgress.beginTask(`makeTimezoneBoundary for ${tzid}`, true)
evansiroky35f64342016-06-16 22:17:04 -0700362
Evan Siroky7891a6e2016-11-05 11:50:50 -0700363 var ops = zoneCfg[tzid]
364 var geom
evansiroky4be1c7a2016-06-16 18:23:34 -0700365
Evan Siroky7891a6e2016-11-05 11:50:50 -0700366 asynclib.eachSeries(ops, function (task, cb) {
evansiroky4be1c7a2016-06-16 18:23:34 -0700367 var taskData = getDataSource(task)
evansiroky6f9d8f72016-06-21 16:27:54 -0700368 console.log('-', task.op, task.id)
Evan Siroky7891a6e2016-11-05 11:50:50 -0700369 if (task.op === 'init') {
evansiroky4be1c7a2016-06-16 18:23:34 -0700370 geom = taskData
Evan Siroky7891a6e2016-11-05 11:50:50 -0700371 } else if (task.op === 'intersect') {
evansiroky6f9d8f72016-06-21 16:27:54 -0700372 geom = debugGeo('intersection', geom, taskData)
Evan Siroky7891a6e2016-11-05 11:50:50 -0700373 } else if (task.op === 'difference') {
evansiroky6f9d8f72016-06-21 16:27:54 -0700374 geom = debugGeo('diff', geom, taskData)
Evan Siroky7891a6e2016-11-05 11:50:50 -0700375 } else if (task.op === 'difference-reverse-order') {
Evan Siroky8ccaf0b2016-09-03 11:36:13 -0700376 geom = debugGeo('diff', taskData, geom)
Evan Siroky7891a6e2016-11-05 11:50:50 -0700377 } else if (task.op === 'union') {
evansiroky6f9d8f72016-06-21 16:27:54 -0700378 geom = debugGeo('union', geom, taskData)
Evan Siroky8ccaf0b2016-09-03 11:36:13 -0700379 } else {
380 var err = new Error('unknown op: ' + task.op)
381 return cb(err)
evansiroky4be1c7a2016-06-16 18:23:34 -0700382 }
evansiroky35f64342016-06-16 22:17:04 -0700383 cb()
Evan Siroky4fc596c2016-09-25 19:52:30 -0700384 },
Evan Siroky7891a6e2016-11-05 11:50:50 -0700385 function (err) {
386 if (err) { return callback(err) }
Evan Siroky4fc596c2016-09-25 19:52:30 -0700387 fs.writeFile(getTzDistFilename(tzid),
Evan Siroky477ece62017-08-01 07:08:51 -0700388 postProcessZone(geom),
evansirokybecb56e2016-07-06 12:42:35 -0700389 callback)
evansiroky4be1c7a2016-06-16 18:23:34 -0700390 })
391}
392
Evan Siroky4fc596c2016-09-25 19:52:30 -0700393var loadDistZonesIntoMemory = function () {
394 console.log('load zones into memory')
Evan Siroky7891a6e2016-11-05 11:50:50 -0700395 var zones = Object.keys(zoneCfg)
396 var tzid
Evan Siroky4fc596c2016-09-25 19:52:30 -0700397
398 for (var i = 0; i < zones.length; i++) {
399 tzid = zones[i]
400 distZones[tzid] = getDataSource({ source: 'dist', id: tzid })
401 }
402}
403
404var getDistZoneGeom = function (tzid) {
405 return distZones[tzid]
406}
407
evansiroky92c15c42018-11-15 20:58:18 -0800408var roundDownToTenth = function (n) {
409 return Math.floor(n * 10) / 10
410}
411
412var roundUpToTenth = function (n) {
413 return Math.ceil(n * 10) / 10
414}
415
416var formatBounds = function (bounds) {
417 let boundsStr = '['
418 boundsStr += roundDownToTenth(bounds[0]) + ', '
419 boundsStr += roundDownToTenth(bounds[1]) + ', '
420 boundsStr += roundUpToTenth(bounds[2]) + ', '
421 boundsStr += roundUpToTenth(bounds[3]) + ']'
422 return boundsStr
423}
424
Evan Siroky4fc596c2016-09-25 19:52:30 -0700425var validateTimezoneBoundaries = function () {
evansiroky5348a6f2019-01-05 15:39:28 -0800426 const numZones = Object.keys(zoneCfg).length
427 const validationProgress = new ProgressStats(
428 'Validation',
429 numZones * (numZones + 1) / 2
430 )
431
evansiroky26325842018-04-03 14:10:42 -0700432 console.log('do validation... this may take a few minutes')
Evan Siroky7891a6e2016-11-05 11:50:50 -0700433 var allZonesOk = true
434 var zones = Object.keys(zoneCfg)
evansiroky3046a3d2019-01-05 21:19:14 -0800435 var lastPct = 0
Evan Siroky7891a6e2016-11-05 11:50:50 -0700436 var compareTzid, tzid, zoneGeom
Evan Siroky4fc596c2016-09-25 19:52:30 -0700437
438 for (var i = 0; i < zones.length; i++) {
439 tzid = zones[i]
440 zoneGeom = getDistZoneGeom(tzid)
441
442 for (var j = i + 1; j < zones.length; j++) {
evansiroky3046a3d2019-01-05 21:19:14 -0800443 const curPct = Math.floor(validationProgress.getPercentage())
444 if (curPct % 10 === 0 && curPct !== lastPct) {
evansiroky5348a6f2019-01-05 15:39:28 -0800445 validationProgress.printStats('Validating zones', true)
evansiroky3046a3d2019-01-05 21:19:14 -0800446 lastPct = curPct
evansiroky5348a6f2019-01-05 15:39:28 -0800447 }
Evan Siroky4fc596c2016-09-25 19:52:30 -0700448 compareTzid = zones[j]
449
450 var compareZoneGeom = getDistZoneGeom(compareTzid)
Evan Siroky070bbb92017-03-07 23:48:29 -0800451
452 var intersects = false
453 try {
454 intersects = debugGeo('intersects', zoneGeom, compareZoneGeom)
455 } catch (e) {
456 console.warn('warning, encountered intersection error with zone ' + tzid + ' and ' + compareTzid)
457 }
458 if (intersects) {
Evan Siroky7891a6e2016-11-05 11:50:50 -0700459 var intersectedGeom = debugGeo('intersection', zoneGeom, compareZoneGeom)
460 var intersectedArea = intersectedGeom.getArea()
Evan Siroky4fc596c2016-09-25 19:52:30 -0700461
Evan Siroky7891a6e2016-11-05 11:50:50 -0700462 if (intersectedArea > 0.0001) {
evansiroky0ea1d1e2018-10-30 22:30:51 -0700463 // check if the intersected area(s) are one of the expected areas of overlap
464 const allowedOverlapBounds = expectedZoneOverlaps[`${tzid}-${compareTzid}`] || expectedZoneOverlaps[`${compareTzid}-${tzid}`]
465 const overlapsGeoJson = geoJsonWriter.write(intersectedGeom)
466
467 // these zones are allowed to overlap in certain places, make sure the
468 // found overlap(s) all fit within the expected areas of overlap
469 if (allowedOverlapBounds) {
470 // if the overlaps are a multipolygon, make sure each individual
471 // polygon of overlap fits within at least one of the expected
472 // overlaps
473 let overlapsPolygons
474 switch (overlapsGeoJson.type) {
evansiroky92c15c42018-11-15 20:58:18 -0800475 case 'MultiPolygon':
476 overlapsPolygons = overlapsGeoJson.coordinates.map(
477 polygonCoords => ({
478 coordinates: polygonCoords,
479 type: 'Polygon'
480 })
481 )
482 break
evansiroky0ea1d1e2018-10-30 22:30:51 -0700483 case 'Polygon':
484 overlapsPolygons = [overlapsGeoJson]
485 break
evansiroky92c15c42018-11-15 20:58:18 -0800486 case 'GeometryCollection':
487 overlapsPolygons = []
488 overlapsGeoJson.geometries.forEach(geom => {
489 if (geom.type === 'Polygon') {
490 overlapsPolygons.push(geom)
491 } else if (geom.type === 'MultiPolygon') {
492 geom.coordinates.forEach(polygonCoords => {
493 overlapsPolygons.push({
494 coordinates: polygonCoords,
495 type: 'Polygon'
496 })
497 })
498 }
499 })
500 break
evansiroky0ea1d1e2018-10-30 22:30:51 -0700501 default:
evansiroky92c15c42018-11-15 20:58:18 -0800502 console.error('unexpected geojson overlap type')
503 console.log(overlapsGeoJson)
evansiroky0ea1d1e2018-10-30 22:30:51 -0700504 break
505 }
506
507 let allOverlapsOk = true
508 overlapsPolygons.forEach((polygon, idx) => {
509 const bounds = bbox(polygon)
evansiroky92c15c42018-11-15 20:58:18 -0800510 const polygonArea = area.geometry(polygon)
evansiroky0ea1d1e2018-10-30 22:30:51 -0700511 if (
evansiroky92c15c42018-11-15 20:58:18 -0800512 polygonArea > 10 && // ignore small polygons
evansiroky0ea1d1e2018-10-30 22:30:51 -0700513 !allowedOverlapBounds.some(allowedBounds =>
evansiroky92c15c42018-11-15 20:58:18 -0800514 allowedBounds.bounds[0] <= bounds[0] && // minX
515 allowedBounds.bounds[1] <= bounds[1] && // minY
516 allowedBounds.bounds[2] >= bounds[2] && // maxX
517 allowedBounds.bounds[3] >= bounds[3] // maxY
evansiroky0ea1d1e2018-10-30 22:30:51 -0700518 )
519 ) {
evansiroky92c15c42018-11-15 20:58:18 -0800520 console.error(`Unexpected intersection (${polygonArea} area) with bounds: ${formatBounds(bounds)}`)
evansiroky0ea1d1e2018-10-30 22:30:51 -0700521 allOverlapsOk = false
522 }
523 })
524
525 if (allOverlapsOk) continue
526 }
527
evansiroky92c15c42018-11-15 20:58:18 -0800528 // at least one unexpected overlap found, output an error and write debug file
evansiroky70b35fe2018-04-01 21:06:36 -0700529 console.error('Validation error: ' + tzid + ' intersects ' + compareTzid + ' area: ' + intersectedArea)
evansiroky92c15c42018-11-15 20:58:18 -0800530 const debugFilename = tzid.replace(/\//g, '-') + '-' + compareTzid.replace(/\//g, '-') + '-overlap.json'
evansiroky70b35fe2018-04-01 21:06:36 -0700531 fs.writeFileSync(
532 debugFilename,
evansiroky0ea1d1e2018-10-30 22:30:51 -0700533 JSON.stringify(overlapsGeoJson)
evansiroky70b35fe2018-04-01 21:06:36 -0700534 )
535 console.error('wrote overlap area as file ' + debugFilename)
536 console.error('To read more about this error, please visit https://git.io/vx6nx')
Evan Siroky4fc596c2016-09-25 19:52:30 -0700537 allZonesOk = false
538 }
539 }
evansiroky5348a6f2019-01-05 15:39:28 -0800540 validationProgress.logNext()
Evan Siroky4fc596c2016-09-25 19:52:30 -0700541 }
542 }
543
544 return allZonesOk ? null : 'Zone validation unsuccessful'
Evan Siroky4fc596c2016-09-25 19:52:30 -0700545}
546
evansiroky26325842018-04-03 14:10:42 -0700547let oceanZoneBoundaries
evansiroky9fd50512019-07-07 12:06:28 -0700548let oceanZones = [
549 { tzid: 'Etc/GMT-12', left: 172.5, right: 180 },
550 { tzid: 'Etc/GMT-11', left: 157.5, right: 172.5 },
551 { tzid: 'Etc/GMT-10', left: 142.5, right: 157.5 },
552 { tzid: 'Etc/GMT-9', left: 127.5, right: 142.5 },
553 { tzid: 'Etc/GMT-8', left: 112.5, right: 127.5 },
554 { tzid: 'Etc/GMT-7', left: 97.5, right: 112.5 },
555 { tzid: 'Etc/GMT-6', left: 82.5, right: 97.5 },
556 { tzid: 'Etc/GMT-5', left: 67.5, right: 82.5 },
557 { tzid: 'Etc/GMT-4', left: 52.5, right: 67.5 },
558 { tzid: 'Etc/GMT-3', left: 37.5, right: 52.5 },
559 { tzid: 'Etc/GMT-2', left: 22.5, right: 37.5 },
560 { tzid: 'Etc/GMT-1', left: 7.5, right: 22.5 },
561 { tzid: 'Etc/GMT', left: -7.5, right: 7.5 },
562 { tzid: 'Etc/GMT+1', left: -22.5, right: -7.5 },
563 { tzid: 'Etc/GMT+2', left: -37.5, right: -22.5 },
564 { tzid: 'Etc/GMT+3', left: -52.5, right: -37.5 },
565 { tzid: 'Etc/GMT+4', left: -67.5, right: -52.5 },
566 { tzid: 'Etc/GMT+5', left: -82.5, right: -67.5 },
567 { tzid: 'Etc/GMT+6', left: -97.5, right: -82.5 },
568 { tzid: 'Etc/GMT+7', left: -112.5, right: -97.5 },
569 { tzid: 'Etc/GMT+8', left: -127.5, right: -112.5 },
570 { tzid: 'Etc/GMT+9', left: -142.5, right: -127.5 },
571 { tzid: 'Etc/GMT+10', left: -157.5, right: -142.5 },
572 { tzid: 'Etc/GMT+11', left: -172.5, right: -157.5 },
573 { tzid: 'Etc/GMT+12', left: -180, right: -172.5 }
574]
575
576if (filteredZones.length > 0) {
577 oceanZones = oceanZones.filter(oceanZone => filteredZones.indexOf(oceanZone) > -1)
578}
evansiroky26325842018-04-03 14:10:42 -0700579
580var addOceans = function (callback) {
581 console.log('adding ocean boundaries')
evansiroky26325842018-04-03 14:10:42 -0700582 const zones = Object.keys(zoneCfg)
583
evansiroky3046a3d2019-01-05 21:19:14 -0800584 const oceanProgress = new ProgressStats(
585 'Oceans',
586 oceanZones.length
587 )
588
evansiroky26325842018-04-03 14:10:42 -0700589 oceanZoneBoundaries = oceanZones.map(zone => {
evansiroky3046a3d2019-01-05 21:19:14 -0800590 oceanProgress.beginTask(zone.tzid, true)
evansiroky26325842018-04-03 14:10:42 -0700591 const geoJson = polygon([[
592 [zone.left, 90],
evansiroky0ea1d1e2018-10-30 22:30:51 -0700593 [zone.left, -90],
594 [zone.right, -90],
595 [zone.right, 90],
evansiroky26325842018-04-03 14:10:42 -0700596 [zone.left, 90]
597 ]]).geometry
598
599 let geom = geoJsonToGeom(geoJson)
600
601 // diff against every zone
602 zones.forEach(distZone => {
603 geom = debugGeo('diff', geom, getDistZoneGeom(distZone))
604 })
605
606 return {
607 geom: postProcessZone(geom, true),
608 tzid: zone.tzid
609 }
610 })
611
612 callback()
613}
614
Evan Siroky7891a6e2016-11-05 11:50:50 -0700615var combineAndWriteZones = function (callback) {
Evan Siroky8b47abe2016-10-02 12:28:52 -0700616 var stream = fs.createWriteStream('./dist/combined.json')
evansiroky26325842018-04-03 14:10:42 -0700617 var streamWithOceans = fs.createWriteStream('./dist/combined-with-oceans.json')
Evan Siroky8b47abe2016-10-02 12:28:52 -0700618 var zones = Object.keys(zoneCfg)
619
620 stream.write('{"type":"FeatureCollection","features":[')
evansiroky26325842018-04-03 14:10:42 -0700621 streamWithOceans.write('{"type":"FeatureCollection","features":[')
Evan Siroky8b47abe2016-10-02 12:28:52 -0700622
623 for (var i = 0; i < zones.length; i++) {
Evan Siroky7891a6e2016-11-05 11:50:50 -0700624 if (i > 0) {
Evan Siroky8b47abe2016-10-02 12:28:52 -0700625 stream.write(',')
evansiroky26325842018-04-03 14:10:42 -0700626 streamWithOceans.write(',')
Evan Siroky8b47abe2016-10-02 12:28:52 -0700627 }
628 var feature = {
629 type: 'Feature',
630 properties: { tzid: zones[i] },
631 geometry: geomToGeoJson(getDistZoneGeom(zones[i]))
632 }
evansiroky26325842018-04-03 14:10:42 -0700633 const stringified = JSON.stringify(feature)
634 stream.write(stringified)
635 streamWithOceans.write(stringified)
Evan Siroky8b47abe2016-10-02 12:28:52 -0700636 }
evansiroky26325842018-04-03 14:10:42 -0700637 oceanZoneBoundaries.forEach(boundary => {
638 streamWithOceans.write(',')
639 var feature = {
640 type: 'Feature',
641 properties: { tzid: boundary.tzid },
642 geometry: boundary.geom
643 }
644 streamWithOceans.write(JSON.stringify(feature))
645 })
646 asynclib.parallel([
647 cb => {
648 stream.end(']}', cb)
649 },
650 cb => {
651 streamWithOceans.end(']}', cb)
652 }
653 ], callback)
Evan Siroky8b47abe2016-10-02 12:28:52 -0700654}
655
evansiroky5348a6f2019-01-05 15:39:28 -0800656const autoScript = {
Evan Siroky7891a6e2016-11-05 11:50:50 -0700657 makeDownloadsDir: function (cb) {
evansiroky5348a6f2019-01-05 15:39:28 -0800658 overallProgress.beginTask('Creating downloads dir')
evansiroky4be1c7a2016-06-16 18:23:34 -0700659 safeMkdir('./downloads', cb)
660 },
Evan Siroky7891a6e2016-11-05 11:50:50 -0700661 makeDistDir: function (cb) {
evansiroky5348a6f2019-01-05 15:39:28 -0800662 overallProgress.beginTask('Creating dist dir')
evansiroky4be1c7a2016-06-16 18:23:34 -0700663 safeMkdir('./dist', cb)
evansirokyd401c892016-06-16 00:05:14 -0700664 },
Evan Siroky7891a6e2016-11-05 11:50:50 -0700665 getOsmBoundaries: ['makeDownloadsDir', function (results, cb) {
evansiroky5348a6f2019-01-05 15:39:28 -0800666 overallProgress.beginTask('Downloading osm boundaries')
Evan Siroky7891a6e2016-11-05 11:50:50 -0700667 asynclib.eachSeries(Object.keys(osmBoundarySources), downloadOsmBoundary, cb)
evansiroky63d35e12016-06-16 10:08:15 -0700668 }],
Evan Siroky7891a6e2016-11-05 11:50:50 -0700669 createZones: ['makeDistDir', 'getOsmBoundaries', function (results, cb) {
evansiroky5348a6f2019-01-05 15:39:28 -0800670 overallProgress.beginTask('Creating timezone boundaries')
Evan Siroky7891a6e2016-11-05 11:50:50 -0700671 asynclib.each(Object.keys(zoneCfg), makeTimezoneBoundary, cb)
evansiroky50216c62016-06-16 17:41:47 -0700672 }],
Evan Siroky7891a6e2016-11-05 11:50:50 -0700673 validateZones: ['createZones', function (results, cb) {
evansiroky5348a6f2019-01-05 15:39:28 -0800674 overallProgress.beginTask('Validating timezone boundaries')
Evan Siroky4fc596c2016-09-25 19:52:30 -0700675 loadDistZonesIntoMemory()
Evan Siroky081648a2017-07-04 09:53:36 -0700676 if (process.argv.indexOf('no-validation') > -1) {
677 console.warn('WARNING: Skipping validation!')
678 cb()
679 } else {
680 cb(validateTimezoneBoundaries())
681 }
Evan Siroky4fc596c2016-09-25 19:52:30 -0700682 }],
evansiroky26325842018-04-03 14:10:42 -0700683 addOceans: ['validateZones', function (results, cb) {
evansiroky5348a6f2019-01-05 15:39:28 -0800684 overallProgress.beginTask('Adding oceans')
evansiroky26325842018-04-03 14:10:42 -0700685 addOceans(cb)
686 }],
687 mergeZones: ['addOceans', function (results, cb) {
evansiroky5348a6f2019-01-05 15:39:28 -0800688 overallProgress.beginTask('Merging zones')
Evan Siroky8b47abe2016-10-02 12:28:52 -0700689 combineAndWriteZones(cb)
690 }],
691 zipGeoJson: ['mergeZones', function (results, cb) {
evansiroky5348a6f2019-01-05 15:39:28 -0800692 overallProgress.beginTask('Zipping geojson')
Evan Siroky8b47abe2016-10-02 12:28:52 -0700693 exec('zip dist/timezones.geojson.zip dist/combined.json', cb)
694 }],
evansiroky26325842018-04-03 14:10:42 -0700695 zipGeoJsonWithOceans: ['mergeZones', function (results, cb) {
evansiroky5348a6f2019-01-05 15:39:28 -0800696 overallProgress.beginTask('Zipping geojson with oceans')
evansiroky26325842018-04-03 14:10:42 -0700697 exec('zip dist/timezones-with-oceans.geojson.zip dist/combined-with-oceans.json', cb)
698 }],
Evan Siroky8b47abe2016-10-02 12:28:52 -0700699 makeShapefile: ['mergeZones', function (results, cb) {
evansiroky5348a6f2019-01-05 15:39:28 -0800700 overallProgress.beginTask('Converting from geojson to shapefile')
evansiroky26325842018-04-03 14:10:42 -0700701 rimraf.sync('dist/combined-shapefile.*')
702 exec(
evansirokye3360f72018-11-16 09:24:26 -0800703 'ogr2ogr -f "ESRI Shapefile" dist/combined-shapefile.shp dist/combined.json',
evansiroky26325842018-04-03 14:10:42 -0700704 function (err, stdout, stderr) {
705 if (err) { return cb(err) }
706 exec('zip dist/timezones.shapefile.zip dist/combined-shapefile.*', cb)
707 }
708 )
709 }],
710 makeShapefileWithOceans: ['mergeZones', function (results, cb) {
evansiroky5348a6f2019-01-05 15:39:28 -0800711 overallProgress.beginTask('Converting from geojson with oceans to shapefile')
evansiroky26325842018-04-03 14:10:42 -0700712 rimraf.sync('dist/combined-shapefile-with-oceans.*')
713 exec(
evansirokye3360f72018-11-16 09:24:26 -0800714 'ogr2ogr -f "ESRI Shapefile" dist/combined-shapefile-with-oceans.shp dist/combined-with-oceans.json',
evansiroky26325842018-04-03 14:10:42 -0700715 function (err, stdout, stderr) {
716 if (err) { return cb(err) }
717 exec('zip dist/timezones-with-oceans.shapefile.zip dist/combined-shapefile-with-oceans.*', cb)
718 }
719 )
evansiroky9fd50512019-07-07 12:06:28 -0700720 }],
721 makeListOfTimeZoneNames: function (cb) {
722 overallProgress.beginTask('Writing timezone names to file')
723 let zoneNames = Object.keys(zoneCfg)
724 oceanZones.forEach(oceanZone => {
725 zoneNames.push(oceanZone.tzid)
726 })
727 if (filteredZones.length > 0) {
728 zoneNames = zoneNames.filter(zoneName => filteredZones.indexOf(zoneName) > -1)
729 }
730 fs.writeFile(
731 'dist/timezone-names.json',
732 JSON.stringify(zoneNames),
733 cb
734 )
735 }
evansiroky5348a6f2019-01-05 15:39:28 -0800736}
737
738const overallProgress = new ProgressStats('Overall', Object.keys(autoScript).length)
739
740asynclib.auto(autoScript, function (err, results) {
evansirokyd401c892016-06-16 00:05:14 -0700741 console.log('done')
Evan Siroky7891a6e2016-11-05 11:50:50 -0700742 if (err) {
evansirokyd401c892016-06-16 00:05:14 -0700743 console.log('error!', err)
evansirokyd401c892016-06-16 00:05:14 -0700744 }
Evan Siroky4fc596c2016-09-25 19:52:30 -0700745})