Elevation tile on demand fetching (valhalla#3391)

* Ondemand elevation fetching added

* On demand elevation tile fetching

* Changes after review v1

* Changes after review v2. Some more tests added

* Changes after review v3

* Parallel call removed from sample.cc. get_files removed from sample.cc. Tests added

* Public methods for setting endpoints removed

* New tests addded

* Changes after review v4

* Additional methods

* Build method changed

* filesystem.h is changed

* New tests added

* Method make_single_url moved from class tile_getter_t and become class helper funciton.

* flag for storing archived elevation tiles removed

* Unnecessary comments removed.

* Local host returned to any in tile_server

* * get_hgt_file_name become util function

* get_no_data_value become util function

* cache_ become smart pointer

* make_single_point_url moved from tilegetter to curl_tilegetter

* Sample code thread synchronisation changed

* Synchronizaton changed in source method

* Test with hash check added

* Fixed: Synchronization issue

Added: addtional tests to elevation_builder

* Added: openssl lib dependencies

* Change: file comparison algorithm in elevation_builder.cc

* Comments removed

* Change: boost -> cxxopt in valhalla_add_elevation.cc

* Added: additional checks

* Code corrected after review v5

* Code correction after review v6

* Lock added to get method in sample

* Update CHANGELOG.md

* Update scripts/valhalla_build_config

Co-authored-by: Pavel Merkis <[email protected]>

* Update src/mjolnir/elevationbuilder.cc

Co-authored-by: Pavel Merkis <[email protected]>

* Update src/mjolnir/elevationbuilder.cc

Co-authored-by: Pavel Merkis <[email protected]>

* Update src/skadi/sample.cc

Co-authored-by: Pavel Merkis <[email protected]>

* Code correction after review v7

* Returned to const ref

Co-authored-by: Pavel Merkis <[email protected]>

Author: Neyromancer

.gitignore

@@ -30,6 +30,7 @@ valhalla_tyr_worker
 valhalla_associate_segments
 valhalla_fetch_transit
 valhalla_pack_elevation
+valhalla_add_elevation
 genfiles/
 locales/*.UTF-8
 !locales/merge-en.sh

CHANGELOG.md

@@ -62,6 +62,7 @@
    * ADDED: New options to control what fields of the pbf are returned when pbf format responses are requested [#3207](https://github.com/valhalla/valhalla/pull/3507)
    * CHANGED: Rename tripcommon to common [#3516](https://github.com/valhalla/valhalla/pull/3516)
    * ADDED: indoor routing - data model, data processing. [#3509](https://github.com/valhalla/valhalla/pull/3509)
+   * ADDED: On-demand elevation tile fetching [#3391](https://github.com/valhalla/valhalla/pull/3391)
 
 ## Release Date: 2021-10-07 Valhalla 3.1.4
 * **Removed**

CMakeLists.txt

@@ -297,7 +297,7 @@ set(valhalla_programs valhalla_run_map_match valhalla_benchmark_loki valhalla_be
 set(valhalla_data_tools valhalla_build_statistics valhalla_ways_to_edges valhalla_validate_transit
   valhalla_benchmark_admins valhalla_build_connectivity	valhalla_build_tiles valhalla_build_admins
   valhalla_convert_transit valhalla_fetch_transit valhalla_query_transit valhalla_add_predicted_traffic
-  valhalla_assign_speeds)
+  valhalla_assign_speeds valhalla_add_elevation)
 
 ## Valhalla services
 set(valhalla_services valhalla_loki_worker valhalla_odin_worker valhalla_thor_worker)

scripts/valhalla_build_config

@@ -62,7 +62,8 @@ config = {
     }
   },
   'additional_data': {
-    'elevation': '/data/valhalla/elevation/'
+    'elevation': '/data/valhalla/elevation/',
+    'elevation_url': Optional(str)
   },
   'loki': {
     'actions':['locate','route','height','sources_to_targets','optimized_route','isochrone','trace_route','trace_attributes','transit_available', 'expansion', 'centroid', 'status'],
@@ -331,7 +332,8 @@ help_text = {
     }
   },
   'additional_data': {
-    'elevation': 'Location of srtmgl1 elevation tiles for using in valhalla_build_tiles'
+    'elevation': 'Location of elevation tiles',
+    'elevation_url': 'Http location to read elevations from. this address is used if elevation tiles were not found in the elevation directory. Ex.: http://<your_valhalla_tile_server_host>:<your_valhalla_tile_server_port>/some/Optional/path/{tilePath}?some=Optional&query=params. Valhalla will look for the {tilePath} portion of the url and fill this out with an elevation path when it makes a request for that particular elevation'
   },
   'loki': {
     'actions': 'Comma separated list of allowable actions for the service, one or more of: locate, route, height, optimized_route, isochrone, trace_route, trace_attributes, transit_available, expansion, centroid, status',

src/baldr/graphreader.cc

@@ -549,8 +549,6 @@ class TarballGraphMemory final : public GraphMemory {
 // Get a pointer to a graph tile object given a GraphId. Return nullptr
 // if the tile is not found/empty
 graph_tile_ptr GraphReader::GetGraphTile(const GraphId& graphid) {
-  // TODO: clear the cache automatically once we become overcommitted by a certain amount
-
   // Return nullptr if not a valid tile
   if (!graphid.Is_Valid()) {
     return nullptr;

src/baldr/graphtile.cc

@@ -1,8 +1,9 @@
 #include "baldr/graphtile.h"
+
 #include "baldr/compression_utils.h"
+#include "baldr/curl_tilegetter.h"
 #include "baldr/datetime.h"
 #include "baldr/sign.h"
-#include "baldr/tilegetter.h"
 #include "baldr/tilehierarchy.h"
 #include "filesystem.h"
 #include "midgard/aabb2.h"
@@ -30,14 +31,6 @@ namespace {
 const AABB2<PointLL> world_box(PointLL(-180, -90), PointLL(180, 90));
 constexpr float COMPRESSION_HINT = 3.5f;
 
-std::string MakeSingleTileUrl(const std::string& tile_url, const valhalla::baldr::GraphId& graphid) {
-  auto id_pos = tile_url.find(valhalla::baldr::GraphTile::kTilePathPattern);
-  return tile_url.substr(0, id_pos) +
-         valhalla::baldr::GraphTile::FileSuffix(graphid.Tile_Base(),
-                                                valhalla::baldr::SUFFIX_NON_COMPRESSED, false) +
-         tile_url.substr(id_pos + std::strlen(valhalla::baldr::GraphTile::kTilePathPattern));
-}
-
 // the point of this function is to avoid race conditions for writing a tile between threads
 // so the easiest thing to do is just use the thread id to differentiate
 std::string GenerateTmpSuffix() {
@@ -104,9 +97,18 @@ graph_tile_ptr GraphTile::DecompressTile(const GraphId& graphid,
 graph_tile_ptr GraphTile::Create(const std::string& tile_dir,
                                  const GraphId& graphid,
                                  std::unique_ptr<const GraphMemory>&& traffic_memory) {
+  if (!graphid.Is_Valid()) {
+    LOG_ERROR("Failed to build GraphTile. Error: GraphId is invalid");
+    return nullptr;
+  }
 
-  // Don't bother with invalid ids
-  if (!graphid.Is_Valid() || graphid.level() > TileHierarchy::get_max_level() || tile_dir.empty()) {
+  if (graphid.level() > TileHierarchy::get_max_level()) {
+    LOG_ERROR("Failed to build GraphTile. Error: GraphId level exceeds tile hierarchy max level");
+    return nullptr;
+  }
+
+  if (tile_dir.empty()) {
+    LOG_ERROR("Failed to build GraphTile. Error: Tile dir is empty");
     return nullptr;
   }
 
@@ -205,28 +207,38 @@ void GraphTile::SaveTileToFile(const std::vector<char>& tile_data, const std::st
     filesystem::remove(tmp_location);
 }
 
+void store(const std::string& cache_location,
+           const GraphId& graphid,
+           const tile_getter_t* tile_getter,
+           const std::vector<char>& raw_data) {
+  if (!cache_location.empty()) {
+    auto suffix =
+        valhalla::baldr::GraphTile::FileSuffix(graphid.Tile_Base(),
+                                               (tile_getter->gzipped()
+                                                    ? valhalla::baldr::SUFFIX_COMPRESSED
+                                                    : valhalla::baldr::SUFFIX_NON_COMPRESSED));
+    auto disk_location = cache_location + filesystem::path::preferred_separator + suffix;
+    filesystem::save(disk_location, raw_data);
+  }
+}
+
 graph_tile_ptr GraphTile::CacheTileURL(const std::string& tile_url,
                                        const GraphId& graphid,
                                        tile_getter_t* tile_getter,
                                        const std::string& cache_location) {
   // Don't bother with invalid ids
-  if (!graphid.Is_Valid() || graphid.level() > TileHierarchy::get_max_level()) {
+  if (!graphid.Is_Valid() || graphid.level() > TileHierarchy::get_max_level() || !tile_getter) {
     return nullptr;
   }
 
-  auto uri = MakeSingleTileUrl(tile_url, graphid);
-  auto result = tile_getter->get(uri);
+  auto fname = valhalla::baldr::GraphTile::FileSuffix(graphid.Tile_Base(),
+                                                      valhalla::baldr::SUFFIX_NON_COMPRESSED, false);
+  auto result = tile_getter->get(baldr::make_single_point_url(tile_url, fname));
   if (result.status_ != tile_getter_t::status_code_t::SUCCESS) {
     return nullptr;
   }
   // try to cache it on disk so we dont have to keep fetching it from url
-  if (!cache_location.empty()) {
-    auto suffix = FileSuffix(graphid.Tile_Base(),
-                             (tile_getter->gzipped() ? valhalla::baldr::SUFFIX_COMPRESSED
-                                                     : valhalla::baldr::SUFFIX_NON_COMPRESSED));
-    auto disk_location = cache_location + filesystem::path::preferred_separator + suffix;
-    SaveTileToFile(result.bytes_, disk_location);
-  }
+  store(cache_location, graphid, tile_getter, result.bytes_);
 
   // turn the memory into a tile
   if (tile_getter->gzipped()) {
@@ -248,9 +260,10 @@ void GraphTile::Initialize(const GraphId& graphid) {
   char* const tile_ptr = memory_->data;
   const size_t tile_size = memory_->size;
 
-  if (tile_size < sizeof(GraphTileHeader))
+  if (tile_size < sizeof(GraphTileHeader)) {
     throw std::runtime_error("Invalid tile data size = " + std::to_string(tile_size) +
                              ". Tile file might me corrupted");
+  }
 
   char* ptr = tile_ptr;
   header_ = reinterpret_cast<GraphTileHeader*>(ptr);