Skip to content

Commit

Permalink
feat: Rework the PBF Download logic
Browse files Browse the repository at this point in the history
  • Loading branch information
@MichaelsJP
MichaelsJP committed Aug 28, 2024
1 parent 6a68f5b commit 09d8973
Show file tree
Hide file tree
Showing 2 changed files with 99 additions and 50 deletions.
11 changes: 7 additions & 4 deletions docker-compose.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,16 +17,17 @@ services:
# Advanced option! If you different ids to 0:0 and 1000:1000, you have to rebuild the container with the build args UID,GID.
# The user command is useful if you want easier bind mount access or better security.
#user: "1000:1000" # Run "mkdir -p ors-docker/config ors-docker/elevation_cache ors-docker/files ors-docker/graphs ors-docker/logs && sudo chown -R 1000:1000 ors" before starting the container!
volumes: # Mount relative directories. ONLY for local container runtime. To switch to docker managed volumes see 'Docker Volumes configuration' section below.
volumes: # Mount relative directories. ONLY for local container runtime. To switch to docker managed volumes see 'Docker Volumes configuration' section below.
- ./ors-docker:/home/ors # Mount the ORS application directory (for logs, graphs, elevation_cache, etc.) into its own directory
#- ./graphs:/home/ors/graphs # Mount graphs directory individually
#- ./elevation_cache:/home/ors/elevation_cache # Mount elevation cache directory individually
#- ./config:/home/ors/config # Mount configuration directory individually
#- ./logs:/home/ors/logs # Mount logs directory individually
#- ./files:/home/ors/files # Mount files directory individually
environment:
REBUILD_GRAPHS: False # Set to True to rebuild graphs on container start.
CONTAINER_LOG_LEVEL: INFO # Log level for the container. Possible values: DEBUG, INFO, WARNING, ERROR, CRITICAL
REBUILD_GRAPHS: False # Set to True to rebuild graphs on container start in any case.
REBUILD_ON_PBF_CHANGE: False # Set to True to rebuild graphs if the PBF file has changed. Overwrites REBUILD_GRAPHS for PBF changes.
CONTAINER_LOG_LEVEL: Info # Log level for the container. Possible values: DEBUG, INFO, WARNING, ERROR, CRITICAL
# If you don't want the default ors-config.yml you can specify a custom file name, that should match the file in
# your 'config' volume mount.
#ORS_CONFIG_LOCATION: /home/ors/config/my-ors-config.yml # Location of your ORS configuration file in the docker container
Expand All @@ -48,7 +49,9 @@ services:
# See the ors-config.env file for more options.
# To have a configuration file-less container, uncomment at least the following properties.
# The values are examples and provide the default configuration.
ors.engine.source_file: https://download.geofabrik.de/europe/andorra-latest.osm.pbf
# It is possible to download a remote PBF file or use a local one.
#ors.engine.source_file: https://download.geofabrik.de/europe/andorra-latest.osm.pbf
ors.engine.source_file: /home/ors/files/example-heidelberg.osm.gz
#ors.engine.graphs_root_path: /home/ors/graphs
#ors.engine.elevation.cache_path: /home/ors/elevation_cache
#ors.engine.profiles.car.enabled: true
Expand Down
138 changes: 92 additions & 46 deletions docker-entrypoint.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ function set_log_level() {
success "CONTAINER_LOG_LEVEL: ${CONTAINER_LOG_LEVEL}. Set CONTAINER_LOG_LEVEL=DEBUG for more details."
}

update_file() {
function update_file() {
local target_file_path="$1"
local original_file_path="$2"
# Default to false
Expand All @@ -86,7 +86,7 @@ update_file() {
fi
}

extract_config_info() {
function extract_config_info() {
local config_location="$1"
local config_variable="$2"
local config_value=""
Expand All @@ -103,53 +103,60 @@ extract_config_info() {
echo "${config_value}"
}

calculate_hash() {
function calculate_hash() {
local file_path="$1"
info "Calculating sha256sum of ${file_path}"
sha256sum "${file_path}" | awk '{print $1}'
}

handle_pbf_file() {
local pbf_file="$1"
local graphs_folder="$2"
local pbf_file_name
pbf_file_name=$(basename "${pbf_file}")

# Check if the PBF file is a URL or a local file
if [[ "${pbf_file}" =~ ^https?:// ]]; then
local local_pbf_file="${ORS_HOME}/files/${pbf_file_name}"
# If the local file doesn't exist, download it
if [ ! -f "${local_pbf_file}" ]; then
info "Downloading PBF file from ${pbf_file} to ${local_pbf_file}"
wget -O "${local_pbf_file}" "${pbf_file}" || critical "Failed to download PBF file from ${pbf_file}. Exiting."
function pbf_is_remote() {
local pbf_source_uri="$1"
if [[ "${pbf_source_uri}" =~ ^https?:// ]]; then
return 0
else
return 1
fi
}

function download_pbf_file() {
local pbf_source_uri="$1"
local pbf_target_file="$2"

if pbf_is_remote "${pbf_source_uri}"; then
if [ ! -f "${pbf_target_file}" ]; then
wget -O "${pbf_target_file}" "${pbf_source_uri}" || critical "Failed to download PBF file from ${pbf_source_uri}. Exiting."
success "Downloaded PBF file from ${pbf_source_uri} to ${pbf_target_file}"
else
success "PBF file is an URL and already exists locally. Skipping download."
info "For a fresh download remove it with: docker exec -it ors-container-name rm ${pbf_target_file}"
fi
pbf_file="${local_pbf_file}"
else
debug "PBF file is a local file."
fi
}

# Calculate the current hash of the PBF file and store it in a file called pbf_file_name_hash
local current_pbf_hash
current_pbf_hash=$(calculate_hash "${pbf_file}")
local stored_pbf_hash_file="${graphs_folder}/${pbf_file_name}_hash"

# Check if the hash file exists and read the stored hash
local stored_pbf_hash=""
function validate_rebuild_needed() {
# Validate the PBF file hash and return 0 if the hash has changed.
local pbf_target_file="$1"
local pbf_target_hash_path="$2"

local external_pbf_file_hash
external_pbf_file_hash=$(calculate_hash "${pbf_target_file}")

local stored_pbf_hash_file="${pbf_target_hash_path}"
local stored_pbf_hash="NONE"
if [ -f "${stored_pbf_hash_file}" ]; then
stored_pbf_hash=$(cat "${stored_pbf_hash_file}")
fi

# Compare the hashes
if [ "${current_pbf_hash}" != "${stored_pbf_hash}" ]; then
info "PBF file hash has changed."
if [ "${REBUILD_ON_PBF_CHANGE}" = "true" ]; then
info "REBUILD_ON_PBF_CHANGE is true. Setting REBUILD_GRAPHS to true."
ors_rebuild_graphs="true"
else
warning "REBUILD_ON_PBF_CHANGE is false. Not rebuilding graphs based on PBF file changes. Manual rebuild is still possible."
fi
debug "External PBF file Hash: ${external_pbf_file_hash}"
debug "Graph PBF file Hash: ${stored_pbf_hash}"

if [ "${external_pbf_file_hash}" != "${stored_pbf_hash}" ]; then
return 0
else
success "PBF file hash has not changed."
return 1
fi
echo "${pbf_file}"
}

echo "#################"
Expand Down Expand Up @@ -304,13 +311,24 @@ else
fi

if [ -n "${ors_engine_source_file}" ]; then
debug "OSM source file set to ${ors_engine_source_file}"
success "OSM source file set to ${ors_engine_source_file}"
# Check if it is the example file in root or the home folder
if [[ "${ors_engine_source_file}" = "${ORS_HOME}/files/example-heidelberg.osm.gz" ]]; then
info "Default to example osm source file: \"${ors_engine_source_file}\""
fi
fi

# Check if osm file is remote and if so, check its reachable
if pbf_is_remote "${ors_engine_source_file}"; then
debug "Testing if OSM remote source file is reachable."
if ! wget --quiet --spider "${ors_engine_source_file}"; then
warning "OSM source file is a remote file but not reachable."
warning "Check the URL and your network connection. Continuing with the broken link."
else
success "OSM source file is a remote file and reachable."
fi
fi

info "Any ENV variables will have precedence over configuration variables from config files."
success "All checks passed. For details set CONTAINER_LOG_LEVEL=DEBUG."

Expand All @@ -325,7 +343,30 @@ update_file "${ORS_HOME}/files/example-heidelberg.osm.gz" "/heidelberg.osm.gz"


# Check if we need to rebuild graphs based on PBF file changes
ors_engine_source_file=$(handle_pbf_file "${ors_engine_source_file}" "${ors_engine_graphs_root_path}")
if pbf_is_remote "${ors_engine_source_file}"; then
debug "ors_engine_source_file is a remote file: ${ors_engine_source_file}"
ors_engine_target_file="${ORS_HOME}/files/$(basename "${ors_engine_source_file}")"
download_pbf_file "${ors_engine_source_file}" "${ors_engine_target_file}"
ors_engine_source_file="${ors_engine_target_file}"
debug "ors_engine_source_file is now set to: ${ors_engine_source_file}"
fi

if validate_rebuild_needed "${ors_engine_source_file}" "${ors_engine_graphs_root_path}/.hash"; then
if [ "${REBUILD_ON_PBF_CHANGE}" = "true" ]; then
ors_rebuild_graphs="true"
success "PBF and Graph hashes not equal and REBUILD_ON_PBF_CHANGE=true. Rebuilding graphs."
else
info "PBF and Graph hashes are not equal but REBUILD_ON_PBF_CHANGE=false. Skipping graph rebuild."
info "Set environment variable REBUILD_ON_PBF_CHANGE=true to rebuild graphs based on PBF file changes."
fi
else
if [ "${ors_rebuild_graphs}" = "true" ]; then
success "PBF and graph hashes are equal but REBUILD_GRAPHS=true. Rebuilding graphs."
else
success "PBF and graph hashes are equal and no rebuild configured. Skipping graph rebuild."
fi
fi


# Remove existing graphs if BUILD_GRAPHS is set to true
if [ "${ors_rebuild_graphs}" = "true" ]; then
Expand All @@ -335,6 +376,8 @@ if [ "${ors_rebuild_graphs}" = "true" ]; then
elif [ -d "${ors_engine_graphs_root_path}" ]; then
# Check the ors.engine.graphs_root_path folder exists
rm -rf "${ors_engine_graphs_root_path:?}"/* || warning "Could not remove ${ors_engine_graphs_root_path}"
# Remove hash file
rm -f "${ors_engine_graphs_root_path}/.hash" || warning "Could not remove ${ors_engine_graphs_root_path}/.hash"
success "Removed graphs at ${ors_engine_graphs_root_path}/*."
else
debug "${ors_engine_graphs_root_path} does not exist (yet). Skipping cleanup."
Expand All @@ -343,6 +386,14 @@ if [ "${ors_rebuild_graphs}" = "true" ]; then
mkdir -p "${ors_engine_graphs_root_path}" || warning "Could not populate graph folder at ${ors_engine_graphs_root_path}"
fi

# Write the hash file for new graphs only.
graph_folder_content=$(find "${ors_engine_graphs_root_path}" -mindepth 1 -maxdepth 1 | wc -l)
if [ "${graph_folder_content}" -eq 0 ]; then
# Don't calculate the hash if an existing graph is present.
success "Calculating hash of ${ors_engine_source_file} and storing it in ${ors_engine_graphs_root_path}/.hash"
calculate_hash "${ors_engine_source_file}" >"${ors_engine_graphs_root_path}/.hash"
fi

success "Container file system preparation complete. For details set CONTAINER_LOG_LEVEL=DEBUG."

echo "#######################################"
Expand Down Expand Up @@ -406,20 +457,15 @@ if [ "${print_migration_info}" = "true" ]; then
info ">>> End of migration information <<<"
fi


# Hash the current pbf file and store it in the graphs folder
if [ -f "${ors_engine_source_file}" ]; then
info "Calculating hash of ${ors_engine_source_file} and storing it in ${ors_engine_graphs_root_path}/$(basename "${ors_engine_source_file}")_hash"
calculate_hash "${ors_engine_source_file}" >"${ors_engine_graphs_root_path}/$(basename "${ors_engine_source_file}")_hash"
fi

echo "#####################"
echo "# ORS startup phase #"
echo "#####################"
# Start the jar with the given arguments and add the ORS_HOME env var
success "🙭 Ready to start the ORS application 🙭"
debug "Startup command: java ${JAVA_OPTS} ${CATALINA_OPTS} -jar ${jar_file}"
debug "Startup command: java ${JAVA_OPTS} ${CATALINA_OPTS} -Dors.engine.source_file=${ors_engine_source_file} -jar ${jar_file}"
# Export ORS_CONFIG_LOCATION to the environment of child processes
export ORS_CONFIG_LOCATION=${ors_config_location}
# shellcheck disable=SC2086 # we need word splitting here
exec java ${JAVA_OPTS} ${CATALINA_OPTS} -jar "${jar_file}" "$@"
exec java ${JAVA_OPTS} ${CATALINA_OPTS} \
"-Dors.engine.source_file=${ors_engine_source_file}" \
-jar "${jar_file}" "$@"

0 comments on commit 09d8973

Please sign in to comment.