Add option --run-in-foreground to qlever start command (#106)

With the new option `--run-in-foreground`, the `qlever start` command now remains in the foreground as long as the process that runs the QLever server is alive, following the log. Addresses #98 

Co-authored-by: Julian Mundhahs <[email protected]>

pyproject.toml

@@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta"
 [project]
 name = "qlever"
 description = "Script for using the QLever SPARQL engine."
-version = "0.5.14"
+version = "0.5.17"
 authors = [
     { name = "Hannah Bast", email = "[email protected]" }
 ]

src/qlever/commands/start.py

@@ -14,7 +14,7 @@
 
 
 # Construct the command line based on the config file.
-def construct_command_line(args) -> str:
+def construct_command(args) -> str:
     start_cmd = (
         f"{args.server_binary}"
         f" -i {args.name}"
@@ -53,7 +53,7 @@ def kill_existing_server(args) -> bool:
 
 
 # Run the command in a container
-def run_command_in_container(args, start_cmd) -> str:
+def wrap_command_in_container(args, start_cmd) -> str:
     if not args.server_container:
         args.server_container = f"qlever.server.{args.name}"
     start_cmd = Containerize().containerize_command(
@@ -85,8 +85,8 @@ def check_binary(binary) -> bool:
         return False
 
 
-# Set the access token if specified. Try to set the index description
-def setting_index_description(access_arg, port, desc) -> bool:
+# Set the index description.
+def set_index_description(access_arg, port, desc) -> bool:
     curl_cmd = (
         f"curl -Gs http://localhost:{port}/api"
         f' --data-urlencode "index-description={desc}"'
@@ -101,8 +101,8 @@ def setting_index_description(access_arg, port, desc) -> bool:
     return True
 
 
-# Set the access token if specified. Try to set the text description
-def setting_text_description(access_arg, port, text_desc) -> bool:
+# Set the text description.
+def set_text_description(access_arg, port, text_desc) -> bool:
     curl_cmd = (
         f"curl -Gs http://localhost:{port}/api"
         f' --data-urlencode "text-description={text_desc}"'
@@ -177,6 +177,13 @@ def additional_arguments(self, subparser) -> None:
             default=False,
             help="Do not execute the warmup command",
         )
+        subparser.add_argument(
+            "--run-in-foreground",
+            action="store_true",
+            default=False,
+            help="Run the server in the foreground "
+            "(default: run in the background with `nohup`)",
+        )
 
     def execute(self, args) -> bool:
         # Kill existing server with the same name if so desired.
@@ -197,12 +204,15 @@ def execute(self, args) -> bool:
                 return False
 
         # Construct the command line based on the config file.
-        start_cmd = construct_command_line(args)
+        start_cmd = construct_command(args)
 
         # Run the command in a container (if so desired). Otherwise run with
-        # `nohup` so that it keeps running after the shell is closed.
+        # `nohup` so that it keeps running after the shell is closed. With
+        # `--run-in-foreground`, run the server in the foreground.
         if args.system in Containerize.supported_systems():
-            start_cmd = run_command_in_container(args, start_cmd)
+            start_cmd = wrap_command_in_container(args, start_cmd)
+        elif args.run_in_foreground:
+            start_cmd = f"{start_cmd}"
         else:
             start_cmd = f"nohup {start_cmd} &"
 
@@ -213,7 +223,8 @@ def execute(self, args) -> bool:
 
         # When running natively, check if the binary exists and works.
         if args.system == "native":
-            if not check_binary(args.server_binary):
+            ret = check_binary(args.server_binary)
+            if not ret:
                 return False
 
         # Check if a QLever server is already running on this port.
@@ -254,38 +265,51 @@ def execute(self, args) -> bool:
 
         # Execute the command line.
         try:
-            run_command(start_cmd)
+            process = run_command(
+                start_cmd,
+                use_popen=args.run_in_foreground,
+            )
         except Exception as e:
             log.error(f"Starting the QLever server failed ({e})")
             return False
 
         # Tail the server log until the server is ready (note that the `exec`
         # is important to make sure that the tail process is killed and not
         # just the bash process).
-        log.info(
-            f"Follow {args.name}.server-log.txt until the server is ready"
-            f" (Ctrl-C stops following the log, but not the server)"
-        )
+        if args.run_in_foreground:
+            log.info(
+                f"Follow {args.name}.server-log.txt as long as the server is"
+                f" running (Ctrl-C stops the server)"
+            )
+        else:
+            log.info(
+                f"Follow {args.name}.server-log.txt until the server is ready"
+                f" (Ctrl-C stops following the log, but NOT the server)"
+            )
         log.info("")
         tail_cmd = f"exec tail -f {args.name}.server-log.txt"
         tail_proc = subprocess.Popen(tail_cmd, shell=True)
         while not is_qlever_server_alive(endpoint_url):
             time.sleep(1)
 
-        # Set the access token if specified.
+        # Set the description for the index and text.
         access_arg = f'--data-urlencode "access-token={args.access_token}"'
-        if args.description and not setting_index_description(
-            access_arg, args.port, args.description
-        ):
-            return False
-
-        if args.text_description and not setting_text_description(
-            access_arg, args.port, args.text_description
-        ):
-            return False
+        if args.description:
+            ret = set_index_description(
+                access_arg, args.port, args.description
+            )
+            if not ret:
+                return False
+        if args.text_description:
+            ret = set_text_description(
+                access_arg, args.port, args.text_description
+            )
+            if not ret:
+                return False
 
         # Kill the tail process. NOTE: `tail_proc.kill()` does not work.
-        tail_proc.terminate()
+        if not args.run_in_foreground:
+            tail_proc.terminate()
 
         # Execute the warmup command.
         if args.warmup_cmd and not args.no_warmup:
@@ -295,8 +319,18 @@ def execute(self, args) -> bool:
                 return False
 
         # Show cache stats.
-        log.info("")
-        args.detailed = False
-        args.server_url = None
-        CacheStatsCommand().execute(args)
+        if not args.run_in_foreground:
+            log.info("")
+            args.detailed = False
+            args.server_url = None
+            CacheStatsCommand().execute(args)
+
+        # With `--run-in-foreground`, wait until the server is stopped.
+        if args.run_in_foreground:
+            try:
+                process.wait()
+            except KeyboardInterrupt:
+                process.terminate()
+            tail_proc.terminate()
+
         return True

src/qlever/commands/stop.py

@@ -85,6 +85,7 @@ def execute(self, args) -> bool:
         # Check if there is a process running on the server port using psutil.
         # NOTE: On MacOS, some of the proc's returned by psutil.process_iter()
         # no longer exist when we try to access them, so we just skip them.
+        stop_process_results = []
         for proc in psutil.process_iter():
             try:
                 pinfo = proc.as_dict(
@@ -98,7 +99,9 @@ def execute(self, args) -> bool:
                 log.info(f"Found process {pinfo['pid']} from user "
                          f"{pinfo['username']} with command line: {cmdline}")
                 log.info("")
-                return stop_process(proc, pinfo)
+                stop_process_results.append(stop_process(proc, pinfo))
+        if len(stop_process_results) > 0:
+            return all(stop_process_results)
 
         # If no matching process found, show a message and the output of the
         # status command.

src/qlever/util.py

@@ -30,8 +30,11 @@ def get_total_file_size(patterns: list[str]) -> int:
 
 
 def run_command(
-    cmd: str, return_output: bool = False, show_output: bool = False
-) -> Optional[str]:
+    cmd: str,
+    return_output: bool = False,
+    show_output: bool = False,
+    use_popen: bool = False,
+) -> Optional[str | subprocess.Popen]:
     """
     Run the given command and throw an exception if the exit code is non-zero.
     If `return_output` is `True`, return what the command wrote to `stdout`.
@@ -41,24 +44,29 @@ def run_command(
 
     TODO: Find the executable for `bash` in `__init__.py`.
     """
+
     subprocess_args = {
         "executable": shutil.which("bash"),
         "shell": True,
         "text": True,
         "stdout": None if show_output else subprocess.PIPE,
         "stderr": subprocess.PIPE,
     }
+
+    # With `Popen`, the command runs in the current shell and a process object
+    # is returned (which can be used, e.g., to kill the process).
+    if use_popen:
+        if return_output:
+            raise Exception("Cannot return output if `use_popen` is `True`")
+        return subprocess.Popen(f"set -o pipefail; {cmd}", **subprocess_args)
+
+    # With `run`, the command runs in a subshell and the output is captured.
     result = subprocess.run(f"set -o pipefail; {cmd}", **subprocess_args)
+
     # If the exit code is non-zero, throw an exception. If something was
     # written to `stderr`, use that as the exception message. Otherwise, use a
     # generic message (which is also what `subprocess.run` does with
     # `check=True`).
-    # log.debug(f"Command `{cmd}` returned the following result")
-    # log.debug("")
-    # log.debug(f"exit code: {result.returncode}")
-    # log.debug(f"stdout: {result.stdout}")
-    # log.debug(f"stderr: {result.stderr}")
-    # log.debug("")
     if result.returncode != 0:
         if len(result.stderr) > 0:
             raise Exception(result.stderr.replace("\n", " ").strip())