Merge pull request #1141 from NLGithubWP/dev-postgresql

Update doc with singa repo and add system design graph

examples/model_selection/Trails/README.md

@@ -20,6 +20,9 @@
 
 # TRAILS: A Database Native Model Selection System
 
+![image-20230702035806963](documents/ai_db.001.jpeg)
+
+
 # Build & Run examples
 
 ## Singa + PostgreSQL
@@ -30,14 +33,16 @@ docker rm -f singa_trails
 # Create project folder.
 mkdir project && cd project
 # Download the Dockerile.
-wget -O Dockerfile  ??
+wget -O Dockerfile https://raw.githubusercontent.com/apache/singa/dev-postgresql/examples/model_selection/Trails/singa.psql.Dockerfile
 
 # Build Dockerile and run the docker.
 docker build -t singa_trails .
 docker run -d --name singa_trails singa_trails
 # Wait for 5 mins, monitor the logs until it shows "Done!", then exit the monitor
 docker logs -f singa_trails
-
+```
+Execute model selection query
+```bash
 # Connect to the pg server and use pg_extension database.
 docker exec -it singa_trails bash
 psql -h localhost -p 28814 -U postgres
@@ -63,16 +68,18 @@ docker rm -f trails_singa_polardb
 # Create project folder.
 mkdir project_polardb_singa && cd project_polardb_singa
 # Download the Dockerile.
-wget -O Dockerfile ???
+wget -O Dockerfile https://github.com/apache/singa/blob/dev-postgresql/examples/model_selection/Trails/singa.polarDB.Dockerfile
 
 # Build Dockerile and run the docker.
 docker build -t trails_singa_polardb .
-docker run -d --name trails_singa_polardb  trails_singa_polardb
+docker run -d --name trails_singa_polardb trails_singa_polardb
 # Monitor the logs until the setup step is done.
 docker logs -f trails_singa_polardb
 # Run a setup script
 docker exec trails_singa_polardb /bin/bash -c "/home/postgres/Trails/init_polardb.sh"
-
+```
+Execute model selection query
+```bash
 # Connect to the primary pg server and use pg_extension database.
 docker exec -it trails_singa_polardb bash
 psql -h localhost -p 5432 -U postgres 

examples/model_selection/Trails/documents/dev_guide.md

@@ -10,3 +10,80 @@ python3 ./internal/ml/model_selection/exps/0.train_one_model.py --log_name=train
 python3 internal/ml/model_selection/pg_interface.py
 ```
 
+# How to add features to there?
+
+1. fork the git repo https://github.com/apache/singa/tree/dev-postgresql
+
+2. run the docker image as in [README.md](https://github.com/apache/singa/blob/dev-postgresql/examples/model_selection/Trails/README.md)
+
+   ```bash
+   # Remove existing one if there is 
+   docker rm -f singa_trails
+   # Create project folder.
+   mkdir project && cd project
+   # Download the Dockerile.
+   wget -O Dockerfile https://raw.githubusercontent.com/apache/singa/dev-postgresql/examples/model_selection/Trails/singa.psql.Dockerfile
+
+   # Build Dockerile and run the docker.
+   docker build -t singa_trails .
+   docker run -d --name singa_trails singa_trails
+   # Wait for 5 mins, monitor the logs until it shows "Done!", then exit the monitor
+   docker logs -f singa_trails
+   ```
+
+3. after the docker image is running up, go to the code dir, add your own git repo's url as a remote
+
+   ```bash
+   docker exec -it singa_trails bash
+   cd /project/Trails
+   git fetch --all
+   git checkout -b <branch-name> <my-repo>/<branch-name>
+   # eg git checkout -b dev-postgresql nl2/dev-postgresql
+   ```
+
+4. then commit the change to your own git repo, and pull inside the docker image.
+
+5. then, compile and test as following 
+
+## For the psql
+
+```bash
+cd Trails/internal/pg_extension
+cargo pgrx run --release
+
+# tests
+psql -h localhost -p 28814 -U postgres
+\c pg_extension
+
+# Test coordinator
+SELECT coordinator('0.08244', '168.830156', '800', false, '/project/Trails/internal/ml/model_selection/config.ini');
+# Run an example, wait one min, it will run filtering + refinemnt + training the selected model.
+CALL model_selection_end2end('frappe_train', ARRAY['col1', 'col2', 'col3', 'col4','col5','col6','col7','col8','col9','col10', 'label'], '10', '/project/Trails/internal/ml/model_selection/config.ini');
+
+# In other terminal, monitor the running process
+docker exec -it trails_polardb bash
+tail -f /home/postgres/.pgrx/data-14/trails_log_folder/<log_file_name>
+```
+
+## For the polarDB
+
+```bash
+cd Trails/internal/pg_extension
+cargo clean
+cargo pgrx install --pg-config /home/postgres/tmp_basedir_polardb_pg_1100_bld/bin/pg_config
+
+# Connect to the primary pg server and use pg_extension database.
+docker exec -it trails_singa_polardb bash
+psql -h localhost -p 5432 -U postgres 
+\c pg_extension
+
+# Test coordinator
+SELECT coordinator('0.08244', '168.830156', '800', false, '/home/postgres/Trails/internal/ml/model_selection/config.ini');
+# Run an example, wait one min, it will run filtering + refinemnt + training the selected model.
+CALL model_selection_end2end('frappe_train', ARRAY['col1', 'col2', 'col3', 'col4','col5','col6','col7','col8','col9','col10', 'label'], '10', '/home/postgres/Trails/internal/ml/model_selection/config.ini');
+
+# In other terminal, monitor the running process
+docker exec -it trails_singa_polardb bash
+tail -f /var/polardb/primary_datadir/trails_log_folder/<log_file_name>
+```
+

examples/model_selection/Trails/singa.polarDB.Dockerfile

@@ -66,12 +66,18 @@ RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y && \
 
 # Clone code to there, install dependences,
 WORKDIR /home/postgres
-RUN git clone https://github.com/NLGithubWP/Trails.git && \
+RUN mkdir Trails && \
     cd Trails && \
-    git checkout trails_singa && \
-    cp ./internal/pg_extension/template/Cargo.pg11.toml ./internal/pg_extension/Cargo.toml && \
+    git init && \
+    git remote add origin https://github.com/apache/singa.git && \
+    git config core.sparseCheckout true && \
+    echo "examples/model_selection/Trails/*" > .git/info/sparse-checkout && \
+    git pull --depth=1 origin dev-postgresql && \
+    mv examples/model_selection/Trails/* . && \
+    rm -rf examples && \
+    cp ./internal/pg_extension/template/Cargo.pg14.toml ./internal/pg_extension/Cargo.toml && \
     cd ./internal/ml/model_selection && \
-    pip install -r requirement.txt  && \
+    pip install -r requirement.txt && \
     pip install ../../../singa_pkg_code/singa-3.1.0-cp38-cp38-manylinux2014_x86_64.whl
 
 

examples/model_selection/Trails/singa.psql.Dockerfile

@@ -75,14 +75,21 @@ ARG CACHEBUST=1
 
 # Clone code to there, install dependences,
 WORKDIR /project
-RUN git clone https://github.com/NLGithubWP/Trails.git && \
+RUN mkdir Trails && \
     cd Trails && \
-    git checkout trails_singa && \
+    git init && \
+    git remote add origin https://github.com/apache/singa.git && \
+    git config core.sparseCheckout true && \
+    echo "examples/model_selection/Trails/*" > .git/info/sparse-checkout && \
+    git pull --depth=1 origin dev-postgresql && \
+    mv examples/model_selection/Trails/* . && \
+    rm -rf examples && \
     cp ./internal/pg_extension/template/Cargo.pg14.toml ./internal/pg_extension/Cargo.toml && \
     cd ./internal/ml/model_selection && \
-    pip install -r requirement.txt  && \
+    pip install -r requirement.txt && \
     pip install ../../../singa_pkg_code/singa-3.1.0-cp38-cp38-manylinux2014_x86_64.whl
 
+
 WORKDIR /project
 RUN chmod +x ./Trails/init.sh