Merge pull request #922 from skalenetwork/snapshot-from-repair

Set no-snapshot-majority in RepairMonitor

core/schains/cmd.py

@@ -17,8 +17,6 @@
 #   You should have received a copy of the GNU Affero General Public License
 #   along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
-from typing import Optional
-
 from core.schains.config.helper import get_schain_ports
 from core.schains.config.static_params import get_static_schain_cmd
 from core.schains.ssl import get_ssl_filepath
@@ -33,7 +31,7 @@ def get_schain_container_cmd(
     public_key: str = None,
     start_ts: int = None,
     enable_ssl: bool = True,
-    snapshot_from: Optional[str] = None
+    snapshot_from: str = ''
 ) -> str:
     """Returns parameters that will be passed to skaled binary in the sChain container"""
     opts = get_schain_container_base_opts(schain_name, enable_ssl=enable_ssl)

core/schains/monitor/containers.py

@@ -64,6 +64,7 @@ def monitor_schain_container(
             schain=schain,
             public_key=public_key,
             start_ts=start_ts,
+            snapshot_from=schain_record.snapshot_from,
             dutils=dutils
         )
         schain_record.reset_failed_conunters()

core/schains/monitor/repair_monitor.py

@@ -20,6 +20,7 @@
 import logging
 from core.schains.monitor.base_monitor import BaseMonitor
 from tools.notifications.messages import notify_repair_mode
+from web.models.schain import switch_off_repair_mode
 
 logger = logging.getLogger(__name__)
 
@@ -40,7 +41,7 @@ def notify_repair_mode(self) -> None:
         )
 
     def disable_repair_mode(self) -> None:
-        self.schain_record.set_repair_mode(False)
+        switch_off_repair_mode(self.name)
 
     @BaseMonitor.monitor_runner
     def run(self):

core/schains/runner.py

@@ -41,7 +41,6 @@
 )
 from tools.configs import (NODE_DATA_PATH_HOST, SCHAIN_NODE_DATA_PATH, SKALE_DIR_HOST,
                            SKALE_VOLUME_PATH, SCHAIN_CONFIG_DIR_SKALED)
-from tools.node_options import NodeOptions
 
 logger = logging.getLogger(__name__)
 
@@ -159,6 +158,7 @@ def run_schain_container(
     dutils=None,
     volume_mode=None,
     ulimit_check=True,
+    snapshot_from: str = '',
     enable_ssl=True
 ):
     schain_name = schain['name']
@@ -173,9 +173,6 @@ def run_schain_container(
     )
     env = get_schain_env(ulimit_check=ulimit_check)
 
-    node_options = NodeOptions()
-    logger.info('Node options for chians %s', node_options.all())
-    snapshot_from = node_options.snapshot_from
     cmd = get_schain_container_cmd(
         schain_name,
         public_key,

requirements.txt

@@ -25,7 +25,7 @@ config-controller-predeployed==1.0.1b0
 psutil==5.9.3
 
 colorful==0.5.4
-celery==5.0.2
+celery==5.2.2
 
 filelock==3.0.12
 

tests/db_test.py

@@ -7,6 +7,7 @@
     get_schains_statuses,
     mark_schain_deleted,
     set_schains_first_run,
+    switch_off_repair_mode,
     toggle_schain_repair_mode,
     SChainRecord,
     upsert_schain_record
@@ -76,6 +77,25 @@ def test_toggle_repair_mode(db, upsert_db):
     records = list(cursor)
     assert len(records) == 1
     assert records[0].name == 'schain-0'
+    assert records[0].snapshot_from == ''
+
+    result = toggle_schain_repair_mode('schain-0', '1.1.1.1')
+    cursor = SChainRecord.select().where(
+        SChainRecord.repair_mode == True).execute()  # noqa: E712
+    records = list(cursor)
+    assert len(records) == 1
+    assert records[0].name == 'schain-0'
+    assert records[0].snapshot_from == '1.1.1.1'
+
+    switch_off_repair_mode('schain-0')
+    assert SChainRecord.select().where(
+        SChainRecord.repair_mode == True).count() == 0  # noqa: E712
+    cursor = SChainRecord.select().where(
+        SChainRecord.name == 'schain-0').execute()  # noqa: E712
+    records = list(cursor)
+    assert records[0].name == 'schain-0'
+    assert not records[0].repair_mode
+    assert records[0].snapshot_from == ''
 
 
 def test_toggle_repair_mode_schain_not_exists(db, upsert_db):

tests/routes/schains_test.py

@@ -102,6 +102,22 @@ def test_enable_repair_mode(skale_bp, schain_db):
         'payload': {},
         'status': 'ok'
     }
+    r = upsert_schain_record(schain_name)
+    assert r.repair_mode
+    assert r.snapshot_from == ''
+
+    data = post_bp_data(
+        skale_bp,
+        get_api_url(BLUEPRINT_NAME, 'repair'),
+        params={'schain_name': schain_name, 'snapshot_from': '1.1.1.1'}
+    )
+    assert data == {
+        'payload': {},
+        'status': 'ok'
+    }
+    r = upsert_schain_record(schain_name)
+    assert r.repair_mode
+    assert r.snapshot_from == '1.1.1.1'
 
     data = post_bp_data(skale_bp, get_api_url(BLUEPRINT_NAME, 'repair'),
                         params={'schain_name': 'undefined-schain'})

tests/schains/cmd_test.py

@@ -47,6 +47,17 @@ def test_get_schain_container_cmd(schain_config, cert_key_pair):
     )
     assert container_opts == expected_opts
 
+    container_opts = get_schain_container_cmd(schain_name, snapshot_from='')
+    expected_opts = (
+        f'--config {config_filepath} -d /data_dir --ipcpath /data_dir --http-port 10003 '
+        f'--https-port 10008 --ws-port 10002 --wss-port 10007 --sgx-url {SGX_SERVER_URL} '
+        f'--shared-space-path {SHARED_SPACE_CONTAINER_PATH}/data '
+        f'--main-net-url {IMA_ENDPOINT} -v 3 '
+        f'--web3-trace --enable-debug-behavior-apis '
+        f'--aa no --ssl-key {ssl_key_path} --ssl-cert {ssl_cert_path}'
+    )
+    assert container_opts == expected_opts
+
 
 def test_get_schain_container_sync_opts():
     sync_opts = get_schain_container_sync_opts(start_ts=123)

tools/node_options.py

@@ -29,7 +29,3 @@
 class NodeOptions(JsonObject):
     def __init__(self):
         super().__init__(filepath=NODE_OPTIONS_FILEPATH)
-
-    @property
-    def snapshot_from(self) -> bool:
-        return self._get('snapshot_from')

web/migrations.py

@@ -55,6 +55,9 @@ def run_migrations(db, migrator):
     add_restart_count_field(db, migrator)
     add_failed_rpc_count_field(db, migrator)
 
+    # 2.3 -> 2.4 update fields
+    add_failed_snapshot_from(db, migrator)
+
 
 def add_new_schain_field(db, migrator):
     add_column(
@@ -112,6 +115,13 @@ def add_failed_rpc_count_field(db, migrator):
     )
 
 
+def add_failed_snapshot_from(db, migrator):
+    add_column(
+        db, migrator, 'SChainRecord', 'snapshot_from',
+        CharField(default='')
+    )
+
+
 def find_column(db, table_name, column_name):
     columns = db.get_columns(table_name)
     return next((x for x in columns if x.name == column_name), None)

web/models/schain.py

@@ -44,6 +44,7 @@ class SChainRecord(BaseModel):
     monitor_id = IntegerField(default=0)
 
     config_version = CharField(default=DEFAULT_CONFIG_VERSION)
+    snapshot_from = CharField(default='')
     restart_count = IntegerField(default=0)
     failed_rpc_count = IntegerField(default=0)
 
@@ -159,6 +160,11 @@ def set_failed_rpc_count(self, value: int) -> None:
         self.failed_rpc_count = value
         self.save()
 
+    def set_snapshot_from(self, value: str) -> None:
+        logger.info(f'Changing snapshot from for {self.name} to {value}')
+        self.snapshot_from = value
+        self.save()
+
     def reset_failed_conunters(self) -> None:
         logger.info(f'Resetting failed counters for {self.name}')
         self.set_restart_count(0)
@@ -236,9 +242,21 @@ def get_schains_statuses(include_deleted=False):
             for r in SChainRecord.get_all_records(include_deleted)]
 
 
-def toggle_schain_repair_mode(name):
+def toggle_schain_repair_mode(name, snapshot_from: str = ''):
     logger.info(f'Toggling repair mode for schain {name}')
-    query = SChainRecord.update(repair_mode=True).where(
-        SChainRecord.name == name)
+    query = SChainRecord.update(
+        repair_mode=True,
+        snapshot_from=snapshot_from
+    ).where(SChainRecord.name == name)
+    count = query.execute()
+    return count > 0
+
+
+def switch_off_repair_mode(name):
+    logger.info(f'Disabling repair mode for schain {name}')
+    query = SChainRecord.update(
+        repair_mode=False,
+        snapshot_from=''
+    ).where(SChainRecord.name == name)
     count = query.execute()
     return count > 0

web/routes/schains.py

@@ -136,7 +136,8 @@ def firewall_rules():
 def repair():
     logger.debug(request)
     schain_name = request.json.get('schain_name')
-    result = toggle_schain_repair_mode(schain_name)
+    snapshot_from = request.json.get('snapshot_from', '')
+    result = toggle_schain_repair_mode(schain_name, snapshot_from=snapshot_from)
     if result:
         return construct_ok_response()
     else: