Skip to content

Commit

Permalink
[fix](restore) Fix view signature #41120 (#41149)
Browse files Browse the repository at this point in the history
cherry pick from #41120
  • Loading branch information
w41ter authored Sep 23, 2024
1 parent 36c9046 commit 764dcd7
Show file tree
Hide file tree
Showing 5 changed files with 194 additions and 9 deletions.
21 changes: 14 additions & 7 deletions fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
Original file line number Diff line number Diff line change
Expand Up @@ -751,16 +751,23 @@ private void checkAndPrepareMeta() {
if (localTbl != null) {
Preconditions.checkState(localTbl.getType() == TableType.VIEW);
View localView = (View) localTbl;
if (!localView.getSignature(BackupHandler.SIGNATURE_VERSION)
.equals(remoteView.getSignature(BackupHandler.SIGNATURE_VERSION))) {
status = new Status(ErrCode.COMMON_ERROR, "View "
+ jobInfo.getAliasByOriginNameIfSet(backupViewName)
+ " already exist but with different schema");
return;
String localViewSignature = localView.getSignature(BackupHandler.SIGNATURE_VERSION);
// keep compatible with old version, compare the signature without reset view def
if (!localViewSignature.equals(remoteView.getSignature(BackupHandler.SIGNATURE_VERSION))) {
// reset view def to dest db name and compare signature again
String srcDbName = jobInfo.dbName;
remoteView.resetViewDefForRestore(srcDbName, db.getName());
if (!localViewSignature.equals(remoteView.getSignature(BackupHandler.SIGNATURE_VERSION))) {
status = new Status(ErrCode.COMMON_ERROR, "View "
+ jobInfo.getAliasByOriginNameIfSet(backupViewName)
+ " already exist but with different schema");
return;
}
}
} else {
String srcDbName = jobInfo.dbName;
remoteView.resetIdsForRestore(env, srcDbName, db.getFullName());
remoteView.resetViewDefForRestore(srcDbName, db.getName());
remoteView.resetIdsForRestore(env);
restoredTbls.add(remoteView);
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -591,7 +591,7 @@ public Status resetIdsForRestore(Env env, Database db, ReplicaAllocation restore
baseIndexId = newIdxId;
}
MaterializedIndexMeta indexMeta = origIdxIdToMeta.get(entry.getKey());
indexMeta.resetIndexIdForRestore(newIdxId, srcDbName, db.getFullName());
indexMeta.resetIndexIdForRestore(newIdxId, srcDbName, db.getName());
indexIdToMeta.put(newIdxId, indexMeta);
indexNameToId.put(entry.getValue(), newIdxId);
}
Expand Down
4 changes: 3 additions & 1 deletion fe/fe-core/src/main/java/org/apache/doris/catalog/View.java
Original file line number Diff line number Diff line change
Expand Up @@ -247,9 +247,11 @@ public View clone() {
return copied;
}

public void resetIdsForRestore(Env env, String srcDbName, String dbName) {
public void resetIdsForRestore(Env env) {
id = env.getNextId();
}

public void resetViewDefForRestore(String srcDbName, String dbName) {
// the source db name is not setted in old BackupMeta, keep compatible with the old one.
if (srcDbName != null) {
inlineViewDef = inlineViewDef.replaceAll(srcDbName, dbName);
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
-- This file is automatically generated. You should know what you did if you want to edit this
-- !sql --
1 1
2 2
3 3
4 4
5 5
6 6
7 7
8 8
9 9
10 10

-- !sql --
6 6
7 7
8 8
9 9
10 10

-- !sql --
1 1
2 2
3 3
4 4
5 5
6 6
7 7
8 8
9 9
10 10

-- !sql --
6 6
7 7
8 8
9 9
10 10

Original file line number Diff line number Diff line change
@@ -0,0 +1,137 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

suite("test_backup_restore_with_view", "backup_restore") {
String suiteName = "backup_restore_with_view"
String dbName = "${suiteName}_db"
String dbName1 = "${suiteName}_db_1"
String repoName = "repo_" + UUID.randomUUID().toString().replace("-", "")
String snapshotName = "${suiteName}_snapshot"
String tableName = "${suiteName}_table"
String viewName = "${suiteName}_view"

def syncer = getSyncer()
syncer.createS3Repository(repoName)
sql "CREATE DATABASE IF NOT EXISTS ${dbName}"
sql "CREATE DATABASE IF NOT EXISTS ${dbName1}"

int numRows = 10;
sql "DROP TABLE IF EXISTS ${dbName}.${tableName} FORCE"
sql "DROP VIEW IF EXISTS ${dbName}.${viewName}"
sql """
CREATE TABLE ${dbName}.${tableName} (
`id` LARGEINT NOT NULL,
`count` LARGEINT SUM DEFAULT "0"
)
AGGREGATE KEY(`id`)
DISTRIBUTED BY HASH(`id`) BUCKETS 2
PROPERTIES
(
"replication_num" = "1"
)
"""
List<String> values = []
for (int j = 1; j <= numRows; ++j) {
values.add("(${j}, ${j})")
}
sql "INSERT INTO ${dbName}.${tableName} VALUES ${values.join(",")}"

sql """CREATE VIEW ${dbName}.${viewName} (id, count)
AS
SELECT * FROM ${dbName}.${tableName} WHERE count > 5
"""

qt_sql "SELECT * FROM ${dbName}.${tableName} ORDER BY id ASC"
qt_sql "SELECT * FROM ${dbName}.${viewName} ORDER BY id ASC"

sql """
BACKUP SNAPSHOT ${dbName}.${snapshotName}
TO `${repoName}`
"""

syncer.waitSnapshotFinish(dbName)

def snapshot = syncer.getSnapshotTimestamp(repoName, snapshotName)
assertTrue(snapshot != null)

sql "DROP TABLE IF EXISTS ${dbName1}.${tableName} FORCE"
sql "DROP VIEW IF EXISTS ${dbName1}.${viewName}"

sql """
RESTORE SNAPSHOT ${dbName1}.${snapshotName}
FROM `${repoName}`
PROPERTIES
(
"backup_timestamp" = "${snapshot}",
"reserve_replica" = "true"
)
"""

syncer.waitAllRestoreFinish(dbName1)

qt_sql "SELECT * FROM ${dbName1}.${tableName} ORDER BY id ASC"
qt_sql "SELECT * FROM ${dbName1}.${viewName} ORDER BY id ASC"
def show_view_result = sql_return_maparray "SHOW VIEW FROM ${tableName} FROM ${dbName1}"
logger.info("show view result: ${show_view_result}")
assertTrue(show_view_result.size() == 1);
def show_view = show_view_result[0]['Create View']
assertTrue(show_view.contains("${dbName1}"))
assertTrue(show_view.contains("${tableName}"))

// restore to db, test the view signature.
sql """
RESTORE SNAPSHOT ${dbName}.${snapshotName}
FROM `${repoName}`
PROPERTIES
(
"backup_timestamp" = "${snapshot}",
"reserve_replica" = "true"
)
"""

syncer.waitAllRestoreFinish(dbName)
def restore_result = sql_return_maparray """ SHOW RESTORE FROM ${dbName} WHERE Label ="${snapshotName}" """
restore_result.last()
logger.info("show restore result: ${restore_result}")
assertTrue(restore_result.last().State == "FINISHED")

// restore to db1, test the view signature.
sql """
RESTORE SNAPSHOT ${dbName1}.${snapshotName}
FROM `${repoName}`
PROPERTIES
(
"backup_timestamp" = "${snapshot}",
"reserve_replica" = "true"
)
"""

syncer.waitAllRestoreFinish(dbName1)
restore_result = sql_return_maparray """ SHOW RESTORE FROM ${dbName1} WHERE Label ="${snapshotName}" """
restore_result.last()
logger.info("show restore result: ${restore_result}")
assertTrue(restore_result.last().State == "FINISHED")

sql "DROP TABLE ${dbName}.${tableName} FORCE"
sql "DROP VIEW ${dbName}.${viewName}"
sql "DROP DATABASE ${dbName} FORCE"
sql "DROP TABLE ${dbName1}.${tableName} FORCE"
sql "DROP VIEW ${dbName1}.${viewName}"
sql "DROP DATABASE ${dbName1} FORCE"
sql "DROP REPOSITORY `${repoName}`"
}

0 comments on commit 764dcd7

Please sign in to comment.