HADOOP-19351. S3A: Add config option to skip test with performance mo…

…de (#7223)

The option test.fs.s3a.performance.enabled can be set to false to disable tests of
s3 stores which knowingly break posix semantics.
These tests need to be disabled when testing object stores which do enforce
a strict model of directories and files underneath the S3 API.

Contributed by Chung En Lee

hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/performance.md

@@ -294,6 +294,7 @@ understands the risks.
 * If an option is to be tuned which may relax semantics, a new option MUST be defined.
 * Unknown flags are ignored; this is to avoid compatibility.
 * The option `*` means "turn everything on". This is implicitly unstable across releases.
+* Other stores may retain stricter semantics.
 
 | *Option* | *Meaning*          | Since |
 |----------|--------------------|:------|

hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/testing.md

@@ -581,6 +581,10 @@ on third party stores.
     <name>test.fs.s3a.create.create.acl.enabled</name>
     <value>false</value>
   </property>
+  <property>
+    <name>test.fs.s3a.performance.enabled</name>
+    <value>false</value>
+  </property>
 ```
 
 See [Third Party Stores](third_party_stores.html) for more on this topic.
@@ -720,6 +724,18 @@ Tests in `ITestS3AContentEncoding` may need disabling
     <value>false</value>
   </property>
 ```
+
+### Disabling tests running in performance mode
+
+Some tests running in performance mode turn off the safety checks. They expect operations which break POSIX semantics to succeed.
+For stores with stricter semantics, these test cases must be disabled.
+```xml
+  <property>
+    <name>test.fs.s3a.performance.enabled</name>
+    <value>false</value>
+  </property>
+```
+
 ### Tests which may fail (and which you can ignore)
 
 * `ITestS3AContractMultipartUploader` tests `testMultipartUploadAbort` and `testSingleUpload` raising `FileNotFoundException`

hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractCreate.java

@@ -29,9 +29,11 @@
 import org.apache.hadoop.fs.contract.AbstractFSContract;
 import org.apache.hadoop.fs.s3a.S3ATestUtils;
 
+import static org.apache.hadoop.fs.s3a.S3ATestConstants.KEY_PERFORMANCE_TESTS_ENABLED;
 import static org.apache.hadoop.fs.s3a.Constants.CONNECTION_EXPECT_CONTINUE;
 import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides;
 import static org.apache.hadoop.fs.s3a.S3ATestUtils.setPerformanceFlags;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.skipIfNotEnabled;
 
 /**
  * S3A contract tests creating files.
@@ -84,6 +86,9 @@ protected Configuration createConfiguration() {
         conf,
         CONNECTION_EXPECT_CONTINUE);
     conf.setBoolean(CONNECTION_EXPECT_CONTINUE, expectContinue);
+    if (createPerformance) {
+      skipIfNotEnabled(conf, KEY_PERFORMANCE_TESTS_ENABLED, "Skipping tests running in performance mode");
+    }
     S3ATestUtils.disableFilesystemCaching(conf);
     return conf;
   }

hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/contract/s3a/ITestS3AContractMkdirWithCreatePerf.java

@@ -29,7 +29,9 @@
 
 import static org.apache.hadoop.fs.contract.ContractTestUtils.createFile;
 import static org.apache.hadoop.fs.contract.ContractTestUtils.dataset;
+import static org.apache.hadoop.fs.s3a.S3ATestConstants.KEY_PERFORMANCE_TESTS_ENABLED;
 import static org.apache.hadoop.fs.s3a.S3ATestUtils.setPerformanceFlags;
+import static org.apache.hadoop.fs.s3a.S3ATestUtils.skipIfNotEnabled;
 
 /**
  * Test mkdir operations on S3A with create performance mode.
@@ -50,6 +52,8 @@ protected AbstractFSContract createContract(Configuration conf) {
 
   @Test
   public void testMkdirOverParentFile() throws Throwable {
+    skipIfNotEnabled(getContract().getConf(), KEY_PERFORMANCE_TESTS_ENABLED,
+        "Skipping tests running in performance mode");
     describe("try to mkdir where a parent is a file, should pass");
     FileSystem fs = getFileSystem();
     Path path = methodPath();

hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/S3ATestConstants.java

@@ -63,6 +63,11 @@ public interface S3ATestConstants {
    */
   String KEY_ACL_TESTS_ENABLED = TEST_FS_S3A + "create.acl.enabled";
 
+  /**
+   * A property set to true if tests running in performance mode are enabled: {@value }
+   */
+  String KEY_PERFORMANCE_TESTS_ENABLED = TEST_FS_S3A + "performance.enabled";
+
   /**
    * A property set to true if V1 tests are enabled: {@value}.
    */