Skip to content

Commit

Permalink
HADOOP-19385. S3A: add a hadoop-format-testing module for testing
Browse files Browse the repository at this point in the history 
Pom with dependencies

Change-Id: I9c01a724b085f8c409ca22fb8e79de484b0273f8
  • Loading branch information
@steveloughran
steveloughran committed Jan 15, 2025
1 parent d2095fa commit a2083fc
Show file tree
Hide file tree
Showing 4 changed files with 224 additions and 0 deletions.
205 changes: 205 additions & 0 deletions hadoop-cloud-storage-project/hadoop-format-testing/pom.xml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,205 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. See accompanying LICENSE file.
-->
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
<version>3.5.0-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-format-testing</artifactId>
<version>3.5.0-SNAPSHOT</version>
<name>Apache Hadoop Format Regression Testing</name>
<description>
This module is exclusively for use in regression testing
integration between various file formats and
cloud connectors.

* It MUST NOT be declared as a dependency by
any external component.
* It MUST NOT be declared as a dependency except
in test scope.

Consider its status: Private.

For everyone who does a full stack build of the file
format libraries as well as hadoop, this adds more loops
to the build. However, because it is not a public distributable,
it is safe to use a previous build's artifacts...all that is
lost is the immediate regression testing.

Note that all test dependencies are declared as of the
production JAR, not the test one.

This is to simplify use in testing in cloud modules;
import this JAR and get all the transient dependencies
tuned for these tests.
</description>
<packaging>jar</packaging>

<properties>
<file.encoding>UTF-8</file.encoding>
<downloadSources>true</downloadSources>
<parquet.version>1.15.0</parquet.version>
<!-- scope for all the test running artifacts -->
<testing.scope>compile</testing.scope>
</properties>

<dependencies>

<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<scope>provided</scope>
<exclusions>
<exclusion>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</exclusion>
<exclusion>
<groupId>org.checkerframework</groupId>
<artifactId>checker-qual</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-annotations</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<type>test-jar</type>
<scope>provided</scope>
<exclusions>
<exclusion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-api</artifactId>
<scope>${testing.scope}</scope>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>${testing.scope}</scope>
</dependency>
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-engine</artifactId>
<scope>${testing.scope}</scope>
</dependency>
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-params</artifactId>
<scope>${testing.scope}</scope>
</dependency>
<dependency>
<groupId>org.junit.vintage</groupId>
<artifactId>junit-vintage-engine</artifactId>
<scope>${testing.scope}</scope>
</dependency>
<!-- Only required to run tests in an IDE that bundles an older version -->
<dependency>
<groupId>org.junit.platform</groupId>
<artifactId>junit-platform-launcher</artifactId>
<scope>${testing.scope}</scope>
</dependency>
<dependency>
<groupId>org.assertj</groupId>
<artifactId>assertj-core</artifactId>
<scope>${testing.scope}</scope>
</dependency>
<dependency>
<groupId>org.hamcrest</groupId>
<artifactId>hamcrest-library</artifactId>
<scope>${testing.scope}</scope>
</dependency>

<dependency>
<groupId>org.apache.parquet</groupId>
<artifactId>parquet-hadoop</artifactId>
<version>${parquet.version}</version>
</dependency>

</dependencies>

<build>
<resources>
<resource>
<directory>src/main/resources</directory>
<filtering>true</filtering>
</resource>
</resources>
<testResources>
<testResource>
<directory>src/test/resources</directory>
<filtering>true</filtering>
</testResource>
</testResources>
<plugins>

<plugin>
<artifactId>maven-dependency-plugin</artifactId>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>copy-dependencies</goal>
</goals>
<configuration>
<outputDirectory>${project.build.directory}/lib</outputDirectory>
</configuration>
</execution>
<execution>
<id>deplist</id>
<phase>compile</phase>
<goals>
<goal>list</goal>
</goals>
<configuration>
<!-- referenced by a built-in command -->
<outputFile>${project.basedir}/target/hadoop-tools-deps/${project.artifactId}.tools-builtin.txt</outputFile>
</configuration>
</execution>
</executions>
</plugin>

<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-source-plugin</artifactId>
<configuration>
<attach>true</attach>
</configuration>
<executions>
<execution>
<goals>
<goal>jar</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>
1 change: 1 addition & 0 deletions hadoop-cloud-storage-project/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
<modules>
<module>hadoop-cloud-storage</module>
<module>hadoop-cos</module>
<module>hadoop-format-testing</module>
<module>hadoop-huaweicloud</module>
</modules>

Expand Down
12 changes: 12 additions & 0 deletions hadoop-project/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -788,6 +788,18 @@
<type>test-jar</type>
</dependency>

<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-format-testing</artifactId>
<version>${hadoop.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-format-testing</artifactId>
<version>${hadoop.version}</version>
<type>test-jar</type>
</dependency>

<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
Expand Down
6 changes: 6 additions & 0 deletions hadoop-tools/hadoop-aws/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -581,5 +581,11 @@
<artifactId>bcpkix-jdk18on</artifactId>
<scope>test</scope>
</dependency>

<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-format-testing</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
</project>

0 comments on commit a2083fc

Please sign in to comment.