PLATFORM-5417| add maxThreads parameter to S3Put

Author: harnash

docker/maintenance/cronjob-template.yaml

@@ -31,8 +31,6 @@ spec:
                 value: sjc
               - name: WIKIA_ENVIRONMENT
                 value: prod
-              - name: S4CMD_NUM_THREADS
-                value: 2
             resources:
               limits:
                 cpu: "6"

extensions/wikia/WikiFactory/Close/maintenance.php

@@ -161,7 +161,8 @@ public function execute() {
 					try {
 						foreach( $this->tarFiles( $dbname, $cityid ) as $source ) {
 							if ( is_string( $source ) ) {
-								$res = DumpsOnDemand::putToAmazonS3( $source, !$hide, MimeMagic::singleton()->guessMimeType( $source ) );
+								$res = DumpsOnDemand::putToAmazonS3( $source, !$hide, MimeMagic::singleton()
+									->guessMimeType( $source ), 2 );
 								if ( !$res ) {
 									$this->error( "putToAmazonS3 command failed - Can't copy images to remote host. Please, fix that and rerun",
 										[ 'dump_size_bytes' => filesize( $source ), ] );

extensions/wikia/WikiFactory/Dumps/DumpsOnDemand.php

@@ -170,13 +170,13 @@ static public function getPath( $sName ) {
 	 *
 	 * @param string $sPath
 	 * @param bool $bPublic
-	 * @param string|null $sMimeType
-	 * if $sMimeType is set then the specified mime tipe is set, otherwise
+	 * @param null $sMimeType
+	 * if $sMimeType is set then the specified mime type is set, otherwise
 	 *      let AmazonS3 decide on mime type.
+	 * @param int $iMaxThreads set the maximum number of threads used by the s3 upload tool
 	 * @return bool
-	 * @throws Exception
 	 */
-	static public function putToAmazonS3( $sPath, $bPublic = true, $sMimeType = null ) {
+	static public function putToAmazonS3( $sPath, $bPublic = true, $sMimeType = null, $iMaxThreads = 0 ) {
 		$time = wfTime();
 		$size = filesize( $sPath );
 
@@ -186,6 +186,7 @@ static public function putToAmazonS3( $sPath, $bPublic = true, $sMimeType = null
 		}
 		$s3->setContentDisposition( 'attachment' );
 		$s3->setForce();
+		$s3->setMaxThreads( $iMaxThreads );
 
 		if ( !is_null( $sMimeType ) ) {
 			$s3->setContentType( $sMimeType );

lib/Wikia/src/s3/S3Put.php

@@ -23,6 +23,8 @@ class S3Put {
 	private $contentDisposition = null;
 	/** @var bool */
 	private $force = false;
+	/** @var ?integer  */
+	private $maxThreads = null;
 
 	/**
 	 * Makes the upcoming upload public
@@ -34,6 +36,18 @@ public function setPublic() {
 		return $this;
 	}
 
+	/**
+	 * Set the maximum number of threads to be utilized by the upload tool.
+	 * Useful when working in resource restricted environments.
+	 *
+	 * @param int $newMaxThreads
+	 * @return $this
+	 */
+	public function setMaxThreads( int $newMaxThreads ) {
+		$this->maxThreads = $newMaxThreads;
+		return $this;
+	}
+
 	/**
 	 * Sets the Content-Type header for the upcoming upload.
 	 *
@@ -92,10 +106,14 @@ public function execute( string $localPath, string $bucket, string $remotePath )
 		);
 
 		try {
-			$out = wfShellExec( $cmd, $result, [
+			$env = [
 				'S3_ACCESS_KEY' => $wgAWSAccessKey,
 				'S3_SECRET_KEY' => $wgAWSSecretKey
-			] );
+			];
+			if ( $this->maxThreads ) {
+				$env['S4CMD_NUM_THREADS'] = $this->maxThreads;
+			}
+			$out = wfShellExec( $cmd, $result, $env );
 		} catch ( \Exception $e ) {
 			throw new S3Exception( 'Error while executing s4cmd:' . $e->getMessage() );
 		}