Users/v kivlev/backport redaction to 3.227.2 (#4467)

kirill-ivlev · michaelcfanning · web-flow · commit 9667e03e38e5 · 2023-10-12T18:15:10.000+04:00
* Restore expanded redaction as a default feature. Add a set of new, highly accurate detections.

* Simplify provider name.

* Add feature flag support

* Cleanup

* Set off by default / remove knob, add tracing

* added extension / fixed tests

* remove duplicate masker, changes related to tests

* Update tests, remove extra blank lines

* update agent version

---------

Co-authored-by: Michael C. Fanning &lt;mikefan@microsoft.com&gt;
Co-authored-by: v-kivlev &lt;undefined&gt;
diff --git a/src/Agent.Listener/Configuration/FeatureFlagProvider.cs b/src/Agent.Listener/Configuration/FeatureFlagProvider.cs
@@ -49,7 +49,7 @@ public async Task<FeatureFlag> GetFeatureFlagAsync(IHostContext context, string
             var client = vssConnection.GetClient<FeatureAvailabilityHttpClient>();
             try
             {
-                return await client.GetFeatureFlagByNameAsync(featureFlagName);
+                return await client.GetFeatureFlagByNameAsync(featureFlagName, checkFeatureExists: false);
             } catch(VssServiceException e)
             {
                 Trace.Warning("Unable to retrive feature flag status: " + e.ToString());
diff --git a/src/Agent.Listener/JobDispatcher.cs b/src/Agent.Listener/JobDispatcher.cs
@@ -17,7 +17,7 @@
 using System.Linq;
 using Microsoft.VisualStudio.Services.Common;
 using System.Diagnostics;
-
+using Agent.Listener.Configuration;
 
 namespace Microsoft.VisualStudio.Services.Agent.Listener
 {
@@ -88,6 +88,18 @@ public void Run(Pipelines.AgentJobRequestMessage jobRequestMessage, bool runOnce
                 }
             }
 
+            var service = HostContext.GetService<IFeatureFlagProvider>();
+            string ffState;
+            try
+            {
+                ffState = service.GetFeatureFlagAsync(HostContext, "DistributedTask.Agent.EnableAdditionalMaskingRegexes", Trace)?.Result?.EffectiveState;
+            }
+            catch (Exception)
+            {
+                ffState = "Off";
+            }
+            jobRequestMessage.Variables[Constants.Variables.Features.EnableAdditionalMaskingRegexes] = ffState;
+
             WorkerDispatcher newDispatch = new WorkerDispatcher(jobRequestMessage.JobId, jobRequestMessage.RequestId);
             if (runOnce)
             {
diff --git a/src/Agent.Sdk/Knob/AgentKnobs.cs b/src/Agent.Sdk/Knob/AgentKnobs.cs
@@ -284,12 +284,6 @@ public class AgentKnobs
             new EnvironmentKnobSource("SYSTEM_UNSAFEALLOWMULTILINESECRET"),
             new BuiltInDefaultKnobSource("false"));
 
-        public static readonly Knob MaskUsingCredScanRegexes = new Knob(
-            nameof(MaskUsingCredScanRegexes),
-            "Use the CredScan regexes for masking secrets. CredScan is an internal tool developed at Microsoft to keep passwords and authentication keys from being checked in. This defaults to disabled, as there are performance problems with some task outputs.",
-            new EnvironmentKnobSource("AZP_USE_CREDSCAN_REGEXES"),
-            new BuiltInDefaultKnobSource("false"));
-
         public static readonly Knob MaskedSecretMinLength = new Knob(
             nameof(MaskedSecretMinLength),
             "Specify the length of the secrets, which, if shorter, will be ignored in the logs.",
diff --git a/src/Agent.Worker/Worker.cs b/src/Agent.Worker/Worker.cs
@@ -10,6 +10,7 @@
 using System.Threading.Tasks;
 using Microsoft.VisualStudio.Services.WebApi;
 using Agent.Sdk.Util;
+using Agent.Sdk.Knob;
 
 namespace Microsoft.VisualStudio.Services.Agent.Worker
 {
@@ -67,6 +68,19 @@ public async Task<int> RunAsync(string pipeIn, string pipeOut)
                 InitializeSecretMasker(jobMessage);
                 SetCulture(jobMessage);
 
+                var maskUsingCredScanRegexesState = "Off";
+
+                if(jobMessage.Variables.TryGetValue(Constants.Variables.Agent.EnableAdditionalMaskingRegexes, out var enableAdditionalMaskingRegexes))
+                {
+                    maskUsingCredScanRegexesState = enableAdditionalMaskingRegexes.Value;
+                }
+
+                if(maskUsingCredScanRegexesState == "On")
+                {
+                    Trace.Verbose($"{Constants.Variables.Agent.EnableAdditionalMaskingRegexes} is On, adding additional masking regexes");
+                    HostContext.AddAdditionalMaskingRegexes();
+                }
+
                 // Start the job.
                 Trace.Info($"Job message:{Environment.NewLine} {StringUtil.ConvertToJson(WorkerUtilities.ScrubPiiData(jobMessage))}");
                 Task<TaskResult> jobRunnerTask = jobRunner.RunAsync(jobMessage, jobRequestCancellationToken.Token);
diff --git a/src/Microsoft.VisualStudio.Services.Agent/AdditionalMaskingRegexes.CredScan.cs b/src/Microsoft.VisualStudio.Services.Agent/AdditionalMaskingRegexes.CredScan.cs
@@ -30,176 +30,50 @@ public static partial class AdditionalMaskingRegexes
         private static IEnumerable<string> credScanPatterns =
             new List<string>()
             {
-                // AnsibleVaultData
-                  @"" // pre-match
-                + @"\$ANSIBLE_VAULT;\d+\.\d+;AES256\s+\d+" // match
+                // AAD client app, most recent two versions.
+                  @"\b" // pre-match
+                + @"[0-9A-Za-z-_~.]{3}7Q~[0-9A-Za-z-_~.]{31}\b|\b[0-9A-Za-z-_~.]{3}8Q~[0-9A-Za-z-_~.]{34}" // match
+                + @"\b", // post-match
+                  
+                  // Prominent Azure provider 512-bit symmetric keys.
+                  @"\b" // pre-match
+                + @"[0-9A-Za-z+/]{76}(APIM|ACDb|\+(ABa|AMC|ASt))[0-9A-Za-z+/]{5}[AQgw]==" // match
                 + @"", // post-match
-
-                // AzurePowerShellTokenCache
-                  @"" // pre-match
-                + @"[""']TokenCache[""']\s*:\s*\{\s*[""']CacheData[""']\s*:\s*[""'][a-z0-9/\+]{86}" // match
-                + @"", // post-match
-
-                // Base64EncodedStringLiteral
-                  @"(?<=(^|[""'>;=\s#]))" // pre-match
-                + @"(?<DataBlock>(?-i)MI(?i)[a-z0-9/+\s\u0085\u200b""',\\]{200,20000}[a-z0-9/+]={0,2})" // match
+                       // 
+                  // Prominent Azure provider 256-bit symmetric keys.
+                  @"\b" // pre-match
+                + @"[0-9A-Za-z+/]{33}(AIoT|\+(ASb|AEh|ARm))[A-P][0-9A-Za-z+/]{5}=" // match
                 + @"", // post-match
-
-                // JsonWebToken
-                  @"" // pre-match
-                + @"(?-i)(?<JwtToken>eyJ(?i)[a-z0-9\-_%]+\.(?-i)eyJ(?i)[a-z0-9\-_%]+\.[a-z0-9\-_%]+)|([rR]efresh_?[tT]oken|REFRESH_?TOKEN)[""']?\s*[:=]{1,2}\s*[""']?(?<JwtToken>(\w+-)+\w+)[""']?" // match
-                + @"", // post-match
-
-                // SlackTokens
-                  @"" // pre-match
-                + @"xox[pbar]\-[a-z0-9\-]+" // match
-                + @"", // post-match
-
-                // SymmetricKey128
-                  @"(?<=[^\w/\+\._\$,\\])" // pre-match
-                + @"(?<SymmetricKey>[a-z0-9/\+]{22}==)" // match
-                + @"(?=([^\w/\+\.\$]|$))", // post-match
-
-                // SymmetricKey128Hex
-                  @"(?<=[^\w/\+\._\$,\\][dapi]+)" // pre-match
-                + @"(?<SymmetricKey>[a-f0-9]{32})" // match
-                + @"(?=([^\w/\+\.\$]|$))", // post-match
-
-                // SymmetricKey160Hex
-                  @"(?<=[^\w/\+\._\$,\\])" // pre-match
-                + @"(?<Hex160>[a-f0-9/\+]{40})" // match
-                + @"(?=([^\w/\+\.\$]|$))", // post-match
-
-                // SymmetricKey232
-                  @"(?<=[^\w/\+\._\$,\\])" // pre-match
-                + @"(?<SymmetricKey>(?-i)AIza(?i)[a-z0-9_\\\-]{35})" // match
-                + @"(?=([^\w/\+\.\$]|$))", // post-match
-
-                // SymmetricKey240
-                  @"(?<=[^\w/\+\.\-\$,\\])" // pre-match
-                + @"(?<SymmetricKey>[a-z0-9/\+]{40})" // match
-                + @"(?=([^\w/\+\.\-\$,\\]|$))", // post-match
-
-                // SymmetricKey256
-                  @"(?<=[^\w/\+\.\$,\\])" // pre-match
-                + @"(?<SymmetricKey>[a-z0-9/\+]{43}=)" // match
-                + @"(?=([^\w/\+\.\$]|$))", // post-match
-
-                // SymmetricKey256B32
-                  @"(?<=[^\w/\+\._\-\$,\\])" // pre-match
-                + @"(?<SymmetricKey>(?-i)[a-z2-7]{52}(?i))" // match
-                + @"(?=(?<=[0-9]+[a-z]+[0-9]+.{0,49})([^\w/\+\.\-\$,]|$))", // post-match
-
-                // SymmetricKey256UrlEncoded
-                  @"(?<=[^\w/\+\._\-\$,\\%])" // pre-match
-                + @"(?<SymmetricKey>[a-z0-9%]{43,63}%3d)" // match
+                       
+                  // Azure Function key.
+                  @"\b" // pre-match
+                + @"[0-9A-Za-z_\-]{44}AzFu[0-9A-Za-z\-_]{5}[AQgw]==" // match
                 + @"", // post-match
 
-                // SymmetricKey320
-                  @"(?<=[^\w/\+\.\-\$,\\])" // pre-match
-                + @"(?<SymmetricKey>[a-z0-9/\+]{54}={2})" // match
-                + @"(?=([^\w/\+\.\-\$,\\]|$))", // post-match
-
-                // SymmetricKey320UrlEncoded
-                  @"(?<=[^\w/\+\.\-\$,\\%])" // pre-match
-                + @"(?<SymmetricKey>[a-z0-9%]{54,74}(%3d){2})" // match
+                  // Azure Search keys.
+                  @"\b" // pre-match
+                + @"[0-9A-Za-z]{42}AzSe[A-D][0-9A-Za-z]{5}" // match
+                + @"\b", // post-match
+                  
+                  // Azure Container Registry keys.
+                  @"\b" // pre-match
+                + @"[0-9A-Za-z+/]{42}\+ACR[A-D][0-9A-Za-z+/]{5}" // match
+                + @"\b", // post-match
+                  
+                  // Azure Cache for Redis keys.
+                  @"\b" // pre-match
+                + @"[0-9A-Za-z]{33}AzCa[A-P][0-9A-Za-z]{5}=" // match
                 + @"", // post-match
-
-                // SymmetricKey360
-                  @"(?<=[^\w/\+\.\-\$,\\])" // pre-match
-                + @"(?<SymmetricKey>[a-z0-9/\+]{60})" // match
-                + @"(?=[^\w/\+\.\-\$,\\])", // post-match
-
-                // SymmetricKey512
-                  @"(?<=[^\r\n\t\w/\+\.\-\$,\\])" // pre-match
-                + @"(?<SymmetricKey>[a-z0-9/\+]{86}==)" // match
-                + @"(?=([^\w/\+\.\-\$]|$))", // post-match
-
-                // AzureActiveDirectoryLoginCredentials
-                  @"(?<=@([a-z0-9.]+\.(on)?)?microsoft\.com[ -~\s\u200b]{1,80}?(userpass|password|pwd|pw|\wpass[ =:>]+|(get|make)securestring)\W)" // pre-match
-                + @"(?<Password>[^\s;`,""'\(\)]{10,80})" // match
-                + @"(?=[\s;`,""'\(\)])", // post-match
-
-                // AzureActiveDirectoryLoginCredentials
-                  @"" // pre-match
-                + @"(?<MigrationPassword>(\-destinationPasswordPlain ""[^""]+?""))" // match
-                + @"(?=[ -~\s\u200b]{1,150}?@([a-z0-9.]+\.(on)?)?microsoft\.com)", // post-match
-
-                // AzureActiveDirectoryLoginCredentials
-                  @"(?<=(sign_in|SharePointOnlineAuthenticatedContext|(User|Exchange)Credentials?|password)[ -~\s\u200b]{1,100}?@([a-z0-9.]+\.(on)?)?microsoft\.com['""]?( \|\| \w+)?\s*,[\s\u200b]['""]?)" // pre-match
-                + @"(?<ArgumentPassword>[^`'""\s,;\(\)]+?)" // match
-                + @"(?=[`'""\s,;\(\)])", // post-match
-
-                // AzureActiveDirectoryLoginCredentials
-                  @"" // pre-match
-                + @"(?<PrevPassword>password[\W_][ -~\s\u200b]{40,100}?)" // match
-                + @"(?=@([a-z0-9\.]+\.(on)?)?microsoft\.com)", // post-match
-
-                // LoginCredentials
-                  @"" // pre-match
-                + @"[^a-z\$](DB_USER|user id|uid|(sql)?user(name)?|service\s?account)\s*[^\w\s,]([ -~\s\u200b]{2,120}?|[ -~]{2,30}?)([^a-z\s\$]|\s)\s*(DB_PASS|(sql|service)?password|pwd)\s*[^a-z,\+&\)\]\}\[\{_][ -~\s\u200b]{2,700}?([;|<,})]|$)|[^a-z\s\$]\s*(DB_PASS|password|pwd)\s*[^a-z,\+&\)\]\}\[\{_][ -~\s\u200b]{2,60}?[^a-z\$](DB_USER|user id|uid|user(name)?)\s*[^\w\s,]([ -~\s\u200b]{2,60}?|[ -~]{2,30}?)([;|<,})]|$)" // match
-                + @"", // post-match
-
-                // LoginCredentialsInUrl
-                  @"(?<=(amqp|ssh|(ht|f)tps?)://[^%:\s""'/][^:\s""'/\$]+[^:\s""'/\$%]:)" // pre-match
-                + @"(?<Password>[^%\s""'/][^@\s""'/]{0,100}[^%\s""'/])" // match
-                + @"(?=@[\$a-z0-9:\.\-_%\?=/]+)", // post-match
-
-                // CertificatePrivateKeyHeader
-                  @"" // pre-match
-                + @"(?-i)\-{5}BEGIN( ([DR]SA|EC|OPENSSH|PGP))? PRIVATE KEY( BLOCK)?\-{5}" // match
-                + @"", // post-match
-
-                // HttpAuthorizationHeader
-                  @"(?<=authorization[,\[:= ""']+(basic|digest|hoba|mutual|negotiate|oauth( oauth_token=)?|bearer [^e""'&]|scram\-sha\-1|scram\-sha\-256|vapid|aws4\-hmac\-sha256)[\s\r\n]{0,10})" // pre-match
-                + @"(?<Token>[a-z0-9/+_.=]{10,})" // match
-                + @"", // post-match
-
-                // ClientSecretContext
-                  @"(?<=(^|[a-z\s""'_])((app(lication)?|client)[_\- ]?(key(url)?|secret)|refresh[_\-]?token|[^t]AccessToken(Secret)?|(Consumer|api)[_\- ]?(Secret|Key)|(Twilio(Account|Auth)[_\- ]?(Sid|Token)))([\s=:>]{1,10}|[\s""':=|>,]{3,15}|[""'=:\(]{2}))" // pre-match
-                + @"(?<ClientSecret>(""data:text/plain,.+""|[a-z0-9/+=_.-]{10,200}[^\(\[\{;,\r\n]|[^\s""';<,\)]{5,200}))" // match
-                + @"", // post-match
-
-                // CommunityStringContext
-                  @"(?<=(^|\W{2}|set )snmp(\-server)?( | [ -~]+? )(community|priv)\s[""']?)" // pre-match
-                + @"(?<CommunityString>[^\s]+)" // match
-                + @"(?=[""']?(\s|$))", // post-match
-
-                // PasswordContextInScript
-                  @"(?<=\s-(admin|user|vm)?password\s+[""']?)" // pre-match
-                + @"(?<ScriptArgumentPassword>[^$\(\[<\{\-\s,""']+)[""']?(\s|$)" // match
-                + @"", // post-match
-
-                // PasswordContextInScript
-                  @"(?<=certutil(\.exe)?.{1,10}\-p\s+[""']?)" // pre-match
-                + @"(?<CertUtilPassword>[^\s,]{2,50})" // match
-                + @"(?=[""']?)", // post-match
-
-                // PasswordContextInScript
-                  @"(?<=(^|[_\s\$])[a-z]*(password|secret(key)?)[ \t]*[=:]+[ \t]*)" // pre-match
-                + @"(?<ScriptAssignmentPassword>[^:\s""';,<]{2,200})" // match
-                + @"", // post-match
-
-                // PasswordContextInScript
-                  @"(?<=\s-Name\s+[""']\w+Password[""']\s+-Value\s+[""']?)" // pre-match
-                + @"(?<RegistryPassword>[^\s""']{2,1100})" // match
-                + @"(?=[""']?)", // post-match
-
-                // PasswordContextInScript
-                  @"(?<=(^|[\s\r\n\\])net(\.exe)?[""'\s\\]{1,5}(user\s+|share\s+/user:)[^\s,/]+[ \t]+[""']?)" // pre-match
-                + @"(?<NetUsePassword>[^\s,""'>/]{2,50})" // match
-                + @"(?=[""']?)", // post-match
-
-                // PasswordContextInScript
-                  @"(?<=psexec(\.exe)?.{1,50}-u.{1,50}-p\s+[""']?)" // pre-match
-                + @"(?<PsExecPassword>[^\s,]{2,50})" // match
-                + @"(?=[""']?)", // post-match
-
-                // SymmetricKeyContextInXml
-                  @"" // pre-match
-                + @"<(machineKey|parameter name=""|[a-z]+AccountInfo[^a-z])" // match
-                + @"", // post-match
-
+                  
+                  // NuGet API keys.
+                  @"\b" // pre-match
+                + @"oy2[a-p][0-9a-z]{15}[aq][0-9a-z]{11}[eu][bdfhjlnprtvxz357][a-p][0-9a-z]{11}[aeimquy4]" // match
+                + @"\b", // post-match
+                  
+                  // NPM author keys.
+                  @"\b" // pre-match
+                + @"npm_[0-9A-Za-z]{36}" // match
+                + @"\b", // post-match
             };
     }
 }
diff --git a/src/Microsoft.VisualStudio.Services.Agent/Constants.cs b/src/Microsoft.VisualStudio.Services.Agent/Constants.cs
@@ -322,6 +322,7 @@ public static class Agent
                 public static readonly string Version = "agent.version";
                 public static readonly string WorkFolder = "agent.workfolder";
                 public static readonly string WorkingDirectory = "agent.WorkingDirectory";
+                public static readonly string EnableAdditionalMaskingRegexes = "agent.enableadditionalmaskingregexes";
             }
 
             public static class Build
@@ -371,6 +372,7 @@ public static class Features
                 public static readonly string GitLfsSupport = "agent.source.git.lfs";
                 public static readonly string GitShallowDepth = "agent.source.git.shallowFetchDepth";
                 public static readonly string SkipSyncSource = "agent.source.skip";
+                public static readonly string EnableAdditionalMaskingRegexes = "agent.enableadditionalmaskingregexes";
             }
 
             public static class Maintenance
@@ -511,6 +513,7 @@ public static class Task
                 Agent.Version,
                 Agent.WorkFolder,
                 Agent.WorkingDirectory,
+                Agent.EnableAdditionalMaskingRegexes,
                 // Build variables
                 Build.ArtifactStagingDirectory,
                 Build.BinariesDirectory,
diff --git a/src/Microsoft.VisualStudio.Services.Agent/HostContext.cs b/src/Microsoft.VisualStudio.Services.Agent/HostContext.cs
@@ -21,6 +21,7 @@
 using System.Net.Http.Headers;
 using Pipelines = Microsoft.TeamFoundation.DistributedTask.Pipelines;
 using Agent.Sdk.Util;
+using BuildXL.Cache.ContentStore.Interfaces.Tracing;
 
 namespace Microsoft.VisualStudio.Services.Agent
 {
@@ -88,6 +89,7 @@ public class HostContext : EventListener, IObserver<DiagnosticListener>, IObserv
         public ShutdownReason AgentShutdownReason { get; private set; }
         public ILoggedSecretMasker SecretMasker => _secretMasker;
         public ProductInfoHeaderValue UserAgent => _userAgent;
+
         public HostContext(HostType hostType, string logFile = null)
         {
             _secretMasker = new LoggedSecretMasker(_basicSecretMasker);
@@ -106,13 +108,6 @@ public HostContext(HostType hostType, string logFile = null)
             this.SecretMasker.AddValueEncoder(ValueEncoders.UriDataEscape, $"HostContext_{WellKnownSecretAliases.UriDataEscape}");
             this.SecretMasker.AddValueEncoder(ValueEncoders.BackslashEscape, $"HostContext_{WellKnownSecretAliases.UriDataEscape}");
             this.SecretMasker.AddRegex(AdditionalMaskingRegexes.UrlSecretPattern, $"HostContext_{WellKnownSecretAliases.UrlSecretPattern}");
-            if (AgentKnobs.MaskUsingCredScanRegexes.GetValue(this).AsBoolean())
-            {
-                foreach (var pattern in AdditionalMaskingRegexes.CredScanPatterns)
-                {
-                    this.SecretMasker.AddRegex(pattern, $"HostContext_{WellKnownSecretAliases.CredScanPatterns}");
-                }
-            }
 
             // Create the trace manager.
             if (string.IsNullOrEmpty(logFile))
@@ -741,6 +736,15 @@ public static HttpClientHandler CreateHttpClientHandler(this IHostContext contex
 
             return clientHandler;
         }
+
+        public static void AddAdditionalMaskingRegexes(this IHostContext context)
+        {
+            ArgUtil.NotNull(context, nameof(context));
+            foreach (var pattern in AdditionalMaskingRegexes.CredScanPatterns)
+            {
+                context.SecretMasker.AddRegex(pattern, $"HostContext_{WellKnownSecretAliases.CredScanPatterns}");
+            }
+        }
     }
 
     public enum ShutdownReason
diff --git a/src/Test/L0/HostContextL0.cs b/src/Test/L0/HostContextL0.cs
diff --git a/src/Test/L0/Listener/JobDispatcherL0.cs b/src/Test/L0/Listener/JobDispatcherL0.cs
diff --git a/src/agentversion b/src/agentversion

Original file line number	Diff line number	Diff line change
`@@ -49,7 +49,7 @@ public async Task<FeatureFlag> GetFeatureFlagAsync(IHostContext context, string`
`49`	`49`	`var client = vssConnection.GetClient<FeatureAvailabilityHttpClient>();`
`50`	`50`	`try`
`51`	`51`	`{`
`52`		`- return await client.GetFeatureFlagByNameAsync(featureFlagName);`
	`52`	`+ return await client.GetFeatureFlagByNameAsync(featureFlagName, checkFeatureExists: false);`
`53`	`53`	`} catch(VssServiceException e)`
`54`	`54`	`{`
`55`	`55`	`Trace.Warning("Unable to retrive feature flag status: " + e.ToString());`