Merge remote-tracking branch 'upstream/rc/1.25' into merge-rc-1.25

Author: hvitved

Diff for: change-notes/1.25/analysis-java.md

@@ -4,20 +4,26 @@ The following changes in version 1.25 affect Java analysis in all applications.
 
 ## General improvements
 
-## New queries
-
-| **Query**                   | **Tags**  | **Purpose**                                                        |
-|-----------------------------|-----------|--------------------------------------------------------------------|
-
+The Java autobuilder has been improved to detect more Gradle Java versions.
 
 ## Changes to existing queries
 
 | **Query**                    | **Expected impact**    | **Change**                        |
 |------------------------------|------------------------|-----------------------------------|
-
+| Hard-coded credential in API call (`java/hardcoded-credential-api-call`) | More results | The query now recognizes the `BasicAWSCredentials` class of the Amazon client SDK library with hardcoded access key/secret key. |
+| Deserialization of user-controlled data (`java/unsafe-deserialization`) | Fewer false positive results | The query no longer reports results using `org.apache.commons.io.serialization.ValidatingObjectInputStream`. |
+| Use of a broken or risky cryptographic algorithm (`java/weak-cryptographic-algorithm`) | More results | The query now recognizes the `MessageDigest.getInstance` method. |
+| Use of a potentially broken or risky cryptographic algorithm (`java/potentially-weak-cryptographic-algorithm`) | More results | The query now recognizes the `MessageDigest.getInstance` method. |
+| Reading from a world writable file (`java/world-writable-file-read`) | More results | The query now recognizes more JDK file operations. |
 
 ## Changes to libraries
 
+* The data-flow library has been improved with more taint flow modeling for the
+  Collections framework and other classes of the JDK. This affects all security
+  queries using data flow and can yield additional results.
+* The data-flow library has been improved with more taint flow modeling for the
+  Spring framework. This affects all security queries using data flow and can
+  yield additional results on project that rely on the Spring framework.
 * The data-flow library has been improved, which affects most security queries by potentially
   adding more results. Flow through methods now takes nested field reads/writes into account.
   For example, the library is able to track flow from `"taint"` to `sink()` via the method
@@ -39,3 +45,5 @@ The following changes in version 1.25 affect Java analysis in all applications.
     }
   }
   ```
+* The library has been extended with more support for Java 14 features
+  (`switch` expressions and pattern-matching for `instanceof`).

Diff for: change-notes/1.25/analysis-python.md

@@ -1,22 +1,9 @@
 # Improvements to Python analysis
 
-The following changes in version 1.25 affect Python analysis in all applications.
-
-## General improvements
-
-
-## New queries
-
-| **Query**                   | **Tags**  | **Purpose**                                                        |
-|-----------------------------|-----------|--------------------------------------------------------------------|
-
-
-## Changes to existing queries
-
-| **Query**                  | **Expected impact**    | **Change**                                                       |
-|----------------------------|------------------------|------------------------------------------------------------------|
-
-
-## Changes to libraries
-
 * Importing `semmle.python.web.HttpRequest` will no longer import `UntrustedStringKind` transitively. `UntrustedStringKind` is the most commonly used non-abstract subclass of `ExternalStringKind`. If not imported (by one mean or another), taint-tracking queries that concern `ExternalStringKind` will not produce any results. Please ensure such queries contain an explicit import (`import semmle.python.security.strings.Untrusted`).
+* Added model of taint sources for HTTP servers using `http.server`.
+* Added taint modeling of routed parameters in Flask.
+* Improved modeling of built-in methods on strings for taint tracking.
+* Improved classification of test files.
+* New class `BoundMethodValue` represents a bound method during runtime.
+* The query `py/command-line-injection` now recognizes command execution with the `fabric` and `invoke` Python libraries.

Diff for: csharp/extractor/Semmle.Extraction.CIL/Context.cs

@@ -43,7 +43,7 @@ public Context(Extraction.Context cx, string assemblyPath, bool extractPdbs)
             namespaceFactory = new CachedFunction<StringHandle, Entities.Namespace>(n => CreateNamespace(mdReader.GetString(n)));
             namespaceDefinitionFactory = new CachedFunction<NamespaceDefinitionHandle, Entities.Namespace>(CreateNamespace);
             sourceFiles = new CachedFunction<PDB.ISourceFile, Entities.PdbSourceFile>(path => new Entities.PdbSourceFile(this, path));
-            folders = new CachedFunction<string, Entities.Folder>(path => new Entities.Folder(this, path));
+            folders = new CachedFunction<PathTransformer.ITransformedPath, Entities.Folder>(path => new Entities.Folder(this, path));
             sourceLocations = new CachedFunction<PDB.Location, Entities.PdbSourceLocation>(location => new Entities.PdbSourceLocation(this, location));
 
             defaultGenericContext = new EmptyContext(this);

Diff for: csharp/extractor/Semmle.Extraction.CIL/Entities/Assembly.cs

@@ -5,6 +5,7 @@
 using System;
 using Semmle.Extraction.Entities;
 using System.IO;
+using Semmle.Util;
 
 namespace Semmle.Extraction.CIL.Entities
 {
@@ -134,9 +135,12 @@ public static void ExtractCIL(Layout layout, string assemblyPath, ILogger logger
             extracted = false;
             try
             {
-                var extractor = new Extractor(false, assemblyPath, logger);
-                var project = layout.LookupProjectOrDefault(assemblyPath);
-                using (var trapWriter = project.CreateTrapWriter(logger, assemblyPath + ".cil", true, trapCompression))
+                var canonicalPathCache = CanonicalPathCache.Create(logger, 1000);
+                var pathTransformer = new PathTransformer(canonicalPathCache);
+                var extractor = new Extractor(false, assemblyPath, logger, pathTransformer);
+                var transformedAssemblyPath = pathTransformer.Transform(assemblyPath);
+                var project = layout.LookupProjectOrDefault(transformedAssemblyPath);
+                using (var trapWriter = project.CreateTrapWriter(logger, transformedAssemblyPath.WithSuffix(".cil"), true, trapCompression))
                 {
                     trapFile = trapWriter.TrapFile;
                     if (nocache || !System.IO.File.Exists(trapFile))

Diff for: csharp/extractor/Semmle.Extraction.CIL/Entities/File.cs

@@ -1,4 +1,4 @@
-using System.Collections.Generic;
+using System.Collections.Generic;
 using System.IO;
 
 namespace Semmle.Extraction.CIL.Entities
@@ -13,37 +13,38 @@ interface IFile : IFileOrFolder
 
     public class File : LabelledEntity, IFile
     {
-        protected readonly string path;
+        protected readonly string OriginalPath;
+        protected readonly PathTransformer.ITransformedPath TransformedPath;
 
         public File(Context cx, string path) : base(cx)
         {
-            this.path = Semmle.Extraction.Entities.File.PathAsDatabaseString(path);
+            this.OriginalPath = path;
+            TransformedPath = cx.cx.Extractor.PathTransformer.Transform(OriginalPath);
         }
 
         public override void WriteId(TextWriter trapFile)
         {
-            trapFile.Write(Semmle.Extraction.Entities.File.PathAsDatabaseId(path));
+            trapFile.Write(TransformedPath.DatabaseId);
         }
 
         public override bool Equals(object? obj)
         {
-            return GetType() == obj?.GetType() && path == ((File)obj).path;
+            return GetType() == obj?.GetType() && OriginalPath == ((File)obj).OriginalPath;
         }
 
-        public override int GetHashCode() => 11 * path.GetHashCode();
+        public override int GetHashCode() => 11 * OriginalPath.GetHashCode();
 
         public override IEnumerable<IExtractionProduct> Contents
         {
             get
             {
-                var directoryName = System.IO.Path.GetDirectoryName(path);
-                if (directoryName is null)
-                    throw new InternalError($"Directory name for path '{path}' is null.");
-
-                var parent = cx.CreateFolder(directoryName);
-                yield return parent;
-                yield return Tuples.containerparent(parent, this);
-                yield return Tuples.files(this, path, System.IO.Path.GetFileNameWithoutExtension(path), System.IO.Path.GetExtension(path).Substring(1));
+                if (TransformedPath.ParentDirectory is PathTransformer.ITransformedPath dir)
+                {
+                    var parent = cx.CreateFolder(dir);
+                    yield return parent;
+                    yield return Tuples.containerparent(parent, this);
+                }
+                yield return Tuples.files(this, TransformedPath.Value, TransformedPath.NameWithoutExtension, TransformedPath.Extension);
             }
         }
 
@@ -69,9 +70,9 @@ public override IEnumerable<IExtractionProduct> Contents
                 var text = file.Contents;
 
                 if (text == null)
-                    cx.cx.Extractor.Logger.Log(Util.Logging.Severity.Warning, string.Format("PDB source file {0} could not be found", path));
+                    cx.cx.Extractor.Logger.Log(Util.Logging.Severity.Warning, string.Format("PDB source file {0} could not be found", OriginalPath));
                 else
-                    cx.cx.TrapWriter.Archive(path, text);
+                    cx.cx.TrapWriter.Archive(TransformedPath, text);
 
                 yield return Tuples.file_extraction_mode(this, 2);
             }

Diff for: csharp/extractor/Semmle.Extraction.CIL/Entities/Folder.cs

@@ -9,16 +9,16 @@ interface IFolder : IFileOrFolder
 
     public sealed class Folder : LabelledEntity, IFolder
     {
-        readonly string path;
+        readonly PathTransformer.ITransformedPath TransformedPath;
 
-        public Folder(Context cx, string path) : base(cx)
+        public Folder(Context cx, PathTransformer.ITransformedPath path) : base(cx)
         {
-            this.path = path;
+            this.TransformedPath = path;
         }
 
         public override void WriteId(TextWriter trapFile)
         {
-            trapFile.Write(Semmle.Extraction.Entities.File.PathAsDatabaseId(path));
+            trapFile.Write(TransformedPath.DatabaseId);
         }
 
         public override string IdSuffix => ";folder";
@@ -27,25 +27,21 @@ public override IEnumerable<IExtractionProduct> Contents
         {
             get
             {
-                // On Posix, we could get a Windows directory of the form "C:"
-                bool windowsDriveLetter = path.Length == 2 && char.IsLetter(path[0]) && path[1] == ':';
-
-                var parent = Path.GetDirectoryName(path);
-                if (parent != null && !windowsDriveLetter)
+                if (TransformedPath.ParentDirectory is PathTransformer.ITransformedPath parent)
                 {
                     var parentFolder = cx.CreateFolder(parent);
                     yield return parentFolder;
                     yield return Tuples.containerparent(parentFolder, this);
                 }
-                yield return Tuples.folders(this, Semmle.Extraction.Entities.File.PathAsDatabaseString(path), Path.GetFileName(path));
+                yield return Tuples.folders(this, TransformedPath.Value, TransformedPath.NameWithoutExtension);
             }
         }
 
         public override bool Equals(object? obj)
         {
-            return obj is Folder folder && path == folder.path;
+            return obj is Folder folder && TransformedPath == folder.TransformedPath;
         }
 
-        public override int GetHashCode() => path.GetHashCode();
+        public override int GetHashCode() => TransformedPath.GetHashCode();
     }
 }

Diff for: csharp/extractor/Semmle.Extraction.CIL/Factories.cs

@@ -201,7 +201,7 @@ Namespace CreateNamespace(NamespaceDefinitionHandle handle)
 
         #region Locations
         readonly CachedFunction<PDB.ISourceFile, PdbSourceFile> sourceFiles;
-        readonly CachedFunction<string, Folder> folders;
+        readonly CachedFunction<PathTransformer.ITransformedPath, Folder> folders;
         readonly CachedFunction<PDB.Location, PdbSourceLocation> sourceLocations;
 
         /// <summary>
@@ -216,7 +216,7 @@ Namespace CreateNamespace(NamespaceDefinitionHandle handle)
         /// </summary>
         /// <param name="path">The path of the folder.</param>
         /// <returns>A folder entity.</returns>
-        public Folder CreateFolder(string path) => folders[path];
+        public Folder CreateFolder(PathTransformer.ITransformedPath path) => folders[path];
 
         /// <summary>
         /// Creates a source location.

Diff for: csharp/extractor/Semmle.Extraction.CSharp/Analyser.cs

@@ -27,13 +27,16 @@ public class Analyser : IDisposable
 
         public readonly bool AddAssemblyTrapPrefix;
 
-        public Analyser(IProgressMonitor pm, ILogger logger, bool addAssemblyTrapPrefix)
+        public readonly PathTransformer PathTransformer;
+
+        public Analyser(IProgressMonitor pm, ILogger logger, bool addAssemblyTrapPrefix, PathTransformer pathTransformer)
         {
             Logger = logger;
             AddAssemblyTrapPrefix = addAssemblyTrapPrefix;
             Logger.Log(Severity.Info, "EXTRACTION STARTING at {0}", DateTime.Now);
             stopWatch.Start();
             progressMonitor = pm;
+            PathTransformer = pathTransformer;
         }
 
         CSharpCompilation compilation;
@@ -67,7 +70,7 @@ public void EndInitialize(
             layout = new Layout();
             this.options = options;
             this.compilation = compilation;
-            extractor = new Extraction.Extractor(false, GetOutputName(compilation, commandLineArguments), Logger);
+            extractor = new Extraction.Extractor(false, GetOutputName(compilation, commandLineArguments), Logger, PathTransformer);
             LogDiagnostics();
 
             SetReferencePaths();
@@ -117,7 +120,7 @@ public void InitializeStandalone(CSharpCompilation compilationIn, CommonOptions
         {
             compilation = compilationIn;
             layout = new Layout();
-            extractor = new Extraction.Extractor(true, null, Logger);
+            extractor = new Extraction.Extractor(true, null, Logger, PathTransformer);
             this.options = options;
             LogExtractorInfo(Extraction.Extractor.Version);
             SetReferencePaths();
@@ -230,9 +233,10 @@ void DoAnalyseCompilation(string cwd, string[] args)
             try
             {
                 var assemblyPath = extractor.OutputPath;
+                var transformedAssemblyPath = PathTransformer.Transform(assemblyPath);
                 var assembly = compilation.Assembly;
-                var projectLayout = layout.LookupProjectOrDefault(assemblyPath);
-                var trapWriter = projectLayout.CreateTrapWriter(Logger, assemblyPath, true, options.TrapCompression);
+                var projectLayout = layout.LookupProjectOrDefault(transformedAssemblyPath);
+                var trapWriter = projectLayout.CreateTrapWriter(Logger, transformedAssemblyPath, true, options.TrapCompression);
                 compilationTrapFile = trapWriter;  // Dispose later
                 var cx = extractor.CreateContext(compilation.Clone(), trapWriter, new AssemblyScope(assembly, assemblyPath, true), AddAssemblyTrapPrefix);
 
@@ -260,8 +264,9 @@ void DoAnalyseAssembly(PortableExecutableReference r)
                 stopwatch.Start();
 
                 var assemblyPath = r.FilePath;
-                var projectLayout = layout.LookupProjectOrDefault(assemblyPath);
-                using (var trapWriter = projectLayout.CreateTrapWriter(Logger, assemblyPath, true, options.TrapCompression))
+                var transformedAssemblyPath = PathTransformer.Transform(assemblyPath);
+                var projectLayout = layout.LookupProjectOrDefault(transformedAssemblyPath);
+                using (var trapWriter = projectLayout.CreateTrapWriter(Logger, transformedAssemblyPath, true, options.TrapCompression))
                 {
                     var skipExtraction = options.Cache && File.Exists(trapWriter.TrapFile);
 
@@ -360,16 +365,17 @@ void DoExtractTree(SyntaxTree tree)
                 var stopwatch = new Stopwatch();
                 stopwatch.Start();
                 var sourcePath = tree.FilePath;
+                var transformedSourcePath = PathTransformer.Transform(sourcePath);
 
-                var projectLayout = layout.LookupProjectOrNull(sourcePath);
+                var projectLayout = layout.LookupProjectOrNull(transformedSourcePath);
                 bool excluded = projectLayout == null;
-                string trapPath = excluded ? "" : projectLayout.GetTrapPath(Logger, sourcePath, options.TrapCompression);
+                string trapPath = excluded ? "" : projectLayout.GetTrapPath(Logger, transformedSourcePath, options.TrapCompression);
                 bool upToDate = false;
 
                 if (!excluded)
                 {
                     // compilation.Clone() is used to allow symbols to be garbage collected.
-                    using (var trapWriter = projectLayout.CreateTrapWriter(Logger, sourcePath, false, options.TrapCompression))
+                    using (var trapWriter = projectLayout.CreateTrapWriter(Logger, transformedSourcePath, false, options.TrapCompression))
                     {
                         upToDate = options.Fast && FileIsUpToDate(sourcePath, trapWriter.TrapFile);
 

Diff for: csharp/extractor/Semmle.Extraction.CSharp/Entities/Compilation.cs

@@ -3,6 +3,7 @@
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
+using Semmle.Util;
 
 namespace Semmle.Extraction.CSharp.Entities
 {
@@ -22,7 +23,7 @@ protected override void Populate(TextWriter trapFile)
         {
             Extraction.Entities.Assembly.CreateOutputAssembly(cx);
 
-            trapFile.compilations(this, Extraction.Entities.File.PathAsDatabaseString(cwd));
+            trapFile.compilations(this, FileUtils.ConvertToUnix(cwd));
 
             // Arguments
             int index = 0;

Diff for: csharp/extractor/Semmle.Extraction.CSharp/Extractor.cs

@@ -76,16 +76,16 @@ public static ExitCode Run(string[] args)
                 return ExitCode.Ok;
             }
 
-            using (var analyser = new Analyser(new LogProgressMonitor(logger), logger, commandLineArguments.AssemblySensitiveTrap))
+            var canonicalPathCache = CanonicalPathCache.Create(logger, 1000);
+            var pathTransformer = new PathTransformer(canonicalPathCache);
+
+            using (var analyser = new Analyser(new LogProgressMonitor(logger), logger, commandLineArguments.AssemblySensitiveTrap, pathTransformer))
             using (var references = new BlockingCollection<MetadataReference>())
             {
                 try
                 {
                     var compilerVersion = new CompilerVersion(commandLineArguments);
 
-                    bool preserveSymlinks = Environment.GetEnvironmentVariable("SEMMLE_PRESERVE_SYMLINKS") == "true";
-                    var canonicalPathCache = CanonicalPathCache.Create(logger, 1000, preserveSymlinks ? CanonicalPathCache.Symlinks.Preserve : CanonicalPathCache.Symlinks.Follow);
-
                     if (compilerVersion.SkipExtraction)
                     {
                         logger.Log(Severity.Warning, "  Unrecognized compiler '{0}' because {1}", compilerVersion.SpecifiedCompiler, compilerVersion.SkipReason);
@@ -318,7 +318,10 @@ public static void ExtractStandalone(
             ILogger logger,
             CommonOptions options)
         {
-            using (var analyser = new Analyser(pm, logger, false))
+            var canonicalPathCache = CanonicalPathCache.Create(logger, 1000);
+            var pathTransformer = new PathTransformer(canonicalPathCache);
+
+            using (var analyser = new Analyser(pm, logger, false, pathTransformer))
             using (var references = new BlockingCollection<MetadataReference>())
             {
                 try