Skip to content

Commit 0c9f83b

Browse files
grendellojonpryor
grendello
authored and
jonpryor
committed
[build] Remove Mono submodule (dotnet#3371)
Context: https://github.com/xamarin/xamarin-android/projects/10 One of the goals behind the Mono SDK archive was to enable product builds without having the entire Mono source tree present, which means it should be possible to remove the Mono git submodule and build the entire xamarin-android repo without any issues. At the same time, we want to keep the ability to build Mono from source, if necessary, for situations when a developer works on the BCL or the runtime itself and there's no appropriate binary archive present. To support these two requirements: * Remove the Mono submodule * Put Mono repo commit reference into the `.external` file * Make distinction between "commercial" and "regular" bits in the `.external` file so that we don't attempt to check out everything mentioned in there in situations when a developer doesn't have access to the commercial bits. * Use Mono linker sources from the archive * Remove `zlib-helper.c` from the `libmonodroid` build - the API implemented in there is no longer used by Mono. * `Xamarin.Android.Cecil.dll` and `Xamarin.Android.Cecil.Mdb.dll` are no longer built from source. Instead we use Cecil to reshape `Mono.Cecil` and conjure the two renamed and re-signed assemblies. * All the code which used `Xamarin.Android.Cecil` project reference now simply references the conjured assemblies (which are also added to the bundle) * Third Party Notices are generated from licenses found in the Mono archive. * Windows build downloads also the Darwin archive - in order to be able to use Mono Linker sources as well as licenses contained in this archive.
1 parent 694617e commit 0c9f83b

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

43 files changed

+636
-240
lines changed

‎.external

+1
Original file line numberDiff line numberDiff line change
@@ -1 +1,2 @@
11
xamarin/monodroid:master@237e0bd9f105b9842778ef82161a20c6d4497a40
2+
mono/mono:2019-06@afcf28a3660b069721e5c441dfe5bb7548e3204f

‎.gitignore

-1
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@
33
bin
44
Configuration.Override.props
55
Configuration.OperatingSystem.props
6-
msfinal.pub
76
obj
87
packages
98
.DS_Store

‎.gitmodules

-4
Original file line numberDiff line numberDiff line change
@@ -22,10 +22,6 @@
2222
path = external/mman-win32
2323
url = https://github.com/witwall/mman-win32.git
2424
branch = master
25-
[submodule "external/mono"]
26-
path = external/mono
27-
url = https://github.com/mono/mono.git
28-
branch = 2019-06
2925
[submodule "external/mxe"]
3026
path = external/mxe
3127
url = https://github.com/xamarin/mxe.git

‎Configuration.props

+1-1
Original file line numberDiff line numberDiff line change
@@ -88,7 +88,7 @@
8888
<MonoRequiredMaximumVersion Condition=" '$(MonoRequiredMaximumVersion)' == '' ">6.5.0</MonoRequiredMaximumVersion>
8989
<IgnoreMaxMonoVersion Condition=" '$(IgnoreMaxMonoVersion)' == '' ">True</IgnoreMaxMonoVersion>
9090
<MonoRequiredDarwinMinimumVersion>$(MonoRequiredMinimumVersion).98</MonoRequiredDarwinMinimumVersion>
91-
<LinkerSourceDirectory>$(MSBuildThisFileDirectory)external\mono\external\linker\src</LinkerSourceDirectory>
91+
<LinkerSourceDirectory>$(MSBuildThisFileDirectory)external\mono\sdks\out\android-sources\external\linker\src</LinkerSourceDirectory>
9292
<OpenTKSourceDirectory>$(MSBuildThisFileDirectory)external\opentk</OpenTKSourceDirectory>
9393
<MingwZlibRootDirectory Condition=" '$(ZlibRootDirectory)' == '' And '$(HostOS)' == 'Linux' ">\usr</MingwZlibRootDirectory>
9494
<MingwZlibRootDirectory Condition=" '$(ZlibRootDirectory)' == '' And '$(HostOS)' == 'Darwin' ">$(HostHomebrewPrefix)\opt\mingw-zlib\usr</MingwZlibRootDirectory>

‎build-tools/conjure-xamarin-android-cecil/conjure-xamarin-android-cecil.cs

+63
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
using System;
2+
using System.Collections.Generic;
3+
using System.IO;
4+
using System.Linq;
5+
6+
using Mono.Cecil;
7+
8+
public class ConjureXamarinAndroidCecil
9+
{
10+
const string BaseNameReplacement = "Xamarin.Android.Cecil";
11+
const string CecilAssemblyName = BaseNameReplacement;
12+
const string CecilMdbAssemblyName = BaseNameReplacement + ".Mdb";
13+
14+
static readonly List<string> internalsVisibleTo = new List<string> {
15+
"Xamarin.Android.Cecil.Pdb, PublicKey=0024000004800000940000000602000000240000525341310004000011000000438ac2a5acfbf16cbd2b2b47a62762f273df9cb2795ceccdf77d10bf508e69e7a362ea7a45455bbf3ac955e1f2e2814f144e5d817efc4c6502cc012df310783348304e3ae38573c6d658c234025821fda87a0be8a0d504df564e2c93b2b878925f42503e9d54dfef9f9586d9e6f38a305769587b1de01f6c0410328b2c9733db",
16+
"Xamarin.Android.Cecil.Mdb, PublicKey=0024000004800000940000000602000000240000525341310004000011000000438ac2a5acfbf16cbd2b2b47a62762f273df9cb2795ceccdf77d10bf508e69e7a362ea7a45455bbf3ac955e1f2e2814f144e5d817efc4c6502cc012df310783348304e3ae38573c6d658c234025821fda87a0be8a0d504df564e2c93b2b878925f42503e9d54dfef9f9586d9e6f38a305769587b1de01f6c0410328b2c9733db"
17+
};
18+
19+
public static int Main (string[] args)
20+
{
21+
if (args.Length < 2) {
22+
Console.WriteLine ("Usage: <input directory> <output directory>");
23+
Console.WriteLine (" <input directory> must have Mono.Cecil.dll and Mono.Cecil.Mdb.dll assemblies");
24+
return 1;
25+
}
26+
27+
string inputDir = args [0];
28+
string inputFilePath = Path.Combine (inputDir, "Mono.Cecil.dll");
29+
string outputDirPath = args [1];
30+
31+
var resolver = new DefaultAssemblyResolver ();
32+
resolver.AddSearchDirectory (Path.GetDirectoryName (inputFilePath));
33+
var rp = new ReaderParameters () {
34+
AssemblyResolver = resolver,
35+
ReadSymbols = true
36+
};
37+
var monoCecil = AssemblyDefinition.ReadAssembly (inputFilePath, rp);
38+
monoCecil.Name.Name = CecilAssemblyName;
39+
40+
var ivtCtor = monoCecil.MainModule.ImportReference (typeof (System.Runtime.CompilerServices.InternalsVisibleToAttribute).GetConstructor (new []{typeof(string)}));
41+
foreach (string ivtParam in internalsVisibleTo) {
42+
var ca = new CustomAttribute (ivtCtor);
43+
ca.ConstructorArguments.Add (new CustomAttributeArgument (monoCecil.MainModule.TypeSystem.String, ivtParam));
44+
monoCecil.CustomAttributes.Add (ca);
45+
}
46+
47+
var wp = new WriterParameters {
48+
WriteSymbols = true
49+
};
50+
51+
monoCecil.Write (Path.Combine (outputDirPath, $"{CecilAssemblyName}.dll"), wp);
52+
53+
inputFilePath = Path.Combine (inputDir, "Mono.Cecil.Mdb.dll");
54+
var monoCecilMdb = AssemblyDefinition.ReadAssembly (inputFilePath, rp);
55+
monoCecilMdb.Name.Name = CecilMdbAssemblyName;
56+
57+
AssemblyNameReference monoCecilRef = monoCecilMdb.MainModule.AssemblyReferences.Single (r => String.Compare ("Mono.Cecil", r.Name, StringComparison.Ordinal) == 0);
58+
monoCecilRef.Name = CecilAssemblyName;
59+
monoCecilMdb.Write (Path.Combine (outputDirPath, $"{CecilMdbAssemblyName}.dll"), wp);
60+
61+
return 0;
62+
}
63+
}

‎build-tools/conjure-xamarin-android-cecil/conjure-xamarin-android-cecil.csproj

+38
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
<?xml version="1.0" encoding="utf-8"?>
2+
<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
3+
<PropertyGroup>
4+
<Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
5+
<Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
6+
<ProjectGuid>{C876DA71-8573-4CEF-9149-716D72455ED4}</ProjectGuid>
7+
<OutputType>Exe</OutputType>
8+
<RootNamespace>Xamarin.Android.Prepare</RootNamespace>
9+
<AssemblyName>conjure-xamarin-android-cecil</AssemblyName>
10+
</PropertyGroup>
11+
<Import Project="..\..\Configuration.props" />
12+
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
13+
<DebugSymbols>true</DebugSymbols>
14+
<Optimize>false</Optimize>
15+
<OutputPath>..\..\bin\BuildDebug</OutputPath>
16+
<DefineConstants>DEBUG;</DefineConstants>
17+
<ErrorReport>prompt</ErrorReport>
18+
<WarningLevel>4</WarningLevel>
19+
<ExternalConsole>true</ExternalConsole>
20+
</PropertyGroup>
21+
<PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
22+
<Optimize>true</Optimize>
23+
<OutputPath>..\..\bin\BuildRelease</OutputPath>
24+
<ErrorReport>prompt</ErrorReport>
25+
<WarningLevel>4</WarningLevel>
26+
<ExternalConsole>true</ExternalConsole>
27+
</PropertyGroup>
28+
<ItemGroup>
29+
<Reference Include="System" />
30+
<Reference Include="Mono.Cecil">
31+
<HintPath>$(XamarinAndroidSourcePath)external\mono\sdks\out\android-bcl\monodroid_tools\Mono.Cecil.dll</HintPath>
32+
</Reference>
33+
</ItemGroup>
34+
<ItemGroup>
35+
<Compile Include="conjure-xamarin-android-cecil.cs" />
36+
</ItemGroup>
37+
<Import Project="$(MSBuildBinPath)\Microsoft.CSharp.targets" />
38+
</Project>

‎build-tools/remap-assembly-ref/remap-assembly-ref.csproj

+4-4
Original file line numberDiff line numberDiff line change
@@ -27,10 +27,10 @@
2727
</PropertyGroup>
2828
<ItemGroup>
2929
<Reference Include="System" />
30-
<ProjectReference Include="..\..\external\Java.Interop\src\Xamarin.Android.Cecil\Xamarin.Android.Cecil.csproj">
31-
<Project>{15945D4B-FF56-4BCC-B598-2718D199DD08}</Project>
32-
<Name>Xamarin.Android.Cecil</Name>
33-
</ProjectReference> </ItemGroup>
30+
<Reference Include="Mono.Cecil">
31+
<HintPath>$(XamarinAndroidSourcePath)external\mono\sdks\out\android-bcl\monodroid_tools\Mono.Cecil.dll</HintPath>
32+
</Reference>
33+
</ItemGroup>
3434
<ItemGroup>
3535
<Compile Include="remap-assembly-ref.cs" />
3636
</ItemGroup>

‎build-tools/scripts/XAVersionInfo.targets

+1-3
Original file line numberDiff line numberDiff line change
@@ -6,15 +6,13 @@
66
<UsingTask AssemblyFile="$(MSBuildThisFileDirectory)..\..\bin\Build$(Configuration)\xa-prep-tasks.dll" TaskName="Xamarin.Android.BuildTools.PrepTasks.GitCommitsInRange" />
77
<UsingTask AssemblyFile="$(MSBuildThisFileDirectory)..\..\bin\Build$(Configuration)\xa-prep-tasks.dll" TaskName="Xamarin.Android.BuildTools.PrepTasks.GitCommitTime" />
88
<UsingTask AssemblyFile="$(MSBuildThisFileDirectory)..\..\bin\Build$(Configuration)\xa-prep-tasks.dll" TaskName="Xamarin.Android.BuildTools.PrepTasks.GitCommitInfo" />
9+
<Import Project="..\..\bin\Build$(Configuration)\MonoGitHash.props" />
910
<Target Name="_GetSubmodulesVersionInfo">
1011
<ItemGroup>
1112
<!-- If anything changes in this list, make sure to update:
1213
src/Xamarin.Android.Build.Tasks/Xamarin.Android.Build.Tasks.targets (the _GenerateXACommonProps target)
1314
build-tools/create-vsix/create-vsix.targets (the _CreateDependencies target)
1415
-->
15-
<_SubmoduleBranchInfo Include="external/mono">
16-
<OutputPropertyName>_BuildInfo_MonoCommit</OutputPropertyName>
17-
</_SubmoduleBranchInfo>
1816
<_SubmoduleBranchInfo Include="external/Java.Interop">
1917
<OutputPropertyName>_BuildInfo_JavaInteropCommit</OutputPropertyName>
2018
</_SubmoduleBranchInfo>

‎build-tools/xaprepare/xaprepare/Application/BuildInfo.cs

+7-1
Original file line numberDiff line numberDiff line change
@@ -105,7 +105,13 @@ void DetermineBundleHashes (Context context)
105105
LibZipHash = EnsureHash ("LibZip", Utilities.ShortenGitHash (FullLibZipHash));
106106

107107
Log.StatusLine ($" {context.Characters.Bullet} Mono commit hash", ConsoleColor.Gray);
108-
FullMonoHash = git.GetTopCommitHash (context.Properties.GetRequiredValue (KnownProperties.MonoSourceFullPath), shortHash: false);
108+
List<ExternalGitDependency> externalDependencies = ExternalGitDependency.GetDependencies (context, Configurables.Paths.ExternalGitDepsFilePath, quiet: true);
109+
ExternalGitDependency mono = externalDependencies?.Where (
110+
eg => eg != null &&
111+
String.Compare ("mono", eg.Owner, StringComparison.Ordinal) == 0 &&
112+
String.Compare ("mono", eg.Name, StringComparison.Ordinal) == 0).FirstOrDefault ();
113+
114+
FullMonoHash = mono?.Commit?.Trim ();
109115
MonoHash = EnsureHash ("Mono", Utilities.ShortenGitHash (FullMonoHash));
110116

111117
if (Configurables.Paths.BundleVersionHashFiles == null || Configurables.Paths.BundleVersionHashFiles.Count == 0) {

‎build-tools/xaprepare/xaprepare/Application/Context.cs

+11-2
Original file line numberDiff line numberDiff line change
@@ -329,6 +329,13 @@ public string DebugFileExtension {
329329
/// </summary>
330330
public RefreshableComponent ComponentsToRefresh { get; set; }
331331

332+
/// <summary>
333+
/// Set by <see cref="Step_DownloadMonoArchive"/> if the archive has been downloaded and validated, so
334+
/// that the <see cref="Step_BuildMonoRuntimes"/> step doesn't have to re-download the archive, should
335+
/// bundle be absent and <see cref="Step_PrepareBundle"/> fails.
336+
/// </summary>
337+
public bool MonoAlreadyBuilt { get; set; }
338+
332339
static Context ()
333340
{
334341
Instance = new Context ();
@@ -742,8 +749,10 @@ public async Task<bool> Init (string scenarioName = null)
742749
Banner ("Updating Git submodules");
743750

744751
var git = new GitRunner (this);
745-
if (!await git.SubmoduleUpdate ())
746-
Log.WarningLine ("Failed to update Git submodules");
752+
if (!await git.SubmoduleUpdate ()) {
753+
Log.ErrorLine ("Failed to update Git submodules");
754+
return false;
755+
}
747756

748757
BuildInfo = new BuildInfo ();
749758
await BuildInfo.GatherGitInfo (this);

‎build-tools/xaprepare/xaprepare/Application/ExternalGitDependency.cs

+15-5
Original file line numberDiff line numberDiff line change
@@ -33,11 +33,13 @@ sealed class ExternalGitDependency : AppObject
3333
public string Name { get; private set; }
3434
public string Owner { get; private set; }
3535

36-
public static List<ExternalGitDependency> GetDependencies (Context context, string externalFilePath)
36+
public static List<ExternalGitDependency> GetDependencies (Context context, string externalFilePath, bool quiet = false)
3737
{
38-
Log.Instance.StatusLine ($" {context.Characters.Bullet} Reading external dependencies from {Utilities.GetRelativePath (BuildPaths.XamarinAndroidSourceRoot, externalFilePath)}");
38+
if (!quiet)
39+
Log.Instance.StatusLine ($" {context.Characters.Bullet} Reading external dependencies from {Utilities.GetRelativePath (BuildPaths.XamarinAndroidSourceRoot, externalFilePath)}");
3940
string[] unparsedExternals = File.ReadAllLines (externalFilePath);
4041
var externals = new List<ExternalGitDependency> (unparsedExternals.Length);
42+
bool includeCommercial = context.CheckCondition (KnownConditions.IncludeCommercial);
4143

4244
foreach (string external in unparsedExternals) {
4345
Match match = externalRegex.Match (external);
@@ -47,14 +49,22 @@ public static List<ExternalGitDependency> GetDependencies (Context context, stri
4749
continue;
4850
}
4951

52+
string owner = match.Groups["owner"].Value;
53+
string repo = match.Groups["repo"].Value;
54+
if (!includeCommercial && Configurables.Defaults.CommercialExternalDependencies.Contains ($"{owner}/{repo}")) {
55+
Log.Instance.DebugLine ($"Ignoring external commercial dependency '{owner}/{repo}'");
56+
continue;
57+
}
58+
5059
var e = new ExternalGitDependency {
5160
Branch = match.Groups["branch"].Value,
5261
Commit = match.Groups["commit"].Value,
53-
Name = match.Groups["repo"].Value,
54-
Owner = match.Groups["owner"].Value,
62+
Name = repo,
63+
Owner = owner,
5564
};
5665
externals.Add (e);
57-
Log.Instance.StatusLine ($" {context.Characters.Bullet} {e.Owner}/{e.Name} ({e.Commit})");
66+
if (!quiet)
67+
Log.Instance.StatusLine ($" {context.Characters.Bullet} {e.Owner}/{e.Name} ({e.Commit})");
5868
}
5969
}
6070

‎build-tools/xaprepare/xaprepare/Application/KnownConditions.cs

+6
Original file line numberDiff line numberDiff line change
@@ -18,5 +18,11 @@ public enum KnownConditions
1818
/// only a single program (e.g. the UpdateMono scenario) but not the rest. Default: unset. <see cref="Scenario_UpdateMono" />
1919
/// </summary>
2020
IgnoreMissingPrograms,
21+
22+
/// <summary>
23+
/// If set, will cause checkout of the commercial dependencies mentioned in the `.external` file at
24+
/// the top of the repository. Default: unset
25+
/// </summary>
26+
IncludeCommercial,
2127
}
2228
}

‎build-tools/xaprepare/xaprepare/Application/Utilities.cs

+78
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,84 @@ static partial class Utilities
2828

2929
public static readonly Encoding UTF8NoBOM = new UTF8Encoding (false);
3030

31+
public static bool AbiChoiceChanged (Context context)
32+
{
33+
string cacheFile = Configurables.Paths.MonoRuntimesEnabledAbisCachePath;
34+
if (!File.Exists (cacheFile)) {
35+
Log.DebugLine ($"Enabled ABI cache file not found at {cacheFile}");
36+
return true;
37+
}
38+
39+
var oldAbis = new HashSet<string> (StringComparer.Ordinal);
40+
foreach (string l in File.ReadAllLines (cacheFile)) {
41+
string line = l?.Trim ();
42+
if (String.IsNullOrEmpty (line) || oldAbis.Contains (line))
43+
continue;
44+
oldAbis.Add (line);
45+
}
46+
47+
HashSet<string> currentAbis = null;
48+
FillCurrentAbis (context, ref currentAbis);
49+
50+
if (oldAbis.Count != currentAbis.Count)
51+
return true;
52+
53+
foreach (string abi in oldAbis) {
54+
if (!currentAbis.Contains (abi))
55+
return true;
56+
}
57+
58+
return false;
59+
}
60+
61+
public static void SaveAbiChoice (Context context)
62+
{
63+
HashSet<string> currentAbis = null;
64+
FillCurrentAbis (context, ref currentAbis);
65+
66+
string cacheFile = Configurables.Paths.MonoRuntimesEnabledAbisCachePath;
67+
Log.DebugLine ($"Writing ABI cache file {cacheFile}");
68+
File.WriteAllLines (cacheFile, currentAbis);
69+
}
70+
71+
static void FillCurrentAbis (Context context, ref HashSet<string> currentAbis)
72+
{
73+
Utilities.AddAbis (context.Properties.GetRequiredValue (KnownProperties.AndroidSupportedTargetJitAbis).Trim (), ref currentAbis);
74+
Utilities.AddAbis (context.Properties.GetRequiredValue (KnownProperties.AndroidSupportedTargetAotAbis).Trim (), ref currentAbis);
75+
Utilities.AddAbis (context.Properties.GetRequiredValue (KnownProperties.AndroidSupportedHostJitAbis).Trim (), ref currentAbis);
76+
}
77+
78+
public static void PropagateXamarinAndroidCecil (Context context)
79+
{
80+
const string CecilAssembly = "Xamarin.Android.Cecil.dll";
81+
82+
CopyFile (
83+
Path.Combine (Configurables.Paths.InstallMSBuildDir, CecilAssembly),
84+
Path.Combine (Configurables.Paths.ExternalJavaInteropDir, "bin", context.Configuration, CecilAssembly)
85+
);
86+
}
87+
88+
public static async Task<bool> BuildRemapRef (Context context, bool haveManagedRuntime, string managedRuntime, bool quiet = false)
89+
{
90+
if (!quiet)
91+
Log.StatusLine ("Building remap-assembly-ref");
92+
93+
var msbuild = new MSBuildRunner (context);
94+
string projectPath = Path.Combine (Configurables.Paths.BuildToolsDir, "remap-assembly-ref", "remap-assembly-ref.csproj");
95+
bool result = await msbuild.Run (
96+
projectPath: projectPath,
97+
logTag: "remap-assembly-ref",
98+
binlogName: "build-remap-assembly-ref"
99+
);
100+
101+
if (!result) {
102+
Log.ErrorLine ("Failed to build remap-assembly-ref");
103+
return false;
104+
}
105+
106+
return true;
107+
}
108+
31109
public static string ToXamarinAndroidPropertyValue (ICollection<string> coll)
32110
{
33111
if (coll == null)

‎build-tools/xaprepare/xaprepare/ConfigAndData/Configurables.Unix.cs

+1
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ partial class Defaults
1212
partial class Paths
1313
{
1414
static string BundleOSType => Context.Instance.OS.Type;
15+
static string ArchiveOSType => Context.Instance.OS.Type;
1516

1617
public static string BCLTestsSourceDir => GetCachedPath (ref bclTestsSourceDir, () => Path.Combine (MonoProfileDir, "tests"));
1718
public static string BCLAssembliesSourceDir => MonoProfileDir;

‎build-tools/xaprepare/xaprepare/ConfigAndData/Configurables.Windows.cs

+1
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ partial class Defaults
3434
partial class Paths
3535
{
3636
static string BundleOSType => "Darwin"; // Windows doesn't build the bundle
37+
static string ArchiveOSType => "Darwin"; // Windows need sources from there
3738

3839
// Windows doesn't build the bundle so we need to look in the XA framework dir as installed
3940
public static string BCLAssembliesSourceDir => InstallBCLFrameworkDir;

0 commit comments

Comments
 (0)