Skip to content

Commit

Permalink
[xaprepare] Update ExternalGitDependency usage (#2)
Browse files Browse the repository at this point in the history
Allows a non-authorized https-based clone for local use cases, and cleans
up various logging events to ensure we don't leak secrets. Also includes
some minor azure-pipelines.yaml cleanup.
  • Loading branch information
pjcollins authored and grendello committed Jun 7, 2019
1 parent 5217736 commit 0f65bbd
Show file tree
Hide file tree
Showing 6 changed files with 64 additions and 25 deletions.
17 changes: 8 additions & 9 deletions build-tools/automation/azure-pipelines.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -8,16 +8,16 @@ trigger:
variables:
BundleArtifactName: bundle
AutoProvisionArgs: /p:AutoProvision=True /p:AutoProvisionUsesSudo=True /p:IgnoreMaxMonoVersion=False
AndroidTargetAbiArgs: >-
/p:AndroidSupportedTargetJitAbis=armeabi-v7a:arm64-v8a:x86:x86_64
/p:AndroidSupportedTargetAotAbis=armeabi-v7a:arm64:x86:x86_64:win-armeabi-v7a:win-arm64:win-x86:win-x86_64
# Stage and Job "display names" are shortened because they are combined to form the name of the corresponding GitHub check.
stages:
- stage: prepare
displayName: Prepare
variables:
MSBuildAbiArgs: >-
/p:AndroidSupportedTargetJitAbis=armeabi-v7a:arm64-v8a:x86:x86_64
/p:AndroidSupportedHostJitAbis=Darwin:mxe-Win32:mxe-Win64
/p:AndroidSupportedTargetAotAbis=armeabi-v7a:arm64:x86:x86_64:win-armeabi-v7a:win-arm64:win-x86:win-x86_64
MSBuildAbiArgs: $(AndroidTargetAbiArgs) /p:AndroidSupportedHostJitAbis=Darwin:mxe-Win32:mxe-Win64
jobs:
- job: create_bundle
displayName: Bundle
Expand All @@ -35,7 +35,7 @@ stages:
- script: |
make prepare-update-mono PREPARE_CI=1 V=1 CONFIGURATION=$(XA.Build.Configuration) MSBUILD_ARGS="$(AutoProvisionArgs) $(MSBuildAbiArgs)"
make prepare PREPARE_CI=1 V=1 CONFIGURATION=$(XA.Build.Configuration) MSBUILD_ARGS="$(AutoProvisionArgs) $(MSBuildAbiArgs)"
displayName: make prepare-props
displayName: create bundle
- task: CopyFiles@2
displayName: copy bundle
Expand Down Expand Up @@ -149,7 +149,6 @@ stages:
cancelTimeoutInMinutes: 5
variables:
JAVA_HOME: '%HOMEDRIVE%%HOMEPATH%\android-toolchain\jdk'
XA_MSBUILD_ARGS: "/p:AndroidSupportedTargetJitAbis=armeabi-v7a:arm64-v8a:x86:x86_64 /p:AndroidSupportedTargetAotAbis=armeabi-v7a:win-armeabi-v7a:arm64:win-arm64:x86:win-x86:x86_64:win-x86_64"
steps:
- checkout: self
clean: true
Expand All @@ -175,20 +174,20 @@ stages:
inputs:
solution: Xamarin.Android.sln
configuration: $(XA.Build.Configuration)
msbuildArguments: /t:Build /bl:$(System.DefaultWorkingDirectory)\bin\Build$(XA.Build.Configuration)\msbuild-build.binlog /p:BundleRootPath=$(System.DefaultWorkingDirectory) $(XA_MSBUILD_ARGS)
msbuildArguments: /t:Build /bl:$(System.DefaultWorkingDirectory)\bin\Build$(XA.Build.Configuration)\msbuild-build.binlog /p:BundleRootPath=$(System.DefaultWorkingDirectory) $(AndroidTargetAbiArgs)

- task: MSBuild@1
displayName: msbuild create-vsix
inputs:
solution: build-tools\create-vsix\create-vsix.csproj
configuration: $(XA.Build.Configuration)
msbuildArguments: /p:CreateVsixContainer=True /p:ZipPackageCompressionLevel=Normal /bl:$(System.DefaultWorkingDirectory)\bin\Build$(XA.Build.Configuration)\msbuild-create-vsix.binlog $(XA_MSBUILD_ARGS)
msbuildArguments: /p:CreateVsixContainer=True /p:ZipPackageCompressionLevel=Normal /bl:$(System.DefaultWorkingDirectory)\bin\Build$(XA.Build.Configuration)\msbuild-create-vsix.binlog $(AndroidTargetAbiArgs)

- task: CmdLine@1
displayName: xabuild Xamarin.Android-Tests
inputs:
filename: bin\$(XA.Build.Configuration)\bin\xabuild.exe
arguments: Xamarin.Android-Tests.sln /p:Configuration=$(XA.Build.Configuration) /p:XAIntegratedTests=False /bl:$(System.DefaultWorkingDirectory)\bin\Test$(XA.Build.Configuration)\msbuild-build-tests.binlog $(XA_MSBUILD_ARGS)
arguments: Xamarin.Android-Tests.sln /p:Configuration=$(XA.Build.Configuration) /p:XAIntegratedTests=False /bl:$(System.DefaultWorkingDirectory)\bin\Test$(XA.Build.Configuration)\msbuild-build-tests.binlog $(AndroidTargetAbiArgs)

- task: MSBuild@1
displayName: nunit Xamarin.Android.Build.Tests
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,6 @@ sealed class ExternalGitDependency : AppObject
public string Commit { get; private set; }
public string Name { get; private set; }
public string Owner { get; private set; }
public string Url { get; private set; }

public static List<ExternalGitDependency> GetDependencies (Context context, string externalFilePath)
{
Expand All @@ -54,22 +53,12 @@ public static List<ExternalGitDependency> GetDependencies (Context context, stri
Name = match.Groups["repo"].Value,
Owner = match.Groups["owner"].Value,
};
SetGitHubURL (e);
externals.Add (e);
Log.Instance.StatusLine ($" {context.Characters.Bullet} {e.Owner}/{e.Name} ({e.Commit})");
}
}

return externals;
}

static void SetGitHubURL (ExternalGitDependency egd)
{
string ghToken = Environment.GetEnvironmentVariable ("GH_AUTH_SECRET");
if (!String.IsNullOrEmpty (ghToken))
egd.Url = $"https://{ghToken}@github.com:/{egd.Owner}/{egd.Name}";
else
egd.Url = $"[email protected]:/{egd.Owner}/{egd.Name}";
}
}
}
4 changes: 3 additions & 1 deletion build-tools/xaprepare/xaprepare/Application/ProcessRunner.cs
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,7 @@ public string FullCommandLine {
public Dictionary <string, string> Environment { get; } = new Dictionary <string, string> (StringComparer.Ordinal);
public int ExitCode { get; private set; } = -1;
public ErrorReasonCode ErrorReason { get; private set; } = ErrorReasonCode.NotExecutedYet;
public bool EchoCmdAndArguments { get; set; } = true;
public bool EchoStandardOutput { get; set; }
public ProcessStandardStreamWrapper.LogLevel EchoStandardOutputLevel { get; set; } = ProcessStandardStreamWrapper.LogLevel.Message;
public bool EchoStandardError { get; set; }
Expand Down Expand Up @@ -232,7 +233,8 @@ public bool Run ()
StartInfo = psi
};

Log.DebugLine ($"Running: {FullCommandLine}");
if (EchoCmdAndArguments)
Log.DebugLine ($"Running: {FullCommandLine}");

try {
process.Start ();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,20 +18,23 @@ protected override async Task<bool> Execute (Context context)
bool failed = false;
Log.StatusLine ();
Log.StatusLine ("Updating external repositories");
var git = new GitRunner (context);
var git = new GitRunner (context) {
EchoCmdAndArguments = false
};
foreach (ExternalGitDependency egd in externalDependencies) {
Log.StatusLine ($" {context.Characters.Bullet} {egd.Name}");
string destDir = Path.Combine (Configurables.Paths.ExternalGitDepsDestDir, egd.Name);
if (!Directory.Exists (destDir)) {
Log.StatusLine ($" {context.Characters.Link} cloning from {egd.Url}");
if (!await git.Clone (egd.Url, destDir)) {
var egdUrl = await GetGitHubURL (egd, git);
Log.StatusLine ($" {context.Characters.Link} cloning from {egd.Owner}/{egd.Name}");
if (!await git.Clone (egdUrl, destDir)) {
Log.ErrorLine ($"Failed to clone {egd.Name}");
failed = true;
continue;
}
}

Log.StatusLine ($" {context.Characters.Link} fetching changes from {egd.Url}");
Log.StatusLine ($" {context.Characters.Link} fetching changes from {egd.Owner}/{egd.Name}");
if (!await git.Fetch (destDir)) {
Log.ErrorLine ($"Failed to fetch changes for {egd.Name}");
failed = true;
Expand Down Expand Up @@ -65,5 +68,19 @@ protected override async Task<bool> Execute (Context context)

return !failed;
}

async Task<string> GetGitHubURL (ExternalGitDependency egd, GitRunner git)
{
string ghToken = Environment.GetEnvironmentVariable("GH_AUTH_SECRET");
if (!String.IsNullOrEmpty (ghToken)) {
return $"https://{ghToken}@github.com:/{egd.Owner}/{egd.Name}";
} else {
if (await git.IsRepoUrlHttps (BuildPaths.XamarinAndroidSourceRoot)) {
return $"https://github.com:/{egd.Owner}/{egd.Name}";
} else {
return $"[email protected]:/{egd.Owner}/{egd.Name}";
}
}
}
}
}
30 changes: 30 additions & 0 deletions build-tools/xaprepare/xaprepare/ToolRunners/GitRunner.cs
Original file line number Diff line number Diff line change
Expand Up @@ -160,6 +160,36 @@ public async Task<IList<BlamePorcelainEntry>> Blame (string filePath, List<strin
return parserState.Entries;
}

public async Task<bool> IsRepoUrlHttps (string workingDirectory)
{
if (!Directory.Exists (workingDirectory))
throw new ArgumentException ("must exist", nameof (workingDirectory));

var runner = CreateGitRunner (workingDirectory);
runner.AddArgument ("config");
runner.AddArgument ("--get");
runner.AddArgument ("remote.origin.url");

bool containsHttps = false;

bool success = await RunTool (
() => {
using (var outputSink = (OutputSink)SetupOutputSink (runner)) {
outputSink.LineCallback = (string line) => {
containsHttps = !string.IsNullOrEmpty (line) && line.Contains ("https://");
};
runner.WorkingDirectory = DetermineRunnerWorkingDirectory (workingDirectory);
return runner.Run ();
}
}
);

if (!success)
return false;

return containsHttps;
}

ProcessRunner CreateGitRunner (string workingDirectory, List<string> arguments = null)
{
var runner = CreateProcessRunner ();
Expand Down
2 changes: 2 additions & 0 deletions build-tools/xaprepare/xaprepare/ToolRunners/ToolRunner.cs
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ abstract partial class ToolRunner : AppObject
public string FullToolPath { get; }
protected string VersionString => GetVersion ();

public bool EchoCmdAndArguments { get; set; } = true;
public bool EchoStandardError { get; set; } = true;
public bool EchoStandardOutput { get; set; }
public string LogMessageIndent { get; set; } = String.Empty;
Expand Down Expand Up @@ -88,6 +89,7 @@ protected virtual ProcessRunner CreateProcessRunner (params string[] initialPara

var runner = new ProcessRunner (managedRunner ?? FullToolPath, initialParams) {
ProcessTimeout = ProcessTimeout,
EchoCmdAndArguments = EchoCmdAndArguments,
EchoStandardError = EchoStandardError,
EchoStandardOutput = EchoStandardOutput,
};
Expand Down

0 comments on commit 0f65bbd

Please sign in to comment.