From b54f43f1025dabf1d897c4d0483e593a5eebbd79 Mon Sep 17 00:00:00 2001 From: K Widholm <279278+apotek@users.noreply.github.com> Date: Wed, 30 Oct 2024 01:30:43 -0400 Subject: [PATCH 1/9] Allow aws commands to use aws environment variables for auth Do not assume credential file creation will have valid credentials. Begin to move away from deprecated ->io constructs. Resolves #222 --- phpstan.neon | 2 +- .../Commands/DevelopmentModeCommands.php | 107 +++++++++-------- src/Robo/Plugin/Commands/ThemeCommands.php | 2 +- .../Plugin/Traits/DatabaseDownloadTrait.php | 113 ++++++++++++------ src/Robo/Plugin/Traits/SitesConfigTrait.php | 33 ++--- 5 files changed, 154 insertions(+), 103 deletions(-) diff --git a/phpstan.neon b/phpstan.neon index 3a7a0b1..db94037 100644 --- a/phpstan.neon +++ b/phpstan.neon @@ -1,7 +1,7 @@ parameters: - checkMissingIterableValueType: false customRulesetUsed: true ignoreErrors: + - identifier: missingType.iterableValue - message: '#\Call to deprecated method io\(\) of class Robo\\Tasks#' paths: - 'src/' diff --git a/src/Robo/Plugin/Commands/DevelopmentModeCommands.php b/src/Robo/Plugin/Commands/DevelopmentModeCommands.php index 4c329bb..3397854 100644 --- a/src/Robo/Plugin/Commands/DevelopmentModeCommands.php +++ b/src/Robo/Plugin/Commands/DevelopmentModeCommands.php @@ -6,6 +6,7 @@ use Robo\Exception\TaskException; use Robo\Result; use Robo\ResultData; +use Robo\Symfony\ConsoleIO; use Robo\Tasks; use Symfony\Component\Yaml\Yaml; use Usher\Robo\Plugin\Enums\LocalDevEnvironmentTypes; @@ -76,11 +77,13 @@ public function __construct() * @aliases magic */ public function devRefresh( + ConsoleIO $io, string $siteName = 'default', array $options = ['db' => '', 'environment-type' => 'ddev'], ): Result { ['db' => $dbPath, 'environment-type' => $environmentType] = $options; return $this->devRefreshDrupal( + io: $io, environmentType: LocalDevEnvironmentTypes::from($environmentType), siteName: $siteName, databasePath: $dbPath, @@ -103,6 +106,7 @@ public function devRefresh( * Specify alternative (supported) environment type. See LocalDevEnvironmentTypes enum. */ public function devRefreshAll( + ConsoleIO $io, array $options = ['skip-sites' => '', 'environment-type' => 'ddev'] ): Result { ['skip-sites' => $skipSites, 'environment-type' => $environmentType] = $options; @@ -113,6 +117,7 @@ public function devRefreshAll( continue; } $result = $this->devRefreshDrupal( + $io, environmentType: LocalDevEnvironmentTypes::from($environmentType), siteName: $siteName, ); @@ -128,27 +133,32 @@ public function devRefreshAll( * @option db * Provide a path to a database dump to be used instead of downloading the latest dump. */ - public function databaseRefreshDdev(string $siteName = 'default', array $options = ['db' => '']): Result - { - // @todo: Update this method to not be DDEV specific. - $this->io()->title('DDEV database refresh.'); + public function databaseRefreshDdev( + ConsoleIO $io, + string $siteName = 'default', + array $options = ['db' => ''] + ): Result|ResultData { + $io->title('DDEV database refresh.'); ['db' => $dbPath] = $options; // Track whether a database path was provided by the user or not. $dbPathProvidedByUser = $dbPath !== ''; if (!$dbPathProvidedByUser) { - $dbPath = $this->databaseDownload($siteName); + $dbPath = $this->databaseDownload($io, $siteName); + if ($dbPath instanceof ResultData) { + return $dbPath; + } } - $this->io()->section("refreshing $siteName database."); - $this->say("Dropping existing database for $siteName"); + $io->section("refreshing $siteName database."); + $io->say("Dropping existing database for $siteName"); $this->taskExec('drush') ->arg('sql:drop') ->option('uri', $siteName) ->option('yes') ->run(); - $this->say("Importing $dbPath"); + $io->say("Importing $dbPath"); $this->_exec("zcat '$dbPath' | drush sql:cli --uri=$siteName"); // If a database was downloaded as part of this process, delete it. if (!$dbPathProvidedByUser) { @@ -156,6 +166,7 @@ public function databaseRefreshDdev(string $siteName = 'default', array $options } return $this->drushDeployWith( + io: $io, localEnvironmentType: LocalDevEnvironmentTypes::DDEV, siteDir: $siteName, ); @@ -164,23 +175,23 @@ public function databaseRefreshDdev(string $siteName = 'default', array $options /** * Refresh database on Tugboat. */ - public function databaseRefreshTugboat(): ResultData + public function databaseRefreshTugboat(ConsoleIO $io): ResultData { - $this->io()->title('refresh tugboat databases.'); + $io->title('refresh tugboat databases.'); $resultData = new ResultData(); foreach (array_keys($this->getAllSitesConfig()) as $siteName) { $dbPath = ''; try { - $dbPath = $this->databaseDownload($siteName); + $dbPath = $this->databaseDownload($io, $siteName); } catch (TaskException $e) { - $this->yell("$siteName: No database configured. Download/import skipped."); + $io->yell("$siteName: No database configured. Download/import skipped."); $resultData->append($e->getMessage()); // @todo: Should we run a site-install by default? continue; } if (!is_string($dbPath) || $dbPath === '') { - $this->yell("'$siteName' database path not found."); + $io->yell("'$siteName' database path not found."); $resultData->append("'$siteName' database path not found."); continue; } @@ -198,7 +209,7 @@ public function databaseRefreshTugboat(): ResultData ->option('-e', "drop database if exists $dbName; create database $dbName;") ->run(); $resultData->append($taskResult); - $this->io()->section("import $siteName database."); + $io->section("import $siteName database."); $taskResult = $this->taskExec("zcat $dbPath | $dbDriver -h mariadb -u tugboat -ptugboat $dbName") ->run(); $resultData->append($taskResult); @@ -216,16 +227,17 @@ public function databaseRefreshTugboat(): ResultData * @aliases uli * * @param string $environmentType - * Specify local development enviroment: ddev. This value is a string instead of LocalDevEnvironmentTypes since + * Specify local development environment: ddev. This value is a string instead of LocalDevEnvironmentTypes since * it is a public command that can be called from the command line. * @param string $siteDir * The Drupal site directory name. */ public function drupalLoginLink( + ConsoleIO $io, string $environmentType, string $siteDir = 'default', ): Result { - $this->io()->section("create login link."); + $io->section("create login link."); $uid = $this->getDrupalSiteAdminUid(siteName: $siteDir); if ($environmentType === 'ddev') { return $this->taskExec('drush') @@ -254,22 +266,22 @@ public function drupalLoginLink( * * @option boolean $yes Default answers to yes. * @aliases fedd - * - * @return \Robo\Result - * The result of the set of tasks. */ - public function frontendDevDisable(string $siteDir = 'default', array $opts = ['yes|y' => false]) - { + public function frontendDevDisable( + ConsoleIO $io, + string $siteDir = 'default', + array $opts = ['yes|y' => false] + ): Result|ResultData { $devSettingsPath = "$this->drupalRoot/sites/$siteDir/settings.local.php"; if (!$opts['yes']) { - $this->yell("This command will overwrite any customizations you have made to $devSettingsPath and + $io->yell("This command will overwrite any customizations you have made to $devSettingsPath and $this->devServicesPath."); - $yes = $this->io()->confirm('This command is destructive. Do you wish to continue?'); + $yes = $io->confirm('This command is destructive. Do you wish to continue?'); if (!$yes) { return Result::cancelled(); } } - $this->io()->title('disabling front-end development mode.'); + $io->title('disabling front-end development mode.'); return $this->collectionBuilder() ->taskFilesystemStack() ->remove($devSettingsPath) @@ -281,23 +293,22 @@ public function frontendDevDisable(string $siteDir = 'default', array $opts = [' * Refreshes a development environment based upon the Drupal version. */ protected function devRefreshDrupal( + ConsoleIO $io, LocalDevEnvironmentTypes $environmentType, string $siteName = 'default', string $databasePath = '', ): Result { - $this->io()->title('development environment refresh. 🦄✨'); - $result = $this->taskComposerInstall()->run(); - + $io->title('development environment refresh. 🦄✨'); + $this->taskComposerInstall()->run(); // There isn't a great way to call a command in one class from another. // https://github.com/consolidation/Robo/issues/743 // For now, it seems like calling robo from within robo works. - $result = $this->taskExec("composer robo theme:build $siteName") + $this->taskExec("composer robo theme:build $siteName") ->run(); - $result = $this->frontendDevEnable($siteName, ['yes' => true]); + $this->frontendDevEnable($io, $siteName, ['yes' => true]); + $this->databaseRefreshDdev($io, siteName: $siteName, options: ['db' => $databasePath]); - $result = $this->databaseRefreshDdev(siteName: $siteName, options: ['db' => $databasePath]); - - return $this->drupalLoginLink($environmentType->value, $siteName); + return $this->drupalLoginLink($io, $environmentType->value, $siteName); } /** @@ -306,10 +317,11 @@ protected function devRefreshDrupal( * @see https://www.drush.org/deploycommand */ protected function drushDeployWith( + ConsoleIO $io, LocalDevEnvironmentTypes $localEnvironmentType, string $siteDir = 'default', ): Result { - $this->io()->section('drush deploy.'); + $io->section('drush deploy.'); if (!class_exists(\Drush\Commands\core\DeployCommands::class)) { throw new TaskException( $this, @@ -341,30 +353,30 @@ protected function drushDeployWith( * * @option boolean $yes Default answers to yes. * @aliases fede - * - * @return \Robo\Result - * The result of the set of tasks. */ - public function frontendDevEnable(string $siteDir = 'default', array $opts = ['yes|y' => false]) - { + public function frontendDevEnable( + ConsoleIO $io, + string $siteDir = 'default', + array $opts = ['yes|y' => false], + ): Result|ResultData { $devSettingsPath = "$this->drupalRoot/sites/$siteDir/settings.local.php"; if (!$opts['yes']) { - $this->yell("This command will overwrite any customizations you have made to $devSettingsPath and + $io->yell("This command will overwrite any customizations you have made to $devSettingsPath and $this->devServicesPath."); - $yes = $this->io()->confirm('This command is destructive. Do you wish to continue?'); + $yes = $io->confirm('This command is destructive. Do you wish to continue?'); if (!$yes) { return Result::cancelled(); } } - $this->io()->title('enabling front-end development mode.'); - $this->say("copying settings.local.php and development.services.yml into sites/$siteDir."); + $io->title('enabling front-end development mode.'); + $io->say("copying settings.local.php and development.services.yml into sites/$siteDir."); // Copy the example local settings file. $example_local_settings_file = "$this->drupalRoot/sites/example.settings.local.php"; if (file_exists($example_local_settings_file)) { - $result = $this->taskFilesystemStack() + $this->taskFilesystemStack() ->copy($example_local_settings_file, $devSettingsPath) ->run(); } else { @@ -376,7 +388,7 @@ public function frontendDevEnable(string $siteDir = 'default', array $opts = ['y // Copy the development services file. $development_services_file = "$this->drupalRoot/sites/development.services.yml"; if (file_exists($development_services_file)) { - $result = $this->taskFilesystemStack() + $this->taskFilesystemStack() ->copy($development_services_file, $this->devServicesPath, true) ->run(); } else { @@ -386,7 +398,7 @@ public function frontendDevEnable(string $siteDir = 'default', array $opts = ['y ); } - $this->say("optimizing twig for front-end development in development services yml config."); + $io->say("optimizing twig for front-end development in development services yml config."); $devServices = Yaml::parseFile($this->devServicesPath); $devServices['parameters']['twig.config'] = [ 'debug' => true, @@ -394,9 +406,9 @@ public function frontendDevEnable(string $siteDir = 'default', array $opts = ['y 'cache' => false, ]; $this->writeYaml($this->devServicesPath, $devServices); - $this->say("disabling render and dynamic_page_cache in settings.local.php."); + $io->say("disabling render and dynamic_page_cache in settings.local.php."); // https://github.com/consolidation/robo/issues/1059#issuecomment-967732068 - $result = $this->collectionBuilder() + return $this->collectionBuilder() ->taskReplaceInFile($devSettingsPath) ->from('/sites/development.services.yml') ->to("/sites/fe.development.services.yml") @@ -417,6 +429,5 @@ public function frontendDevEnable(string $siteDir = 'default', array $opts = ['y ->line(' */') ->line('$config[\'advagg.settings\'][\'enabled\'] = FALSE;') ->run(); - return $result; } } diff --git a/src/Robo/Plugin/Commands/ThemeCommands.php b/src/Robo/Plugin/Commands/ThemeCommands.php index 407c2a1..5c8291c 100644 --- a/src/Robo/Plugin/Commands/ThemeCommands.php +++ b/src/Robo/Plugin/Commands/ThemeCommands.php @@ -39,7 +39,7 @@ public function themeBuild(string $siteName = 'default'): Result try { $themeBuildConfiguration = $this->getSiteConfigItem('theme_build', $siteName); } catch (TaskException) { - $this->say("'$siteName' theme_build confguration not set."); + $this->say("'$siteName' theme_build configuration not set."); return $this->taskExec('echo skipping')->run(); } foreach ($themeBuildConfiguration as $themeConfig) { diff --git a/src/Robo/Plugin/Traits/DatabaseDownloadTrait.php b/src/Robo/Plugin/Traits/DatabaseDownloadTrait.php index 945b610..b8cbab0 100644 --- a/src/Robo/Plugin/Traits/DatabaseDownloadTrait.php +++ b/src/Robo/Plugin/Traits/DatabaseDownloadTrait.php @@ -2,9 +2,14 @@ namespace Usher\Robo\Plugin\Traits; +use AsyncAws\Core\Sts\StsClient; use AsyncAws\S3\S3Client; +use AsyncAws\S3\ValueObject\AwsObject; +use Robo\Exception\AbortTasksException; use Robo\Exception\TaskException; use Robo\Result; +use Robo\ResultData; +use Robo\Symfony\ConsoleIO; /** * Trait to provide database download functionality to Robo commands. @@ -13,10 +18,8 @@ trait DatabaseDownloadTrait { /** * Default S3 region. - * - * @var string */ - protected $s3DefaultRegion = 'us-east-1'; + protected string $s3DefaultRegion = 'us-east-1'; /** * Download the latest database dump for the site. @@ -26,25 +29,45 @@ trait DatabaseDownloadTrait * * @aliases dbdl * - * @throws \Robo\Exception\TaskException + * @throws \Robo\Exception\TaskException|\Robo\Exception\AbortTasksException */ - public function databaseDownload(string $siteName = 'default'): string|\Robo\Result + public function databaseDownload(ConsoleIO $io, string $siteName = 'default'): string|ResultData { - $this->io()->title('database download.'); + $io->title('database download.'); - $awsConfigDirPath = getenv('HOME') . '/.aws'; - $awsConfigFilePath = "$awsConfigDirPath/credentials"; - if (!is_dir($awsConfigDirPath) || !file_exists($awsConfigFilePath)) { - $result = $this->configureAwsCredentials($awsConfigDirPath, $awsConfigFilePath); + $region = $this->s3RegionForSite($siteName); + $authenticated = false; + $objects = null; + $s3 = new S3Client(['region' => $region]); + try { + $io->say("Connecting to S3..."); + $objects = $s3->listObjectsV2($this->s3BucketRequestConfig($siteName)); + $objects->resolve(); + $authenticated = $objects->info()['status'] === 200; + } catch (\Exception $e) { + $io->error($e->getMessage()); + } + if (!$authenticated) { + if (getenv('AWS_SECRET_ACCESS_KEY')) { + $io->error([ + 'Cannot authenticate to AWS. Please fix your AWS environment', + 'variables, or remove them completely and try again.', + ]); + return Result::cancelled(); + } + $result = $this->configureAwsCredentials($io); if ($result->wasCancelled()) { return Result::cancelled(); } + $s3 = new S3Client(['region' => $region]); + try { + $objects = $s3->listObjectsV2($this->s3BucketRequestConfig($siteName)); + } catch (\Exception $e) { + $io->error($e->getMessage()); + throw new AbortTasksException('Unable to access AWS S3. Giving up.'); + } } - $s3 = new S3Client([ - 'region' => $this->s3RegionForSite($siteName), - ]); - $objects = $s3->listObjectsV2($this->s3BucketRequestConfig($siteName)); $objects = iterator_to_array($objects); if (count($objects) == 0) { throw new TaskException($this, "No database dumps found for '$siteName'."); @@ -52,8 +75,8 @@ public function databaseDownload(string $siteName = 'default'): string|\Robo\Res // Ensure objects are sorted by last modified date. usort( array: $objects, - /** @var \AsyncAws\S3\ValueObject\AwsObject $a */ - callback: fn($a, $b) => $a->getLastModified()->getTimestamp() <=> $b->getLastModified()->getTimestamp(), + callback: fn(AwsObject $a, AwsObject $b) => + $a->getLastModified()->getTimestamp() <=> $b->getLastModified()->getTimestamp(), ); /** @var \AsyncAws\S3\ValueObject\AwsObject $latestDatabaseDump */ $latestDatabaseDump = array_pop(array: $objects); @@ -78,34 +101,44 @@ public function databaseDownload(string $siteName = 'default'): string|\Robo\Res /** * Configure AWS credentials. - * - * @param string $awsConfigDirPath - * Path to the AWS configuration directory. - * @param string $awsConfigFilePath - * Path to the AWS configuration file. */ - protected function configureAwsCredentials(string $awsConfigDirPath, string $awsConfigFilePath): Result + protected function configureAwsCredentials(ConsoleIO $io): Result|ResultData { - $yes = $this->io()->confirm('AWS S3 credentials not detected. Do you wish to configure them?'); - if (!$yes) { - return Result::cancelled(); - } - - if (!is_dir($awsConfigDirPath)) { - $this->_mkdir($awsConfigDirPath); - } + $awsConfigDirPath = getenv('HOME') . '/.aws'; + $awsConfigFilePath = "$awsConfigDirPath/credentials"; + $authenticated = false; - if (!file_exists($awsConfigFilePath)) { - $this->_touch($awsConfigFilePath); + while ($authenticated === false) { + $yes = $io->confirm('Do you wish to configure your AWS S3 credentials?'); + if (!$yes) { + return Result::cancelled(); + } + if (!is_dir($awsConfigDirPath)) { + $this->_mkdir($awsConfigDirPath); + } + if (!file_exists($awsConfigFilePath)) { + $this->_touch($awsConfigFilePath); + } + $awsKeyId = $io->ask("AWS Access Key ID:"); + $awsSecretKey = $io->askHidden("AWS Secret Access Key:"); + $writeResult = $this->taskWriteToFile($awsConfigFilePath) + ->line('[default]') + ->line("aws_access_key_id = $awsKeyId") + ->line("aws_secret_access_key = $awsSecretKey") + ->run(); + try { + $sts = new StsClient(); + $stsResponse = $sts->getCallerIdentity(); + // Ensure the request is completed. + if ($stsResponse->resolve()) { + $authenticated = $stsResponse->info()['status'] === 200; + } + } catch (\Exception $e) { + $io->error($e->getMessage()); + } } - $awsKeyId = $this->ask("AWS Access Key ID:"); - $awsSecretKey = $this->askHidden("AWS Secret Access Key:"); - return $this->taskWriteToFile($awsConfigFilePath) - ->line('[default]') - ->line("aws_access_key_id = $awsKeyId") - ->line("aws_secret_access_key = $awsSecretKey") - ->run(); + return $writeResult; } /** @@ -113,6 +146,8 @@ protected function configureAwsCredentials(string $awsConfigDirPath, string $aws * * @param string $siteName * The site name. + * + * @throws \Robo\Exception\TaskException */ protected function s3BucketRequestConfig(string $siteName): array { diff --git a/src/Robo/Plugin/Traits/SitesConfigTrait.php b/src/Robo/Plugin/Traits/SitesConfigTrait.php index 098f28b..1c5ee98 100644 --- a/src/Robo/Plugin/Traits/SitesConfigTrait.php +++ b/src/Robo/Plugin/Traits/SitesConfigTrait.php @@ -17,15 +17,14 @@ trait SitesConfigTrait * * @var string */ - protected $sitesConfigFile = '.sites.config.yml'; + protected string $sitesConfigFile = '.sites.config.yml'; /** * Load configuration for all sites. * - * @return mixed[] - * A configuration array for all sites. + * @throws \Robo\Exception\TaskException */ - public function getAllSitesConfig(): array + public function getAllSitesConfig(): mixed { if (!file_exists($this->sitesConfigFile)) { throw new TaskException($this, "$this->sitesConfigFile not found."); @@ -38,14 +37,18 @@ public function getAllSitesConfig(): array * * @return string[] * An array of all site names. + * @throws \Robo\Exception\TaskException */ public function getAllSiteNames(): array { + // @todo Fix assumption that Yaml::parseFile always returns an array. return array_keys($this->getAllSitesConfig()); } /** * Determine how many sites are included in sites config. + * + * @throws \Robo\Exception\TaskException */ public function getSitesCount(): int { @@ -58,18 +61,22 @@ public function getSitesCount(): int * @param string $siteName * The site name. * - * @return mixed[] - * The specified site configuration array. + * @throws \Robo\Exception\TaskException */ - protected function getSiteConfig(string $siteName = 'default'): array + protected function getSiteConfig(string $siteName = 'default'): mixed { $allSitesConfig = $this->getAllSitesConfig(); - if (!is_array($allSitesConfig[$siteName])) { + if ( + !is_array($allSitesConfig) + || !array_key_exists($siteName, $allSitesConfig) + || !is_array($allSitesConfig[$siteName]) + ) { throw new TaskException( $this, - "Configuration for '$siteName' missing or malformed in $this->sitesConfigFile." + "Sites configuration in $this->sitesConfigFile is missing or malformed." ); } + return $allSitesConfig[$siteName]; } @@ -82,6 +89,8 @@ protected function getSiteConfig(string $siteName = 'default'): array * The site name. * @param bool $required * Whether the config item is expected to always be present. + * + * @throws \Robo\Exception\TaskException */ public function getSiteConfigItem(string $key, string $siteName = 'default', bool $required = true): mixed { @@ -110,11 +119,7 @@ protected function writeSitesConfig(array $sitesConfig): void /** * Get the Drupal site admin user ID. * - * @param string $siteName - * The site name. - * - * @return int - * The Drupal admin user ID. + * @throws \Robo\Exception\TaskException */ protected function getDrupalSiteAdminUid(string $siteName = 'default'): int { From d3ed8c23f2dcd22ec2795cb5e39989acd1a9e198 Mon Sep 17 00:00:00 2001 From: K Widholm <279278+apotek@users.noreply.github.com> Date: Wed, 30 Oct 2024 09:39:53 -0400 Subject: [PATCH 2/9] Clarify whether sites config is entirely missing or just the site specific config --- src/Robo/Plugin/Traits/SitesConfigTrait.php | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/Robo/Plugin/Traits/SitesConfigTrait.php b/src/Robo/Plugin/Traits/SitesConfigTrait.php index 1c5ee98..a08973b 100644 --- a/src/Robo/Plugin/Traits/SitesConfigTrait.php +++ b/src/Robo/Plugin/Traits/SitesConfigTrait.php @@ -66,14 +66,19 @@ public function getSitesCount(): int protected function getSiteConfig(string $siteName = 'default'): mixed { $allSitesConfig = $this->getAllSitesConfig(); - if ( - !is_array($allSitesConfig) - || !array_key_exists($siteName, $allSitesConfig) + if (!is_array($allSitesConfig)) { + throw new TaskException( + $this, + "Sites configuration in $this->sitesConfigFile is missing or malformed." + ); + } + else if ( + !array_key_exists($siteName, $allSitesConfig) || !is_array($allSitesConfig[$siteName]) ) { throw new TaskException( $this, - "Sites configuration in $this->sitesConfigFile is missing or malformed." + "Sites configuration for '$siteName' in $this->sitesConfigFile is missing or malformed." ); } From d79a0dc311e32a351f2627539440180289542668 Mon Sep 17 00:00:00 2001 From: K Widholm <279278+apotek@users.noreply.github.com> Date: Wed, 30 Oct 2024 10:00:45 -0400 Subject: [PATCH 3/9] phpcs fixes --- src/Robo/Plugin/Traits/SitesConfigTrait.php | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Robo/Plugin/Traits/SitesConfigTrait.php b/src/Robo/Plugin/Traits/SitesConfigTrait.php index a08973b..0bc852f 100644 --- a/src/Robo/Plugin/Traits/SitesConfigTrait.php +++ b/src/Robo/Plugin/Traits/SitesConfigTrait.php @@ -71,8 +71,7 @@ protected function getSiteConfig(string $siteName = 'default'): mixed $this, "Sites configuration in $this->sitesConfigFile is missing or malformed." ); - } - else if ( + } elseif ( !array_key_exists($siteName, $allSitesConfig) || !is_array($allSitesConfig[$siteName]) ) { From 98c2a2271634d111cf78a7adec4a22f3bc4311c3 Mon Sep 17 00:00:00 2001 From: K Widholm <279278+apotek@users.noreply.github.com> Date: Wed, 30 Oct 2024 11:40:05 -0400 Subject: [PATCH 4/9] Improve flow of event signaling in refresh tasks --- .../Plugin/Commands/DevelopmentModeCommands.php | 15 +++++++++++---- src/Robo/Plugin/Traits/DatabaseDownloadTrait.php | 10 +++++++--- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/src/Robo/Plugin/Commands/DevelopmentModeCommands.php b/src/Robo/Plugin/Commands/DevelopmentModeCommands.php index 3397854..079fba4 100644 --- a/src/Robo/Plugin/Commands/DevelopmentModeCommands.php +++ b/src/Robo/Plugin/Commands/DevelopmentModeCommands.php @@ -80,7 +80,7 @@ public function devRefresh( ConsoleIO $io, string $siteName = 'default', array $options = ['db' => '', 'environment-type' => 'ddev'], - ): Result { + ): Result|ResultData { ['db' => $dbPath, 'environment-type' => $environmentType] = $options; return $this->devRefreshDrupal( io: $io, @@ -108,7 +108,7 @@ public function devRefresh( public function devRefreshAll( ConsoleIO $io, array $options = ['skip-sites' => '', 'environment-type' => 'ddev'] - ): Result { + ): Result|ResultData { ['skip-sites' => $skipSites, 'environment-type' => $environmentType] = $options; $siteNames = $this->getAllSiteNames(); $result = null; @@ -121,6 +121,10 @@ public function devRefreshAll( environmentType: LocalDevEnvironmentTypes::from($environmentType), siteName: $siteName, ); + if ($result instanceof ResultData) { + $io->say("Cancelling the refresh for all sites."); + return $result; + } } return $result; } @@ -297,7 +301,7 @@ protected function devRefreshDrupal( LocalDevEnvironmentTypes $environmentType, string $siteName = 'default', string $databasePath = '', - ): Result { + ): Result|ResultData { $io->title('development environment refresh. 🦄✨'); $this->taskComposerInstall()->run(); // There isn't a great way to call a command in one class from another. @@ -306,7 +310,10 @@ protected function devRefreshDrupal( $this->taskExec("composer robo theme:build $siteName") ->run(); $this->frontendDevEnable($io, $siteName, ['yes' => true]); - $this->databaseRefreshDdev($io, siteName: $siteName, options: ['db' => $databasePath]); + $result = $this->databaseRefreshDdev($io, siteName: $siteName, options: ['db' => $databasePath]); + if ($result instanceof ResultData) { + return $result; + } return $this->drupalLoginLink($io, $environmentType->value, $siteName); } diff --git a/src/Robo/Plugin/Traits/DatabaseDownloadTrait.php b/src/Robo/Plugin/Traits/DatabaseDownloadTrait.php index b8cbab0..7520613 100644 --- a/src/Robo/Plugin/Traits/DatabaseDownloadTrait.php +++ b/src/Robo/Plugin/Traits/DatabaseDownloadTrait.php @@ -2,6 +2,7 @@ namespace Usher\Robo\Plugin\Traits; +use AsyncAws\Core\Exception\Http\ClientException; use AsyncAws\Core\Sts\StsClient; use AsyncAws\S3\S3Client; use AsyncAws\S3\ValueObject\AwsObject; @@ -44,17 +45,20 @@ public function databaseDownload(ConsoleIO $io, string $siteName = 'default'): s $objects = $s3->listObjectsV2($this->s3BucketRequestConfig($siteName)); $objects->resolve(); $authenticated = $objects->info()['status'] === 200; - } catch (\Exception $e) { + } catch (ClientException $e) { $io->error($e->getMessage()); } if (!$authenticated) { if (getenv('AWS_SECRET_ACCESS_KEY')) { $io->error([ - 'Cannot authenticate to AWS. Please fix your AWS environment', + 'Cannot authenticate to AWS S3. Please fix your AWS environment', 'variables, or remove them completely and try again.', ]); return Result::cancelled(); } + $io->say("Unable to authenticate to AWS S3. + You can either set your credentials as environment variables and try again, + or you can continue by configuring your AWS credentials file."); $result = $this->configureAwsCredentials($io); if ($result->wasCancelled()) { return Result::cancelled(); @@ -109,7 +113,7 @@ protected function configureAwsCredentials(ConsoleIO $io): Result|ResultData $authenticated = false; while ($authenticated === false) { - $yes = $io->confirm('Do you wish to configure your AWS S3 credentials?'); + $yes = $io->confirm('Do you wish to configure your AWS S3 credentials file?'); if (!$yes) { return Result::cancelled(); } From 75cc234406c96c2861db3d0750652a50c760a0d6 Mon Sep 17 00:00:00 2001 From: K Widholm <279278+apotek@users.noreply.github.com> Date: Wed, 30 Oct 2024 16:33:30 -0400 Subject: [PATCH 5/9] Streamline s3 request configuration; remove duplicate calls --- .../Plugin/Traits/DatabaseDownloadTrait.php | 47 +++++++++++-------- 1 file changed, 27 insertions(+), 20 deletions(-) diff --git a/src/Robo/Plugin/Traits/DatabaseDownloadTrait.php b/src/Robo/Plugin/Traits/DatabaseDownloadTrait.php index 7520613..6f99b13 100644 --- a/src/Robo/Plugin/Traits/DatabaseDownloadTrait.php +++ b/src/Robo/Plugin/Traits/DatabaseDownloadTrait.php @@ -36,13 +36,19 @@ public function databaseDownload(ConsoleIO $io, string $siteName = 'default'): s { $io->title('database download.'); - $region = $this->s3RegionForSite($siteName); $authenticated = false; $objects = null; + $region = $this->s3RegionForSite($siteName); + $s3Bucket = $this->s3BucketForSite($siteName); + $s3Prefix = $this->s3PrefixForSite($siteName); + $requestConfig = ['Bucket' => $s3Bucket]; + if ($s3Prefix) { + $requestConfig[] = ['Prefix' => $s3Prefix]; + } $s3 = new S3Client(['region' => $region]); try { $io->say("Connecting to S3..."); - $objects = $s3->listObjectsV2($this->s3BucketRequestConfig($siteName)); + $objects = $s3->listObjectsV2($requestConfig); $objects->resolve(); $authenticated = $objects->info()['status'] === 200; } catch (ClientException $e) { @@ -65,7 +71,7 @@ public function databaseDownload(ConsoleIO $io, string $siteName = 'default'): s } $s3 = new S3Client(['region' => $region]); try { - $objects = $s3->listObjectsV2($this->s3BucketRequestConfig($siteName)); + $objects = $s3->listObjectsV2($requestConfig); } catch (\Exception $e) { $io->error($e->getMessage()); throw new AbortTasksException('Unable to access AWS S3. Giving up.'); @@ -91,7 +97,7 @@ public function databaseDownload(ConsoleIO $io, string $siteName = 'default'): s $this->say("Skipping download. Latest database dump file exists >>> $downloadFileName"); } else { $result = $s3->getObject([ - 'Bucket' => $this->s3BucketForSite($siteName), + 'Bucket' => $s3Bucket, 'Key' => $dbFilename, ]); stream_copy_to_stream( @@ -146,41 +152,42 @@ protected function configureAwsCredentials(ConsoleIO $io): Result|ResultData } /** - * Build S3 request configuration from sites config. + * Get S3 Bucket for site. * * @param string $siteName * The site name. * * @throws \Robo\Exception\TaskException */ - protected function s3BucketRequestConfig(string $siteName): array + protected function s3BucketForSite(string $siteName): string { - $s3ConfigArray = ['Bucket' => $this->s3BucketForSite($siteName)]; - try { - $s3KeyPrefix = $this->getSiteConfigItem('database_s3_key_prefix_string', $siteName); - $this->say("'$siteName' S3 Key prefix: '$s3KeyPrefix'"); - $s3ConfigArray['Prefix'] = $s3KeyPrefix; - } catch (TaskException) { - $this->say("No S3 Key prefix found for $siteName."); + if (!is_string($bucket = $this->getSiteConfigItem('database_s3_bucket', $siteName, true))) { + throw new TaskException($this, "database_s3_bucket value not set for '$siteName'."); } - return $s3ConfigArray; + $this->say("'$siteName' S3 bucket: $bucket"); + return $bucket; } /** - * Get S3 Bucket for site. + * Get S3 Prefix from sites config. * * @param string $siteName * The site name. * * @throws \Robo\Exception\TaskException */ - protected function s3BucketForSite(string $siteName): string + protected function s3PrefixForSite(string $siteName): string { - if (!is_string($bucket = $this->getSiteConfigItem('database_s3_bucket', $siteName))) { - throw new TaskException($this, "database_s3_bucket value not set for '$siteName'."); + try { + $s3KeyPrefix = $this->getSiteConfigItem('database_s3_key_prefix_string', $siteName); + } catch (TaskException) { + $this->say("No S3 Key prefix found for $siteName."); } - $this->say("'$siteName' S3 bucket: $bucket"); - return $bucket; + if (!empty($s3KeyPrefix) && is_string($s3KeyPrefix)) { + $this->say("'$siteName' S3 Key prefix: '$s3KeyPrefix'"); + return $s3KeyPrefix; + } + return ''; } /** From 368421d8a26d9275f5843010ad73e24e0dcc56fb Mon Sep 17 00:00:00 2001 From: DDEV User Date: Wed, 30 Oct 2024 22:08:42 +0000 Subject: [PATCH 6/9] Clean up the cancelled ResultData chain --- .../Plugin/Commands/DevelopmentModeCommands.php | 13 +++++++++---- src/Robo/Plugin/Traits/DatabaseDownloadTrait.php | 6 +++--- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/src/Robo/Plugin/Commands/DevelopmentModeCommands.php b/src/Robo/Plugin/Commands/DevelopmentModeCommands.php index 079fba4..b1db2a5 100644 --- a/src/Robo/Plugin/Commands/DevelopmentModeCommands.php +++ b/src/Robo/Plugin/Commands/DevelopmentModeCommands.php @@ -111,21 +111,23 @@ public function devRefreshAll( ): Result|ResultData { ['skip-sites' => $skipSites, 'environment-type' => $environmentType] = $options; $siteNames = $this->getAllSiteNames(); - $result = null; + $result = ResultData::message('No sites were refreshed because all sites were configured to be skipped.'); foreach ($siteNames as $siteName) { if (in_array($siteName, explode(separator: ',', string: (string) $skipSites), true)) { continue; } + /** @var Result|ResultData $result */ $result = $this->devRefreshDrupal( $io, environmentType: LocalDevEnvironmentTypes::from($environmentType), siteName: $siteName, ); - if ($result instanceof ResultData) { + if ($result->wasCancelled()) { $io->say("Cancelling the refresh for all sites."); return $result; } } + return $result; } @@ -150,7 +152,9 @@ public function databaseRefreshDdev( if (!$dbPathProvidedByUser) { $dbPath = $this->databaseDownload($io, $siteName); - if ($dbPath instanceof ResultData) { + // If we don't have a file path string here, the action was + // cancelled and we should respond with the cancellation. + if (!is_string($dbPath)) { return $dbPath; } } @@ -310,8 +314,9 @@ protected function devRefreshDrupal( $this->taskExec("composer robo theme:build $siteName") ->run(); $this->frontendDevEnable($io, $siteName, ['yes' => true]); + /** @var Result|ResultData $result */ $result = $this->databaseRefreshDdev($io, siteName: $siteName, options: ['db' => $databasePath]); - if ($result instanceof ResultData) { + if ($result->wasCancelled()) { return $result; } diff --git a/src/Robo/Plugin/Traits/DatabaseDownloadTrait.php b/src/Robo/Plugin/Traits/DatabaseDownloadTrait.php index 6f99b13..d224587 100644 --- a/src/Robo/Plugin/Traits/DatabaseDownloadTrait.php +++ b/src/Robo/Plugin/Traits/DatabaseDownloadTrait.php @@ -42,7 +42,7 @@ public function databaseDownload(ConsoleIO $io, string $siteName = 'default'): s $s3Bucket = $this->s3BucketForSite($siteName); $s3Prefix = $this->s3PrefixForSite($siteName); $requestConfig = ['Bucket' => $s3Bucket]; - if ($s3Prefix) { + if ($s3Prefix !== '') { $requestConfig[] = ['Prefix' => $s3Prefix]; } $s3 = new S3Client(['region' => $region]); @@ -67,7 +67,7 @@ public function databaseDownload(ConsoleIO $io, string $siteName = 'default'): s or you can continue by configuring your AWS credentials file."); $result = $this->configureAwsCredentials($io); if ($result->wasCancelled()) { - return Result::cancelled(); + return $result; } $s3 = new S3Client(['region' => $region]); try { @@ -183,7 +183,7 @@ protected function s3PrefixForSite(string $siteName): string } catch (TaskException) { $this->say("No S3 Key prefix found for $siteName."); } - if (!empty($s3KeyPrefix) && is_string($s3KeyPrefix)) { + if (isset($s3KeyPrefix) && is_string($s3KeyPrefix) && $s3KeyPrefix !== '') { $this->say("'$siteName' S3 Key prefix: '$s3KeyPrefix'"); return $s3KeyPrefix; } From fea16749573fc0e5f3cda5c272f37fe026e8f135 Mon Sep 17 00:00:00 2001 From: K Widholm <279278+apotek@users.noreply.github.com> Date: Thu, 31 Oct 2024 00:31:16 -0400 Subject: [PATCH 7/9] Don't run cron or cache clears on a site with no data --- .../Commands/DevelopmentModeCommands.php | 15 +++++++++--- src/Robo/Plugin/Traits/SitesConfigTrait.php | 24 ++++++++++++++++--- 2 files changed, 33 insertions(+), 6 deletions(-) diff --git a/src/Robo/Plugin/Commands/DevelopmentModeCommands.php b/src/Robo/Plugin/Commands/DevelopmentModeCommands.php index b1db2a5..8f0d288 100644 --- a/src/Robo/Plugin/Commands/DevelopmentModeCommands.php +++ b/src/Robo/Plugin/Commands/DevelopmentModeCommands.php @@ -151,7 +151,15 @@ public function databaseRefreshDdev( $dbPathProvidedByUser = $dbPath !== ''; if (!$dbPathProvidedByUser) { - $dbPath = $this->databaseDownload($io, $siteName); + try { + $dbPath = $this->databaseDownload($io, $siteName); + } catch (TaskException $e) { + $resultData = new ResultData(ResultData::EXITCODE_ERROR, $e->getMessage()); + $io->yell("$siteName: No database configured. Download/import skipped."); + // @todo: Should we run a site-install by default? The "common" + // site ends up here, but we could change that. + return $resultData; + } // If we don't have a file path string here, the action was // cancelled and we should respond with the cancellation. if (!is_string($dbPath)) { @@ -195,7 +203,8 @@ public function databaseRefreshTugboat(ConsoleIO $io): ResultData } catch (TaskException $e) { $io->yell("$siteName: No database configured. Download/import skipped."); $resultData->append($e->getMessage()); - // @todo: Should we run a site-install by default? + // @todo: Should we run a site-install by default? The "common" + // site ends up here, but we could change that. continue; } if (!is_string($dbPath) || $dbPath === '') { @@ -316,7 +325,7 @@ protected function devRefreshDrupal( $this->frontendDevEnable($io, $siteName, ['yes' => true]); /** @var Result|ResultData $result */ $result = $this->databaseRefreshDdev($io, siteName: $siteName, options: ['db' => $databasePath]); - if ($result->wasCancelled()) { + if ($result->wasCancelled() || $result->getExitCode() !== ResultData::EXITCODE_OK) { return $result; } diff --git a/src/Robo/Plugin/Traits/SitesConfigTrait.php b/src/Robo/Plugin/Traits/SitesConfigTrait.php index 0bc852f..362e93f 100644 --- a/src/Robo/Plugin/Traits/SitesConfigTrait.php +++ b/src/Robo/Plugin/Traits/SitesConfigTrait.php @@ -26,10 +26,14 @@ trait SitesConfigTrait */ public function getAllSitesConfig(): mixed { - if (!file_exists($this->sitesConfigFile)) { - throw new TaskException($this, "$this->sitesConfigFile not found."); + static $allSitesConfig = null; + if (is_null($allSitesConfig)) { + if (!file_exists($this->sitesConfigFile)) { + throw new TaskException($this, "$this->sitesConfigFile not found."); + } + $allSitesConfig = Yaml::parseFile($this->sitesConfigFile); } - return Yaml::parseFile($this->sitesConfigFile); + return $allSitesConfig; } /** @@ -108,6 +112,20 @@ public function getSiteConfigItem(string $key, string $siteName = 'default', boo return $siteConfig[$key]; } + /** + * Check if a site has an S3 bucket/has data. + * + * @throws \Robo\Exception\TaskException + */ + protected function hasData(string $siteName = 'default'): bool + { + $db = $this->getSiteConfigItem( + key: 'database_s3_bucket', + siteName: $siteName + ); + return (is_string($db) && mb_strlen($db)); + } + /** * Write sites configuration file. * From dd2655984c3e4b2fe749f1d57beba683ade84c70 Mon Sep 17 00:00:00 2001 From: K Widholm <279278+apotek@users.noreply.github.com> Date: Thu, 31 Oct 2024 00:39:48 -0400 Subject: [PATCH 8/9] phpstan fixes --- src/Robo/Plugin/Traits/SitesConfigTrait.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Robo/Plugin/Traits/SitesConfigTrait.php b/src/Robo/Plugin/Traits/SitesConfigTrait.php index 362e93f..d8ccfd2 100644 --- a/src/Robo/Plugin/Traits/SitesConfigTrait.php +++ b/src/Robo/Plugin/Traits/SitesConfigTrait.php @@ -123,7 +123,7 @@ protected function hasData(string $siteName = 'default'): bool key: 'database_s3_bucket', siteName: $siteName ); - return (is_string($db) && mb_strlen($db)); + return (is_string($db) && (bool) mb_strlen($db)); } /** From 7077872b00b1887ce3bcbaeac669ee705011ef3d Mon Sep 17 00:00:00 2001 From: K Widholm <279278+apotek@users.noreply.github.com> Date: Thu, 31 Oct 2024 12:46:33 -0400 Subject: [PATCH 9/9] Persist credentials through restarts --- src/Robo/Plugin/Traits/DatabaseDownloadTrait.php | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/Robo/Plugin/Traits/DatabaseDownloadTrait.php b/src/Robo/Plugin/Traits/DatabaseDownloadTrait.php index d224587..58711d2 100644 --- a/src/Robo/Plugin/Traits/DatabaseDownloadTrait.php +++ b/src/Robo/Plugin/Traits/DatabaseDownloadTrait.php @@ -58,7 +58,7 @@ public function databaseDownload(ConsoleIO $io, string $siteName = 'default'): s if (getenv('AWS_SECRET_ACCESS_KEY')) { $io->error([ 'Cannot authenticate to AWS S3. Please fix your AWS environment', - 'variables, or remove them completely and try again.', + 'variables in .env, or remove them completely and try again.', ]); return Result::cancelled(); } @@ -116,6 +116,7 @@ protected function configureAwsCredentials(ConsoleIO $io): Result|ResultData { $awsConfigDirPath = getenv('HOME') . '/.aws'; $awsConfigFilePath = "$awsConfigDirPath/credentials"; + $persistentCredentialsPath = '.ddev/homeadditions/.aws'; $authenticated = false; while ($authenticated === false) { @@ -131,11 +132,14 @@ protected function configureAwsCredentials(ConsoleIO $io): Result|ResultData } $awsKeyId = $io->ask("AWS Access Key ID:"); $awsSecretKey = $io->askHidden("AWS Secret Access Key:"); - $writeResult = $this->taskWriteToFile($awsConfigFilePath) + $collection = $this->collectionBuilder($io); + $collection->taskWriteToFile($awsConfigFilePath) ->line('[default]') ->line("aws_access_key_id = $awsKeyId") - ->line("aws_secret_access_key = $awsSecretKey") - ->run(); + ->line("aws_secret_access_key = $awsSecretKey"); + $collection->taskCopyDir([$awsConfigDirPath => $persistentCredentialsPath]) + ->overwrite(true); + $writeResult = $collection->run(); try { $sts = new StsClient(); $stsResponse = $sts->getCallerIdentity();