From c915db91d71a498e1a75e046c6969e732d9d4642 Mon Sep 17 00:00:00 2001 From: Dante Soares Date: Fri, 18 Oct 2024 18:06:01 -0500 Subject: [PATCH] Replaced Selenium with Playwright, reorganized .gitignore, updated README --- .github/workflows/migrations.yml | 9 +- .github/workflows/tests.yml | 11 +- .gitignore | 66 ++++------- Gemfile | 4 +- Gemfile.lock | 13 +- README.md | 196 ++++++++++++++++++++++--------- package.json | 13 ++ spec/rails_helper.rb | 91 ++++---------- yarn.lock | 41 +++++++ 9 files changed, 258 insertions(+), 186 deletions(-) create mode 100644 package.json create mode 100644 yarn.lock diff --git a/.github/workflows/migrations.yml b/.github/workflows/migrations.yml index 87daadf9b..e7ee948fe 100644 --- a/.github/workflows/migrations.yml +++ b/.github/workflows/migrations.yml @@ -51,9 +51,8 @@ jobs: OXA_DB_USER: postgres OXA_DB_PASS: postgres run: | - bundle install --jobs 4 --retry 3 - bundle exec rake db:create db:schema:load db:seed --trace - bundle exec rails runner '3.times { FactoryBot.create :user }' + bin/rails db:setup + bin/rails runner '3.times { FactoryBot.create :user }' git checkout --force - # Install PR ruby version @@ -66,6 +65,4 @@ jobs: env: OXA_DB_USER: postgres OXA_DB_PASS: postgres - run: | - bundle install --jobs 4 --retry 3 - bundle exec rake db:migrate + run: bin/rails db:migrate diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 8e2dc73f0..7939cc7a1 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -35,8 +35,13 @@ jobs: - uses: ruby/setup-ruby@v1 with: bundler-cache: true + - name: Playwright + run: | + yarn + yarn playwright install - name: Test + env: + WORKERS: 4 run: | - bundle install --jobs 4 --retry 3 - bundle exec rake parallel:create parallel:load_schema parallel:seed --trace - WORKERS=4 bundle exec rake parallel:spec + bin/rake parallel:setup + bin/rake parallel:spec diff --git a/.gitignore b/.gitignore index c3e4d535f..5ed5d9507 100644 --- a/.gitignore +++ b/.gitignore @@ -4,62 +4,38 @@ # or operating system, you probably want to add a global ignore instead: # git config --global core.excludesfile ~/.gitignore_global -# Ignore bundler config. +# Bundler config and custom gem path /.bundle - -# Ignore environment variables -.env - -# Ignore the default SQLite database. -/db/*.sqlite3 -/db/*.sqlite3-journal - -# Ignore all logfiles and tempfiles. -/log/* -!/log/.keep -/tmp - -# Ignore public precompiled assets -/public/assets - -# Ignore precompiled static error pages -/public/404.html -/public/422.html -/public/500.html -/public/503.html - -# Ignore /vendor/bundle -# Ignore yaml_db database files -/db/data.yml* - -# Ignore Cucumber and RSpec failure information -cucumber_rerun.txt -.rspec_last_failures - -# Ignore byebug history files +# Byebug history files .byebug_history -# Ignore webdrivers lock file -.webdrivers_update +# Codecov coverage reports +/coverage -# Ignore Coverage reports -coverage/* +# Environment variables +.env -# Ignore Brakeman security scan results -brakeman.html +# Logfiles and tempfiles +/log +/tmp -# Ignore OS X's Desktop Service Store files +# OS X's Desktop Service Store and Unix editor temp files *.DS_Store +*~ -.generators -.rakeTasks +# Node modules (Playwright) +node_modules/ -# Ignore unix editor temp files -*~ +# Public precompiled assets and static error pages +/public/assets +/public/404.html +/public/500.html +/public/503.html -# Ignore binary created by redis-server +# Redis-server DB dump dump.rdb -.idea +# RSpec failure information +.rspec_last_failures diff --git a/Gemfile b/Gemfile index 94ac8c47d..f05970372 100644 --- a/Gemfile +++ b/Gemfile @@ -274,10 +274,10 @@ group :test do gem 'db-query-matchers' - # Run feature tests with Capybara + Selenium; choose which driver gems to use + # Run feature tests with Capybara + Playwright; choose which driver gems to use # based on test environment. gem 'capybara' - gem 'selenium-webdriver', require: false + gem 'capybara-playwright-driver' gem 'webdrivers', require: false # Testing emails diff --git a/Gemfile.lock b/Gemfile.lock index 9dd659a56..babd56cd4 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -220,6 +220,10 @@ GEM capybara-email (3.0.2) capybara (>= 2.4, < 4.0) mail + capybara-playwright-driver (0.5.2) + addressable + capybara + playwright-ruby-client (>= 1.16.0) capybara-screenshot (1.0.26) capybara (>= 1.0, < 4) launchy @@ -444,6 +448,10 @@ GEM rack-contrib (>= 1.1, < 3) railties (>= 3.0.0, < 8) method_source (1.1.0) + mime-types (3.6.0) + logger + mime-types-data (~> 3.2015) + mime-types-data (3.2024.1001) mini_mime (1.1.5) mini_portile2 (2.8.7) mini_racer (0.16.0) @@ -556,6 +564,9 @@ GEM ast (~> 2.4.1) racc pg (1.5.8) + playwright-ruby-client (1.47.0) + concurrent-ruby (>= 1.1.6) + mime-types (>= 3.0) premailer (1.27.0) addressable css_parser (>= 1.19.0) @@ -832,6 +843,7 @@ DEPENDENCIES byebug capybara capybara-email + capybara-playwright-driver capybara-screenshot cgi chronic @@ -911,7 +923,6 @@ DEPENDENCIES sass (= 3.4.19) sass-rails sd_notify - selenium-webdriver sentry-delayed_job sentry-rails sentry-ruby diff --git a/README.md b/README.md index dc3d976f4..5101f479f 100644 --- a/README.md +++ b/README.md @@ -16,13 +16,8 @@ OpenStax Accounts is a centralized User Account services provider for various Op It uses OAuth mechanisms and API keys to provide these services to OpenStax products and their users. -Accounts requires the repeatable read isolation level to work properly. If using PostgreSQL, add the following to your `postgresql.conf`: - -```Ruby -default_transaction_isolation = 'repeatable read' -``` - ## Usage — topics + * [Different ways to create a user account](#different-ways-to-create-a-user-account) * [What happens during sign up](#what-happens-during-sign-up) Which records need to be created and why. @@ -35,71 +30,92 @@ The app behavior changes depending on the value of these parameters. * [Salesforce](#salesforce) ## Concepts + * [Lev handlers](#lev-handlers) * [Lev routines](#lev-routines) * [Leads](#"leads") -## Dev Environment Setup +## Local dev environment -Accounts can be run as a normal Rails app on your machine or in a Docker container. +Accounts can be run as a normal Rails app on your machine. -### Docker +### Initial setup -```bash -# build the image -$> docker-compose build -# run it; Accounts available at localhost:2999 -$> docker-compose up -d -# run it allowing for debugging -$> docker-compose up -d && docker attach accounts_app_1 +First, install the required dependencies: libpq, postgresql and rbenv. +For example, on OS X, with Homebrew: + +```sh +brew install libpq postgresql rbenv ruby-build ``` -### Directly on your machine +Then follow any installation instructions that Homebrew may print +to activate rbenv or to configure PostgreSQL. -#### Database setup +Once all of that is done, start PostgreSQL: -If you don't have postgresql already installed, on Mac: +```sh +brew services start postgresql +``` + +Then create the PostgreSQL user that Accounts will use: ```sh -$ brew install postgresql -$ brew services start postgresql -$ psql postgres -CREATE ROLE ox_accounts WITH LOGIN; -ALTER USER ox_accounts WITH SUPERUSER; -\q +psql postgres -c "CREATE USER ox_accounts with SUPERUSER PASSWORD 'ox_accounts';" ``` -#### Running as a normal Rails app on your machine +Then clone the repository, navigate to it, and install the version of Ruby listed in .ruby-version: -First, ensure you have a Ruby version manager installed, such as [rbenv](https://github.com/rbenv/rbenv#installation) or RVM to manage your ruby versions. Then, install the Ruby version specified in the `.ruby-version` file (2.3.3 at the time of this writing, or above). +```sh +rbenv install $(cat .ruby-version) +``` -To start running Accounts in a development environment, clone the repository and then run: +Install all the required gems: ```sh -$ bundle install --without production +bundle ``` -Just like with any Rails app, you need to create, migrate, and then seed the database with some default records: +Copy `.env.example` into a new `.env` file: ```sh -$ rake db:create db:setup +cp .env.example .env ``` -Before starting the server, you'll need to create a `.env` file based off of the example: +To create a new empty database: ```sh -$ cp .env.example .env +rails db:setup ``` Now you can run: ```sh -$ rails server +rails s ``` -which will start Accounts up on port 2999. Visit http://localhost:2999. +After a short delay, Accounts will start up on port 2999. Visit http://localhost:2999. -#### Running background jobs +### If you get an error about loading the incorrect version of some gem + +Calling `rails`, `rake`, etc should execute the scripts in the `bin` directory +via rbenv shims, which should load the proper versions of your gems. +This can be verified by running the following command: + +```sh +which rails +``` + +The output should be something like `/Users/yourusername/.rbenv/shims/rails`. + +If that doesn't happen automatically, you can try one of the following: +A. Install an rbenv plugin that does this for you, for example: + https://github.com/Purple-Devs/rbenv-binstubs +B. Add `./bin` to your `$PATH` in your shell profile. + This is unsafe if you happen to browse untrusted projects with `bin` folders on your computer. +C. Manually prefix all commands with `bin/`, like `bin/rails`. +D. Manually prefix all commands with `bundle exec`. + +### Running background jobs Accounts in production runs background jobs using `delayed_job`. In the development environment, however, background jobs are run "inline", i.e. in the foreground. @@ -108,56 +124,120 @@ To actually run these jobs in the background in the development environment, set the environment variable `USE_REAL_BACKGROUND_JOBS=true` in your `.env` file and then start the `delayed_job` daemon: -`bin/rake jobs:work` +```sh +rake jobs:work +``` -## Running Specs (Automated Tests) +### Running specs (automated tests) -Using Docker, you can +To run specs, first install playwright and its drivers: ```sh -$> docker-compose run --rm app /bin/bash +yarn +yarn playwright install ``` -to drop into a container with everything installed. Then before your first test run +Then simply run: ```sh -$ /code> rake db:create -$ /code> rake db:migrate # run again if add migrations later +rake ``` -Then to run tests... +To rerun an individual test after failure, grab the command from the error message: ```sh -$ /code> rspec -$ /code> rspec ./spec/features -$ /code> rspec ./spec/some/specific_spec.rb -$ /code> rspec ./spec/some/specific_spec.rb:42 # the spec at line 42 +rspec ./spec/path/to/some_spec.rb:linenumber +``` + +You may have to rerun `yarn playwright install` from time to time to update your browser drivers. + +#### Debugging spec failures + +##### Debug console + +If you need to get a debugger into some test or application code, +an easy way is to activate byebug like this: + +```sh +export DEBUGGER=byebug ``` -Or if you want to install directly on your machine... +Then add a `debugger` statement where you want a breakpoint, save the file, +then just rerun the test as normal and you should get a debugger console: -Specs require phantomjs. On Mac: ```sh -$ brew install phantomjs +rspec ./spec/path/to/some_spec.rb:linenumber ``` -To run specs, +##### Debugging exceptions thrown in feature specs + +When running feature specs, the default behavior is for exceptions to be rescued and +nice error pages to be shown. This can make debugging difficult if you're not expecting an error. +To not rescue exceptions, run with `RAISE=true`: ```sh -$ rake +RAISE=true rspec ./spec/path/to/some_spec.rb:linenumber ``` -When running feature specs, the default behavior is for exceptions to be rescued and nice error pages to be shown. This can make debugging difficult if you're not expecting an error. To not rescue exceptions, do: +The exception and backtrace may still only be displayed in `page.body`, +so you may need to print that out. + +##### Debugging exceptions thrown in gems + +You can get a backtrace that includes gem code with: ```sh -$ RAISE=true rspec +BACKTRACE=true rspec ./spec/path/to/some_spec.rb:linenumber +``` + +To test changes to gems, clone the relevant gem repo and then use the following in the Gemfile: + +```rb +gem 'some-gem', path: './some-gem' +``` + +Or to load the gem from GitHub: + +```rb +gem 'some-gem', github: 'openstax/some-gem', ref: 'commit-or-branch' ``` -If you encounter issues running features specs, check the version of chromedriver you have installed. Version 2.38 is known to work. +## Docker (we have not used this in a while, it may or may not work) + +### Initial setup + +```bash +# build the image +$> docker-compose build +# run it; Accounts available at localhost:2999 +$> docker-compose up -d +# run it allowing for debugging +$> docker-compose up -d && docker attach accounts_app_1 +``` -## Debugging +### Running specs (automated tests) -Set `DEBUGGER=byebug` to use byebug (e.g. in `.env`), otherwise defaults to the VS Code debugger. +Using Docker, you can + +```sh +$> docker-compose run --rm app /bin/bash +``` + +to drop into a container with everything installed. Then before your first test run + +```sh +$ /code> rake db:create +$ /code> rake db:migrate # run again if add migrations later +``` + +Then to run tests... + +```sh +$ /code> rspec +$ /code> rspec ./spec/features +$ /code> rspec ./spec/some/specific_spec.rb +$ /code> rspec ./spec/some/specific_spec.rb:42 # the spec at line 42 +``` # Usage — topics diff --git a/package.json b/package.json new file mode 100644 index 000000000..c02063058 --- /dev/null +++ b/package.json @@ -0,0 +1,13 @@ +{ + "name": "accounts", + "version": "1.0.0", + "main": "index.js", + "repository": "https://github.com/openstax/accounts.git", + "author": "OpenStax", + "license": "AGPL-3.0-or-later", + "devDependencies": { + "@playwright/test": "^1.48.1", + "@types/node": "^22.7.6" + }, + "scripts": {} +} diff --git a/spec/rails_helper.rb b/spec/rails_helper.rb index 68e6004df..4b1cb59bc 100644 --- a/spec/rails_helper.rb +++ b/spec/rails_helper.rb @@ -31,86 +31,35 @@ Config for Capybara """ # https://robots.thoughtbot.com/headless-feature-specs-with-chrome -Capybara.register_driver :selenium_chrome do |app| - options = Selenium::WebDriver::Chrome::Options.new args: [ 'lang=en' ] - - Capybara::Selenium::Driver.new app, browser: :chrome, options: options +Capybara.register_driver :playwright_chromium do |app| + Capybara::Playwright::Driver.new app, browser_type: :chromium, args: [ '--lang=en' ] end -# no-sandbox and disable-gpu are required for Chrome to work with Travis -Capybara.register_driver :selenium_chrome_headless do |app| - options = Selenium::WebDriver::Chrome::Options.new args: [ - 'no-sandbox', 'headless', 'disable-dev-shm-usage', - 'disable-gpu', 'disable-extensions', 'disable-infobars' +Capybara.register_driver :playwright_chromium_headless do |app| + Capybara::Playwright::Driver.new app, browser_type: :chromium, args: [ + '--disable-dev-shm-usage', + '--disable-extensions', + '--disable-gpu', + '--disable-infobars', + '--headless', + '--lang=en', + '--no-sandbox', ] - - Capybara::Selenium::Driver.new app, browser: :chrome, options: options end -# The webdrivers gem uses selenium-webdriver. Our docker approach needs selenium-webdriver -# but gets upset if webdriver is loaded. So in the Gemfile, we `require: false` both of -# these and explicitly require them based on where we're running. We also only register -# the docker flavor of the driver if we are indeed running in docker. +if EnvUtilities.load_boolean(name: 'HEADLESS', default: true) + # Run the feature specs in a full browser (note, this takes over your computer's focus) + Capybara.javascript_driver = :playwright_chromium_headless +else + Capybara.javascript_driver = :playwright_chromium +end CAPYBARA_PROTOCOL = DEV_PROTOCOL +CAPYBARA_HOST = DEV_HOST +CAPYBARA_HOST_REGEX = /\A(.*\.)?#{Regexp.escape CAPYBARA_HOST.sub('*.', '').chomp('.*')}\z/ CAPYBARA_PORT = ENV.fetch('PORT', DEV_PORT) -if in_docker? - require 'selenium-webdriver' - - Capybara.register_driver :selenium_chrome_headless_in_docker do |app| - chrome_capabilities = ::Selenium::WebDriver::Remote::Capabilities.chrome( - 'goog:chromeOptions' => { 'args': %w[no-sandbox headless disable-gpu] } - ) - - Capybara::Selenium::Driver.new(app, - browser: :remote, - url: ENV['HUB_URL'], - desired_capabilities: chrome_capabilities) - end - - Capybara.javascript_driver = :selenium_chrome_headless_in_docker - - # Normally the Capybara host is 'localhost', but within Docker it may not be. - CAPYBARA_HOST = IPSocket.getaddress(Socket.gethostname) - - Capybara.asset_host = "#{CAPYBARA_PROTOCOL}://#{CAPYBARA_HOST}:#{CAPYBARA_PORT}" - Capybara.app_host = "#{CAPYBARA_PROTOCOL}://#{CAPYBARA_HOST}:#{CAPYBARA_PORT}" - Capybara.server_host = CAPYBARA_HOST - Capybara.server_port = CAPYBARA_PORT -else - require 'webdrivers/chromedriver' - - # Use a lockfile so we don't get errors due to downloading webdrivers multiple times concurrently - File.open('.webdrivers_update', File::RDWR|File::CREAT, 0640) do |file| - file.flock File::LOCK_EX - update_time = Time.parse(file.read) rescue nil - current_time = Time.current - - if update_time.nil? || current_time - update_time > 60 - Webdrivers::Chromedriver.update - - file.rewind - file.write current_time.iso8601 - file.flush - file.truncate file.pos - end - ensure - file.flock File::LOCK_UN - end - - if EnvUtilities.load_boolean(name: 'HEADLESS', default: true) - # Run the feature specs in a full browser (note, this takes over your computer's focus) - Capybara.javascript_driver = :selenium_chrome_headless - else - Capybara.javascript_driver = :selenium_chrome - end - - CAPYBARA_HOST = DEV_HOST - CAPYBARA_HOST_REGEX = /\A(.*\.)?#{Regexp.escape CAPYBARA_HOST.sub('*.', '').chomp('.*')}\z/ - - Capybara.asset_host = "#{CAPYBARA_PROTOCOL}://#{CAPYBARA_HOST}:#{CAPYBARA_PORT}" -end +Capybara.asset_host = "#{CAPYBARA_PROTOCOL}://#{CAPYBARA_HOST}:#{CAPYBARA_PORT}" Capybara.server = :puma, { Silent: true } # To clean up your test output diff --git a/yarn.lock b/yarn.lock new file mode 100644 index 000000000..0cf7144da --- /dev/null +++ b/yarn.lock @@ -0,0 +1,41 @@ +# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. +# yarn lockfile v1 + + +"@playwright/test@^1.48.1": + version "1.48.1" + resolved "https://registry.yarnpkg.com/@playwright/test/-/test-1.48.1.tgz#343e710fcf2e559529e3ec8d7782e09f325b9396" + integrity sha512-s9RtWoxkOLmRJdw3oFvhFbs9OJS0BzrLUc8Hf6l2UdCNd1rqeEyD4BhCJkvzeEoD1FsK4mirsWwGerhVmYKtZg== + dependencies: + playwright "1.48.1" + +"@types/node@^22.7.6": + version "22.7.6" + resolved "https://registry.yarnpkg.com/@types/node/-/node-22.7.6.tgz#3ec3e2b071e136cd11093c19128405e1d1f92f33" + integrity sha512-/d7Rnj0/ExXDMcioS78/kf1lMzYk4BZV8MZGTBKzTGZ6/406ukkbYlIsZmMPhcR5KlkunDHQLrtAVmSq7r+mSw== + dependencies: + undici-types "~6.19.2" + +fsevents@2.3.2: + version "2.3.2" + resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-2.3.2.tgz#8a526f78b8fdf4623b709e0b975c52c24c02fd1a" + integrity sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA== + +playwright-core@1.48.1: + version "1.48.1" + resolved "https://registry.yarnpkg.com/playwright-core/-/playwright-core-1.48.1.tgz#5fe28fb9a9326dae88d4608c35e819163cceeb23" + integrity sha512-Yw/t4VAFX/bBr1OzwCuOMZkY1Cnb4z/doAFSwf4huqAGWmf9eMNjmK7NiOljCdLmxeRYcGPPmcDgU0zOlzP0YA== + +playwright@1.48.1: + version "1.48.1" + resolved "https://registry.yarnpkg.com/playwright/-/playwright-1.48.1.tgz#2a920cfbec4572c84789e757d8b044baaed49435" + integrity sha512-j8CiHW/V6HxmbntOfyB4+T/uk08tBy6ph0MpBXwuoofkSnLmlfdYNNkFTYD6ofzzlSqLA1fwH4vwvVFvJgLN0w== + dependencies: + playwright-core "1.48.1" + optionalDependencies: + fsevents "2.3.2" + +undici-types@~6.19.2: + version "6.19.8" + resolved "https://registry.yarnpkg.com/undici-types/-/undici-types-6.19.8.tgz#35111c9d1437ab83a7cdc0abae2f26d88eda0a02" + integrity sha512-ve2KP6f/JnbPBFyobGHuerC9g1FYGn/F8n1LWTwNxCEzd6IfqTwUQcNXgEtmmQ6DlRrC1hrSrBnCZPokRrDHjw==