pogmommy/zellij
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

improve build/ci times (#2396)

Browse source 
- avoid building all workspace crates with `cargo x build` (only plugins and main binary)
- only set the target triple in tests for plugins
- add new profile for `cargo x run` to build with optimized dependencies => FAST plugins when developing (thanks [Bevy Book](https://bevyengine.org/learn/book/getting-started/setup/#compile-with-performance-optimizations) for the idea)
- use https://github.com/Swatinem/rust-cache to avoid rebuilding dependencies every time in ci
- split `Build & Test` job into two so they run in parallel
- hopefully improve the flaky tests situation, this also makes the e2e tests run much faster (some tests produced correct snapshots but had some logic errors causing them to loop for much longer than necessary). Add some output to the tests so it is easier to see if something goes wrong.
- remove verbose build output from e2e test build
This commit is contained in:
Thomas Linford 2023-05-03 21:16:38 +02:00 • committed by GitHub
parent ecd63a100b
commit f598ca3738
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
13 changed files with 167 additions and 117 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
.github/workflows/e2e.yml vendored
View file

@ -10,7 +10,7 @@ env:
CARGO_TERM_COLOR: always
jobs:
test:
test-e2e:
name: Build generic binary and run tests on it
runs-on: ubuntu-latest
environment: cachix
@ -37,8 +37,10 @@ jobs:
- name: Add musl target
run: rustup target add x86_64-unknown-linux-musl
#run: cargo install --debug cargo-make
- name: Build asset
run: cargo xtask ci e2e --build
- uses: Swatinem/rust-cache@v2
# ensure the target folder exists, otherwise fixtures won't be in the right place
- name: Create target folder
run: mkdir -p ${{ github.workspace }}/target
# we copy this manually into the target folder instead of mounting it because
# github actions creates the service first, and if it has a mount that is part
# of your yet unchecked out code, you cannot checkout the code after the mount

52
.github/workflows/rust.yml vendored
View file

@ -11,7 +11,7 @@ env:
jobs:
build:
name: Build & Test
name: Build
strategy:
fail-fast: false
@ -22,18 +22,27 @@ jobs:
steps:
- uses: actions/checkout@v3
- uses: actions/cache@v3
with:
path: |
~/.cargo/bin/
~/.cargo/registry
~/.cargo/git
target
key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
- name: Add WASM target
run: rustup target add wasm32-wasi
- name: Setup toolchain
run: rustup show
- uses: Swatinem/rust-cache@v2
- name: Build
run: cargo xtask build
test:
name: Test
strategy:
fail-fast: false
matrix:
os: [ubuntu-latest, macos-latest]
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v3
- name: Setup toolchain
run: rustup show
- uses: Swatinem/rust-cache@v2
- name: Test
run: cargo xtask test
@ -43,14 +52,8 @@ jobs:
steps:
- uses: actions/checkout@v3
- uses: actions/cache@v3
with:
path: |
~/.cargo/bin/
~/.cargo/registry
~/.cargo/git
target
key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
- name: Setup toolchain
run: rustup show
- name: Check Format
run: cargo xtask format --check
@ -60,13 +63,8 @@ jobs:
steps:
- uses: actions/checkout@v3
- uses: actions/cache@v3
with:
path: |
~/.cargo/bin/
~/.cargo/registry
~/.cargo/git
target
key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
- name: Setup toolchain
run: rustup show
- uses: Swatinem/rust-cache@v2
- name: Check clippy lints
run: cargo xtask clippy

6
Cargo.toml
View file

@ -45,6 +45,12 @@ members = [
".",
]
[profile.dev-opt]
inherits = "dev"
[profile.dev-opt.package."*"]
opt-level = 3
[profile.release]
lto = true
strip = true

31
src/tests/e2e/cases.rs
View file

@ -190,8 +190,7 @@ pub fn cannot_split_terminals_vertically_when_active_terminal_is_too_small() {
name: "Split pane to the right",
instruction: |mut remote_terminal: RemoteTerminal| -> bool {
let mut step_is_complete = false;
if remote_terminal.status_bar_appears() && remote_terminal.cursor_position_is(3, 2)
{
if remote_terminal.cursor_position_is(3, 2) {
remote_terminal.send_key(&PANE_MODE);
remote_terminal.send_key(&SPLIT_RIGHT_IN_PANE_MODE);
// back to normal mode after split
@ -205,7 +204,12 @@ pub fn cannot_split_terminals_vertically_when_active_terminal_is_too_small() {
name: "Make sure only one pane appears",
instruction: |remote_terminal: RemoteTerminal| -> bool {
let mut step_is_complete = false;
if remote_terminal.cursor_position_is(3, 2) {
if remote_terminal.cursor_position_is(3, 2)
//two empty lines at the bottom to make sure there is no plugin output
&& remote_terminal
.current_snapshot()
.ends_with(" \n ")
{
// ... is the truncated tip line
step_is_complete = true;
}
@ -967,7 +971,7 @@ pub fn detach_and_attach_session() {
name: "Wait for session to be attached",
instruction: |remote_terminal: RemoteTerminal| -> bool {
let mut step_is_complete = false;
if remote_terminal.status_bar_appears() && remote_terminal.cursor_position_is(3, 2)
if remote_terminal.status_bar_appears() && remote_terminal.cursor_position_is(77, 2)
{
// we're back inside the session
step_is_complete = true;
@ -1003,9 +1007,7 @@ pub fn status_bar_loads_custom_keybindings() {
name: "Wait for app to load",
instruction: |remote_terminal: RemoteTerminal| -> bool {
let mut step_is_complete = false;
if remote_terminal.cursor_position_is(3, 1)
&& remote_terminal.snapshot_contains("$ █ ││$")
&& remote_terminal.snapshot_contains("$ ") {
if remote_terminal.cursor_position_is(3, 2) && remote_terminal.tip_appears() {
step_is_complete = true;
}
step_is_complete
@ -1316,7 +1318,7 @@ pub fn mirrored_sessions() {
name: "take snapshot after",
instruction: |remote_terminal: RemoteTerminal| -> bool {
let mut step_is_complete = false;
if remote_terminal.cursor_position_is(3, 2)
if remote_terminal.cursor_position_is(63, 2)
&& remote_terminal.snapshot_contains("┐┌")
{
// cursor is back in the first tab
@ -1329,7 +1331,7 @@ pub fn mirrored_sessions() {
name: "take snapshot after",
instruction: |remote_terminal: RemoteTerminal| -> bool {
let mut step_is_complete = false;
if remote_terminal.cursor_position_is(3, 2)
if remote_terminal.cursor_position_is(63, 2)
&& remote_terminal.snapshot_contains("┐┌")
{
// cursor is back in the first tab
@ -1592,6 +1594,7 @@ pub fn multiple_users_in_different_tabs() {
let mut step_is_complete = false;
if remote_terminal.cursor_position_is(3, 2)
&& remote_terminal.tip_appears()
&& remote_terminal.snapshot_contains("Tab #1 [ ]")
&& remote_terminal.snapshot_contains("Tab #2")
&& remote_terminal.status_bar_appears()
{
@ -1608,7 +1611,7 @@ pub fn multiple_users_in_different_tabs() {
let mut step_is_complete = false;
if remote_terminal.cursor_position_is(3, 2)
&& remote_terminal.tip_appears()
&& remote_terminal.snapshot_contains("Tab #2")
&& remote_terminal.snapshot_contains("Tab #2 [ ]")
&& remote_terminal.status_bar_appears()
{
// cursor is in the newly opened second tab
@ -1649,7 +1652,9 @@ pub fn bracketed_paste() {
name: "Send pasted text followed by normal text",
instruction: |mut remote_terminal: RemoteTerminal| -> bool {
let mut step_is_complete = false;
if remote_terminal.status_bar_appears() && remote_terminal.cursor_position_is(3, 2)
if remote_terminal.status_bar_appears()
&& remote_terminal.tab_bar_appears()
&& remote_terminal.cursor_position_is(3, 2)
{
remote_terminal.send_key(&BRACKETED_PASTE_START);
remote_terminal.send_key(&TAB_MODE);
@ -1717,7 +1722,9 @@ pub fn toggle_floating_panes() {
name: "Wait for new pane to appear",
instruction: |remote_terminal: RemoteTerminal| -> bool {
let mut step_is_complete = false;
if remote_terminal.cursor_position_is(33, 7) && remote_terminal.tip_appears() {
if remote_terminal.cursor_position_is(33, 7)
&& remote_terminal.snapshot_contains("FLOATING PANES VISIBLE")
{
// cursor is in the newly opened second pane
step_is_complete = true;
}

15
src/tests/e2e/remote_runner.rs
View file

@ -299,11 +299,15 @@ impl RemoteTerminal {
x == self.cursor_x && y == self.cursor_y
}
pub fn tip_appears(&self) -> bool {
self.last_snapshot.lock().unwrap().contains("Tip:")
let snapshot = self.last_snapshot.lock().unwrap();
snapshot.contains("Tip:") || snapshot.contains("QuickNav:")
}
pub fn status_bar_appears(&self) -> bool {
self.last_snapshot.lock().unwrap().contains("Ctrl +")
}
pub fn tab_bar_appears(&self) -> bool {
self.last_snapshot.lock().unwrap().contains("Tab #1")
}
pub fn snapshot_contains(&self, text: &str) -> bool {
self.last_snapshot.lock().unwrap().contains(text)
}
@ -625,6 +629,10 @@ impl RemoteRunner {
}
pub fn run_next_step(&mut self) {
if let Some(next_step) = self.steps.get(self.current_step_index) {
println!(
"running step: {}, retries left: {}",
next_step.name, self.retries_left
);
let (cursor_x, cursor_y) = *self.cursor_coordinates.lock().unwrap();
let remote_terminal = RemoteTerminal {
cursor_x,
@ -650,6 +658,10 @@ impl RemoteRunner {
let mut retries_left = RETRIES;
let instruction = step.instruction;
loop {
println!(
"taking snapshot: {}, retries left: {}",
step.name, retries_left
);
if retries_left == 0 {
self.test_timed_out = true;
return self.last_snapshot.lock().unwrap().clone();
@ -671,6 +683,7 @@ impl RemoteRunner {
}
}
pub fn run_all_steps(&mut self) {
println!();
loop {
self.run_next_step();
if !self.steps_left() {

19
xtask/src/build.rs
View file

@ -4,7 +4,7 @@
//!
//! - [`build`]: Builds general cargo projects (i.e. zellij components) with `cargo build`
//! - [`manpage`]: Builds the manpage with `mandown`
use crate::flags;
use crate::{flags, WorkspaceMember};
use anyhow::Context;
use std::path::{Path, PathBuf};
use xshell::{cmd, Shell};
@ -22,10 +22,13 @@ pub fn build(sh: &Shell, flags: flags::Build) -> anyhow::Result<()> {
std::process::exit(1);
}
for subcrate in crate::WORKSPACE_MEMBERS.iter() {
let err_context = || format!("failed to build '{subcrate}'");
for WorkspaceMember { crate_name, .. } in crate::WORKSPACE_MEMBERS
.iter()
.filter(|member| member.build)
{
let err_context = || format!("failed to build '{crate_name}'");
if subcrate.contains("plugins") {
if crate_name.contains("plugins") {
if flags.no_plugins {
continue;
}
@ -35,10 +38,10 @@ pub fn build(sh: &Shell, flags: flags::Build) -> anyhow::Result<()> {
}
}
let _pd = sh.push_dir(Path::new(subcrate));
let _pd = sh.push_dir(Path::new(crate_name));
// Tell the user where we are now
println!();
let msg = format!(">> Building '{subcrate}'");
let msg = format!(">> Building '{crate_name}'");
crate::status(&msg);
println!("{}", msg);
@ -48,8 +51,8 @@ pub fn build(sh: &Shell, flags: flags::Build) -> anyhow::Result<()> {
}
base_cmd.run().with_context(err_context)?;
if subcrate.contains("plugins") {
let (_, plugin_name) = subcrate
if crate_name.contains("plugins") {
let (_, plugin_name) = crate_name
.rsplit_once('/')
.context("Cannot determine plugin name from '{subcrate}'")?;

20
xtask/src/ci.rs
View file

@ -90,7 +90,7 @@ fn e2e_build(sh: &Shell) -> anyhow::Result<()> {
.and_then(|cargo| {
cmd!(
sh,
"{cargo} build --verbose --release --target x86_64-unknown-linux-musl"
"{cargo} build --release --target x86_64-unknown-linux-musl"
)
.run()
.map_err(anyhow::Error::new)
@ -101,23 +101,17 @@ fn e2e_build(sh: &Shell) -> anyhow::Result<()> {
fn e2e_test(sh: &Shell, args: Vec<OsString>) -> anyhow::Result<()> {
let err_context = "failed to run E2E tests";
let _pd = sh.push_dir(crate::project_root());
e2e_build(sh).context(err_context)?;
// Build debug plugins for test binary
build::build(
sh,
flags::Build {
release: false,
no_plugins: false,
plugins_only: true,
},
)
.context(err_context)?;
let _pd = sh.push_dir(crate::project_root());
// set --no-default-features so the test binary gets built with the plugins from assets/plugins that just got built
crate::cargo()
.and_then(|cargo| {
cmd!(sh, "{cargo} test -- --ignored --nocapture --test-threads 1")
cmd!(
sh,
"{cargo} test --no-default-features -- --ignored --nocapture --test-threads 1"
)
.args(args)
.run()
.map_err(anyhow::Error::new)

10
xtask/src/clippy.rs
View file

@ -1,5 +1,5 @@
//! Handle running `cargo clippy` on the sources.
use crate::{build, flags};
use crate::{build, flags, WorkspaceMember};
use anyhow::Context;
use std::path::{Path, PathBuf};
use xshell::{cmd, Shell};
@ -21,17 +21,17 @@ pub fn clippy(sh: &Shell, _flags: flags::Clippy) -> anyhow::Result<()> {
.and_then(|_| crate::cargo())
.context("failed to run task 'clippy'")?;
for subcrate in crate::WORKSPACE_MEMBERS.iter() {
let _pd = sh.push_dir(Path::new(subcrate));
for WorkspaceMember { crate_name, .. } in crate::WORKSPACE_MEMBERS.iter() {
let _pd = sh.push_dir(Path::new(crate_name));
// Tell the user where we are now
println!();
let msg = format!(">> Running clippy on '{subcrate}'");
let msg = format!(">> Running clippy on '{crate_name}'");
crate::status(&msg);
println!("{}", msg);
cmd!(sh, "{cargo} clippy --all-targets --all-features")
.run()
.with_context(|| format!("failed to run task 'clippy' on '{subcrate}'"))?;
.with_context(|| format!("failed to run task 'clippy' on '{crate_name}'"))?;
}
Ok(())
}

3
xtask/src/flags.rs
View file

@ -69,6 +69,8 @@ xflags::xflags! {
optional --data-dir path: PathBuf
/// Enable the singlepass compiler for WASM plugins
optional --singlepass
/// Disable optimizing dependencies
optional --disable-deps-optimize
/// Arguments to pass after `cargo run --`
repeated args: OsString
}
@ -182,6 +184,7 @@ pub struct Run {
pub data_dir: Option<PathBuf>,
pub singlepass: bool,
pub disable_deps_optimize: bool,
}
#[derive(Debug)]

10
xtask/src/format.rs
View file

@ -1,5 +1,5 @@
//! Handle running `cargo fmt` on the sources.
use crate::flags;
use crate::{flags, WorkspaceMember};
use anyhow::Context;
use std::path::{Path, PathBuf};
use xshell::{cmd, Shell};
@ -11,11 +11,11 @@ pub fn format(sh: &Shell, flags: flags::Format) -> anyhow::Result<()> {
.and_then(|_| crate::cargo())
.context("failed to run task 'format'")?;
for subcrate in crate::WORKSPACE_MEMBERS.iter() {
let _pd = sh.push_dir(Path::new(subcrate));
for WorkspaceMember { crate_name, .. } in crate::WORKSPACE_MEMBERS.iter() {
let _pd = sh.push_dir(Path::new(crate_name));
// Tell the user where we are now
println!();
let msg = format!(">> Formatting '{subcrate}'");
let msg = format!(">> Formatting '{crate_name}'");
crate::status(&msg);
println!("{}", msg);
@ -24,7 +24,7 @@ pub fn format(sh: &Shell, flags: flags::Format) -> anyhow::Result<()> {
cmd = cmd.arg("--check");
}
cmd.run()
.with_context(|| format!("Failed to format '{subcrate}'"))?;
.with_context(|| format!("Failed to format '{crate_name}'"))?;
}
Ok(())
}

27
xtask/src/main.rs
View file

@ -23,18 +23,23 @@ use std::{
};
use xshell::Shell;
pub struct WorkspaceMember {
crate_name: &'static str,
build: bool,
}
lazy_static::lazy_static! {
pub static ref WORKSPACE_MEMBERS: Vec<&'static str> = vec![
"default-plugins/compact-bar",
"default-plugins/status-bar",
"default-plugins/strider",
"default-plugins/tab-bar",
"zellij-utils",
"zellij-tile-utils",
"zellij-tile",
"zellij-client",
"zellij-server",
".",
pub static ref WORKSPACE_MEMBERS: Vec<WorkspaceMember> = vec![
WorkspaceMember{crate_name: "default-plugins/compact-bar", build: true},
WorkspaceMember{crate_name: "default-plugins/status-bar", build: true},
WorkspaceMember{crate_name: "default-plugins/strider", build: true},
WorkspaceMember{crate_name: "default-plugins/tab-bar", build: true},
WorkspaceMember{crate_name: "zellij-utils", build: false},
WorkspaceMember{crate_name: "zellij-tile-utils", build: false},
WorkspaceMember{crate_name: "zellij-tile", build: false},
WorkspaceMember{crate_name: "zellij-client", build: false},
WorkspaceMember{crate_name: "zellij-server", build: false},
WorkspaceMember{crate_name: ".", build: true},
];
}

24
xtask/src/pipelines.rs
View file

@ -1,8 +1,8 @@
//! Composite pipelines for the build system.
//!
//! Defines multiple "pipelines" that run specific individual steps in sequence.
use crate::flags;
use crate::{build, clippy, format, test};
use crate::{flags, WorkspaceMember};
use anyhow::Context;
use xshell::{cmd, Shell};
@ -95,6 +95,12 @@ pub fn run(sh: &Shell, flags: flags::Run) -> anyhow::Result<()> {
let singlepass = flags.singlepass.then_some(["--features", "singlepass"]);
let profile = if flags.disable_deps_optimize {
"dev"
} else {
"dev-opt"
};
if let Some(ref data_dir) = flags.data_dir {
let data_dir = sh.current_dir().join(data_dir);
@ -105,6 +111,7 @@ pub fn run(sh: &Shell, flags: flags::Run) -> anyhow::Result<()> {
.arg("--no-default-features")
.args(["--features", "disable_automatic_asset_installation"])
.args(singlepass.iter().flatten())
.args(["--profile", profile])
.args(["--", "--data-dir", &format!("{}", data_dir.display())])
.args(&flags.args)
.run()
@ -124,6 +131,7 @@ pub fn run(sh: &Shell, flags: flags::Run) -> anyhow::Result<()> {
.and_then(|cargo| {
cmd!(sh, "{cargo} run")
.args(singlepass.iter().flatten())
.args(["--profile", profile])
.args(["--"])
.args(&flags.args)
.run()
@ -288,18 +296,18 @@ pub fn publish(sh: &Shell, flags: flags::Publish) -> anyhow::Result<()> {
}
// Publish all the crates
for member in crate::WORKSPACE_MEMBERS.iter() {
if member.contains("plugin") || member.contains("xtask") {
for WorkspaceMember { crate_name, .. } in crate::WORKSPACE_MEMBERS.iter() {
if crate_name.contains("plugin") || crate_name.contains("xtask") {
continue;
}
let _pd = sh.push_dir(project_dir.join(member));
let _pd = sh.push_dir(project_dir.join(crate_name));
loop {
let msg = format!(">> Publishing '{member}'");
let msg = format!(">> Publishing '{crate_name}'");
crate::status(&msg);
println!("{}", msg);
let more_args = match *member {
let more_args = match *crate_name {
// This is needed for zellij to pick up the plugins from the assets included in
// the released zellij-utils binary
"." => Some("--no-default-features"),
@ -314,7 +322,7 @@ pub fn publish(sh: &Shell, flags: flags::Publish) -> anyhow::Result<()> {
.context(err_context)
{
println!();
println!("Publishing crate '{member}' failed with error:");
println!("Publishing crate '{crate_name}' failed with error:");
println!("{:?}", err);
println!();
println!("Retry? [y/n]");
@ -346,7 +354,7 @@ pub fn publish(sh: &Shell, flags: flags::Publish) -> anyhow::Result<()> {
if retry {
continue;
} else {
println!("Aborting publish for crate '{member}'");
println!("Aborting publish for crate '{crate_name}'");
return Err::<(), _>(err);
}
} else {

23
xtask/src/test.rs
View file

@ -1,4 +1,4 @@
use crate::{build, flags};
use crate::{build, flags, WorkspaceMember};
use anyhow::{anyhow, Context};
use std::path::Path;
use xshell::{cmd, Shell};
@ -20,18 +20,29 @@ pub fn test(sh: &Shell, flags: flags::Test) -> anyhow::Result<()> {
)
.context(err_context)?;
for subcrate in crate::WORKSPACE_MEMBERS.iter() {
let _pd = sh.push_dir(Path::new(subcrate));
for WorkspaceMember { crate_name, .. } in crate::WORKSPACE_MEMBERS.iter() {
// the workspace root only contains e2e tests, skip it
if *crate_name == "." {
continue;
}
let _pd = sh.push_dir(Path::new(crate_name));
// Tell the user where we are now
println!("");
let msg = format!(">> Testing '{}'", subcrate);
let msg = format!(">> Testing '{}'", crate_name);
crate::status(&msg);
println!("{}", msg);
// Override wasm32-wasi target for plugins only
let cmd = if crate_name.contains("plugins") {
cmd!(sh, "{cargo} test --target {host_triple} --")
.args(&flags.args)
} else {
cmd!(sh, "{cargo} test --")
};
cmd.args(&flags.args)
.run()
.with_context(|| format!("Failed to run tests for '{}'", subcrate))?;
.with_context(|| format!("Failed to run tests for '{}'", crate_name))?;
}
Ok(())
}